Skip to content

Commit a83f519

Browse files
lzjunlzjun
committed
udapte
1 parent 5b81319 commit a83f519

3 files changed

Lines changed: 209 additions & 109 deletions

File tree

‎weixin_pdf/test2.py‎

Lines changed: 29 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,20 @@
1-
# coding=utf-8
2-
from __future__ import unicode_literals
3-
4-
import logging
51
import os
6-
import re
72
import time
83

9-
try:
10-
from urllib.parse import urlparse # py3
11-
except:
12-
from urlparse import urlparse # py2
13-
144
import pdfkit
155
import requests
16-
from bs4 import BeautifulSoup
17-
18-
html_template = """
19-
<!DOCTYPE html>
20-
<html lang="en">
21-
<head>
22-
<meta charset="UTF-8">
23-
</head>
24-
<body>
25-
{content}
26-
</body>
27-
</html>
28-
29-
"""
6+
7+
options = {
8+
'page-size': 'Letter',
9+
'margin-top': '0.75in',
10+
'margin-right': '0.75in',
11+
'margin-bottom': '0.75in',
12+
'margin-left': '0.75in',
13+
'encoding': "UTF-8",
14+
'outline-depth': 10,
15+
}
16+
17+
3018
headers = {
3119
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36",
3220
"Host": "mp.weixin.qq.com",
@@ -40,15 +28,13 @@ class Crawler(object):
4028
"""
4129
name = None
4230

43-
def __init__(self, name, start_url):
31+
def __init__(self, name):
4432
"""
4533
初始化
4634
:param name: 将要被保存为PDF的文件名称
4735
:param start_url: 爬虫入口URL
4836
"""
4937
self.name = name
50-
self.start_url = start_url
51-
self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))
5238

5339
@staticmethod
5440
def request(url, **kwargs):
@@ -59,44 +45,26 @@ def request(url, **kwargs):
5945
response = requests.get(url, **kwargs)
6046
return response
6147

62-
def parse_menu(self, response):
63-
"""
64-
从response中解析出所有目录的URL链接
65-
"""
66-
raise NotImplementedError
6748

6849
def parse_body(self, response):
6950
"""
70-
解析正文,由子类实现
51+
解析正文
7152
:param response: 爬虫返回的response对象
72-
:return: 返回经过处理的html正文文本
53+
:return: 返回处理后的html文本
7354
"""
74-
raise NotImplementedError
55+
content = response.text.replace("data-src", 'src')
56+
return content
7557

76-
def run(self):
58+
def run(self, urls):
7759
start = time.time()
78-
options = {
79-
'page-size': 'Letter',
80-
'margin-top': '0.75in',
81-
'margin-right': '0.75in',
82-
'margin-bottom': '0.75in',
83-
'margin-left': '0.75in',
84-
'encoding': "UTF-8",
85-
'custom-header': [
86-
('Accept-Encoding', 'gzip')
87-
],
88-
'cookie': [
89-
('cookie-name1', 'cookie-value1'),
90-
('cookie-name2', 'cookie-value2'),
91-
],
92-
'outline-depth': 10,
93-
}
9460
htmls = []
95-
for index, url in enumerate(self.parse_menu(None)):
96-
html = self.parse_body(self.request(url, headers=headers))
61+
for index, url in enumerate(urls):
9762
f_name = ".".join([str(index), "html"])
98-
with open(f_name, 'wb') as f:
99-
f.write(html)
63+
if not os.path.exists(f_name):
64+
# 方便失败后重试
65+
html = self.parse_body(self.request(url, headers=headers))
66+
with open(f_name, 'w', encoding="utf-8") as f:
67+
f.write(html)
10068
htmls.append(f_name)
10169

10270
pdfkit.from_file(htmls, self.name + ".pdf", options=options)
@@ -106,58 +74,10 @@ def run(self):
10674
print(u"总共耗时：%f 秒" % total_time)
10775

10876

109-
class LiaoxuefengPythonCrawler(Crawler):
110-
"""
111-
廖雪峰Python3教程
112-
"""
113-
114-
def parse_menu(self, response):
115-
"""
116-
解析目录结构,获取所有URL目录列表
117-
:param response 爬虫返回的response对象
118-
:return: url生成器
119-
"""
120-
htmls = [
121-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371855&idx=1&sn=2f15cdfe0b5896b2df7705154571c1b5&chksm=be9ccc5b89eb454d18bb646f3e348a100fe2af38327c947999d047d66d1663f7892afe0044b9&scene=27#wechat_redirect",
122-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371847&idx=1&sn=5cfac634f11913f99c2d1587d0433a53&chksm=be9ccc5389eb45453e0576ee0d949472f61fbe9daea46183494ad1b8f91a367bc36adcf017f2&scene=27#wechat_redirect",
123-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371826&idx=1&sn=7a76628812a3949b0ec1a308ceec3051&chksm=be9ccda689eb44b0f3b0f8327fd4626ea3f1b888b4e755c4c5a2dbeb88954138b73b61887177#rd",
124-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371809&idx=1&sn=c7be295675b11f94b6b33b8c996e9aa9&chksm=be9ccdb589eb44a3cd215766ba3460415e54b9cfa763409689eea789cd47195e1b689f3a09ca&scene=27#wechat_redirect",
125-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371794&idx=1&sn=5afc8a0e65cbe939467ce87a1c68f547&chksm=be9ccd8689eb4490986dc5be626c8c755406926262da94b7014d8aad29d631cd62ec14a907a4&scene=27#wechat_redirect",
126-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371769&idx=1&sn=2f20c33c2454c5ec68792c081d5998f1&chksm=be9ccded89eb44fbda608edc9103d4d62430bb0adf182383e8c20474aa354faad07b24952db2#rd",
127-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371760&idx=1&sn=0ebd9f172de570ec3b10ab0ed5aa9c63&chksm=be9ccde489eb44f2549fcf9ae0c555109d952dc6db8e8e7c4024ae64372016c9cc78e63f13ac&scene=27#wechat_redirect",
128-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371704&idx=1&sn=0145ca19dabc03cda6ac860caf0c2dd0&chksm=be9ccd2c89eb443ab809e05484f58da87aa1ba2b0e678fbd2a06b0ba2c4df600d001e86d6838&scene=27#wechat_redirect",
129-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371678&idx=1&sn=de791b0341e9075064b02124af1fe864&chksm=be9ccd0a89eb441c54837140cfa0c3070431dc2c4fa0fb8a64a91419dd7923490273dff305c8&scene=27#wechat_redirect",
130-
"http://mp.weixin.qq.com/s?__biz=MjM5MzgyODQxMQ==&mid=2650371650&idx=1&sn=48fd3821d43dd7374813618be0b3d42c&chksm=be9ccd1689eb4400fa0b42e9ec6676607caf93916071ca909715d05e2d4246af5cf81a8c17f3&scene=27#wechat_redirect"
131-
]
132-
return htmls
133-
134-
def parse_body(self, response):
135-
"""
136-
解析正文
137-
:param response: 爬虫返回的response对象
138-
:return: 返回处理后的html文本
139-
"""
140-
try:
141-
return response.content
142-
# soup = BeautifulSoup(response.content, 'html.parser')
143-
# body = soup
144-
#
145-
# # 加入标题, 居中显示
146-
# title = soup.find('h4').get_text()
147-
# center_tag = soup.new_tag("center")
148-
# title_tag = soup.new_tag('h1')
149-
# title_tag.string = title
150-
# center_tag.insert(1, title_tag)
151-
# body.insert(1, center_tag)
152-
#
153-
# html = str(body)
154-
# html = html.encode("utf-8")
155-
# return html
156-
except Exception as e:
157-
logging.error("解析错误", exc_info=True)
77+
if __name__ == '__main__':
78+
crawler = Crawler("歪理邪说")
15879

80+
with open("url.txt") as f:
81+
urls = f.readlines()
15982

160-
if __name__ == '__main__':
161-
start_url = "http://www.liaoxuefeng.com/wiki/0013739516305929606dd18361248578c67b8067c8c017b000"
162-
crawler = LiaoxuefengPythonCrawler("廖雪峰Git", start_url)
163-
crawler.run()
83+
crawler.run(urls)

‎中文转二进制.py‎

Lines changed: 11 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)