refactor crawler

lzjun567 · lzjun567 · commit ac103b225830 · 2017-02-20T18:17:44.000+08:00
diff --git a/heart/test.py b/heart/test.py
@@ -0,0 +1,4 @@
+# -*- coding:utf-8 -*-
+import jieba
+
+print " ".join(jieba.cut(u"::	[AVI/329M][C0930-hitozuma1007] &#22338;&#20117;&#38634;&#24693; Yukie Sakai  (308123)"))
diff --git a/pdf/crawler1.py b/pdf/crawler1.py
@@ -1,8 +1,9 @@
 # coding=utf-8
+import logging
 import os
 import re
 import time
-import logging
+
 import pdfkit
 import requests
 from bs4 import BeautifulSoup
@@ -21,6 +22,22 @@
 """
 
 
+class Crawler(object):
+    """
+    &#29228;&#34411;&#22522;&#31867;&#65292;&#25152;&#26377;&#29228;&#34411;&#37117;&#24212;&#35813;&#32487;&#25215;&#27492;&#31867;
+    """
+    name = None
+
+    def __init__(self, name=None, **kwargs):
+        if name is not None:
+            self.name = name
+        elif not getattr(self, 'name', None):
+            raise ValueError("%s must have a name" % type(self).__name__)
+
+    def parse(self):
+        raise NotImplementedError
+
+
 def parse_url_to_html(url, name):
     """
     &#35299;&#26512;URL&#65292;&#36820;&#22238;HTML&#20869;&#23481;
@@ -51,7 +68,8 @@ def func(m):
                 rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)
                 return rtn
             else:
-                return m.group(1)+m.group(2)+m.group(3)
+                return m.group(1) + m.group(2) + m.group(3)
+
         html = re.compile(pattern).sub(func, html)
         html = html_template.format(content=html)
         html = html.encode("utf-8")
@@ -120,4 +138,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    Crawler().parse()
diff --git a/pdf/requirement.txt b/pdf/requirement.txt
@@ -0,0 +1,3 @@
+requests==2.12.4
+beautifulsoup4==4.5.3
+pdfkit==0.6.1

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +# -*- coding:utf-8 -*-
 +import jieba
++
 +print " ".join(jieba.cut(u"::	[AVI/329M][C0930-hitozuma1007] 坂井雪恵 Yukie Sakai  (308123)"))
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+requests==2.12.4`
	`2`	`+beautifulsoup4==4.5.3`
	`3`	`+pdfkit==0.6.1`