蜂鸟网爬虫1

案例6/demo.py

+from six import indexbytes
+from http_help import R  # 这个文件自己去上篇博客找，或者去 github 找
+import threading
+import time
+import json
+import re
+
+img_list = []
+imgs_lock = threading.Lock()  # 图片操作锁
+
+
+# 生产者类
+class Product(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.__headers = {"Referer": "http://image.fengniao.com/",
+                          "Host": "image.fengniao.com",
+                          "X-Requested-With": "XMLHttpRequest"
+                          }
+        # 链接模板
+        self.__start = "http://image.fengniao.com/list.php?action=getList&class_id=192&sub_classid=1587&page={}&not_in_id=0"
+        self.__res = R(headers=self.__headers)
+
+    def run(self):
+
+        # 因为不知道循环次数，所有采用while循环
+        index = 2  # 起始页码设置为1
+
+        while True:
+            url = self.__start.format(index)
+            print("开始操作:{}".format(url))
+            index += 1
+
+            content = self.__res.get_content(url, charset="gbk")
+
+            if content is None:
+                print("数据可能已经没有了====")
+                continue
+
+            time.sleep(3)  # 睡眠3秒
+            json_content = json.loads(content)
+
+            if json_content["status"] == 1:
+                for item in json_content["data"]:
+                    title = item["title"]
+                    child_url = item["url"]   # 获取到链接之后
+
+                    img_content = self.__res.get_content(
+                        child_url, charset="gbk")
+
+                    pattern = re.compile('"pic_url_1920_b":"(.*?)"')
+                    imgs_json = pattern.findall(img_content)
+                    if len(imgs_json) > 0:
+
+                        if imgs_lock.acquire():
+                            # 这个地方，我用的是字典+列表的方式，主要是想后面生成文件夹用，你可以进行改造
+                            img_list.append(
+                                {"title": title, "urls": imgs_json})
+                            imgs_lock.release()
+
+
+# 消费者
+class Consumer(threading.Thread):
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.__res = R()
+
+    def run(self):
+
+        while True:
+            if len(img_list) <= 0:
+                continue  # 进入下一次循环
+
+            if imgs_lock.acquire():
+
+                data = img_list[0]
+                del img_list[0]  # 删除第一项
+
+                imgs_lock.release()
+
+            urls = [url.replace("\\", "") for url in data["urls"]]
+
+            # 创建文件目录
+            for item_url in urls:
+                try:
+                    file = self.__res.get_file(item_url)
+                    # 记得在项目根目录先把fengniaos文件夹创建完毕
+                    with open("./fengniaos/{}".format(str(time.time())+".jpg"), "wb+") as f:
+                        f.write(file)
+                except Exception as e:
+                    print(e)
+if __name__ == '__main__':
+    p = Product()
+    p.start()
+
+    c = Consumer()
+    c.start()
\ No newline at end of file

案例6/http_help.py

+import requests
+from retrying import retry
+import random
+import datetime
+
+class R:
+	# 类的初始化方法
+    def __init__(self,method="get",params=None,headers=None,cookies=None):
+        self.__method = method
+        myheaders = self.get_headers()
+        if headers is not None:
+            myheaders.update(headers)
+        self.__headers = myheaders
+        self.__cookies = cookies
+        self.__params = params
+
+
+    def get_headers(self):
+        user_agent_list = [ \
+            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1" \
+            "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", \
+            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", \
+            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", \
+            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", \
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", \
+            "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", \
+            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", \
+            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", \
+            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", \
+            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
+        ]
+        UserAgent = random.choice(user_agent_list)
+        headers = {'User-Agent': UserAgent}
+        return headers
+ 
+
+    @retry(stop_max_attempt_number=3)
+    def __retrying_requests(self,url):
+        if self.__method == "get":
+            response = requests.get(url,headers=self.__headers,cookies=self.__cookies,timeout=3)
+        else:
+            response = requests.post(url,params=self.__params,headers=self.__headers,cookies=self.__cookies,timeout=3)
+        return response.content
+ 
+
+    # get请求
+    def get_content(self,url,charset="utf-8"):
+        try:
+            html_str = self.__retrying_requests(url).decode(charset)
+        except:
+            html_str = None
+        return html_str
+
+    def get_file(self,file_url):
+        try:
+            file = self.__retrying_requests(file_url)
+        except:
+            file = None
+        return file
+