1 from queue import Queue, Empty
2 import threading
3 import requests
4 from pyquery import PyQuery
5 import time
6
7 index = 'https://www.chazidian.com'
8 list_page = index + '/xiaohua{}/{}'
9
10 headers = {
11 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36'
12 }
13 timeout = 1
14 thread_quantity = 5
15
16
17 def get_url(queue):
18 for i in range(1, 75 + 1):
19 r = requests.get(list_page.format('', str(i)), headers=headers)
20 pq = PyQuery(r.text)
21 doc = pq('div.arctcot h3 a')
22 for j in doc.items():
23 queue.put(index + j.attr('href'))
24 # print(doc)
25 pass
26
27
28 def get_content(queue):
29 try:
30 while True:
31 url = queue.get(timeout=timeout)
32 if 'https://' in url:
33 r = requests.get(url, headers=headers)
34 pq = PyQuery(r.text)
35 doc = pq('div.arctcot')
36 title = doc('a').text()
37 content = doc('div.article_detail').text()
38 img = doc('div.article_detail img').attr('src')
39 # print(title.text())
40 # print(content.text())
41 if title and img:
42 if not 'http://' in img:
43 print(url)
44 print({title: index + img})
45 elif title and content:
46 if not (content in title):
47 print(url)
48 print({title: content})
49 except Empty:
50 print('-' * 100)
51 print('抓取完毕')
52
53
54 def main():
55 queue_ = Queue(maxsize=1000)
56 list_ = threading.Thread(target=get_url, args=(queue_,))
57 list_.start()
58 if True:
59 for i in range(thread_quantity):
60 content = threading.Thread(target=get_content, args=(queue_,))
61 content.start()
62 content.join()
63 list_.join()
64
65
66 if __name__ == '__main__':
67 start = time.time()
68 main()
69 end = time.time()
70 print('用时: ', end - start - timeout)
