python-多线程队列爬虫

发表于 更新于 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

# coding:utf-8
__author__ = 'Administrator'

from urllib import *
from bs4 import BeautifulSoup
import socket
import threading
import Queue
import time

url = 'http://libopac.btbu.edu.cn:8080/opac/browseByCategory?pager.offset='
socket.setdefaulttimeout(3)  # 3秒超时
maxnum = 634309  # 图书总数
print "\n"

q = Queue.Queue()  # 构造一个队列,它可以put入任务和get出任务,先进先出
# 比如放入1,2,3,4,5    则你取出来的时候也是取出来1,2,3,4,5这样的顺序
f = open('library3.txt', 'a')

def getlinks():
    # 线程1,获取链接的线程
    offset = 192850  # 调这个值可以断点续传
    while offset < maxnum:
        try:
            html = urlopen(url + str(offset)).read().decode('utf-8')
            soup = BeautifulSoup(html)
            links = soup.findAll(target='_blank')  # 寻找所有的链接
            onetask = (offset, links)  # 构建一个(offset, links)结构作为task,其实就是(下到第几本书了, 链接们是什么)
            q.put(onetask)  # 放入queue中
            offset += 25
            print "\r" + str(offset) + "/" + str(maxnum) + "    " + str(offset * 100.0 / maxnum) + "%   第" + str(
                offset / 25) + "页----获取列表"
        except IOError:
            # print 'timeout'
            pass
        except Exception, e:
            print e

class MyThread(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue  # 初始化构造函数,传入queue

    def run(self):
        while True:
            myonetask = self.queue.get()  # 获取任务
            myoffset, mylinks = myonetask  # 解析任务
            print "\r" + str(myoffset) + "/" + str(maxnum) + "    " + str(myoffset * 100.0 / maxnum) + "%   第" \
                  + str(myoffset / 25) + "页----收到任务"
            mywritecontent = ""
            for mylink in mylinks:
                always = True  # 建立一个循环下载,
                while always:
                    try:
                        myurl2 = 'http://libopac.btbu.edu.cn:8080' + mylink['href']
                        # print link.text, url2
                        mywritecontent += myurl2.encode('utf-8', 'ignore') + "\n"
                        mywritecontent += mylink.text.encode('utf-8', 'ignore') + "\n"
                        # 标题信息和内容
                        myhtml2 = urlopen(myurl2).read()
                        mysoup2 = BeautifulSoup(myhtml2)
                        mywritecontent += mysoup2.find(id='detailsTable').text.encode('utf-8', 'ignore') + "\n"
                        # 获得馆藏信息    getHoldingsInformation
                        myjson3 = urlopen(myurl2.replace('book', 'book/getHoldingsInformation')).read()
                        mywritecontent += myjson3 + "\n\n"
                        always = False  # 下载成功之后才取消循环
                    except IOError:
                        pass  # 如果是IOError就是超时了,pass掉(相当于C++的;空语句)
                    except Exception:
                        # 为了防止某些失效链接,我们将跳过失败的书籍
                        # 比如http://libopac.btbu.edu.cn:8080/opac/book/627232
                        always = False
            print "\r" + str(myoffset) + "/" + str(maxnum) + "    " + str(myoffset * 100.0 / maxnum) + "%   第" \
                  + str(myoffset / 25) + "页----下载完成----" + str(myoffset + 25)
            mywritecontent += "\n==========第" + str(myoffset / 25) + "页==========\n\n"
            f.write(mywritecontent)
            f.flush()  # 写出到文件
            time.sleep(0.01)  # 线程累了 休息一下

t1 = threading.Thread(target=getlinks)  # 获取链接的线程
t1.start()

for asd in range(10):
    t2 = MyThread(q)  # 从队列里下载数据的线程
    t2.start()