Python实现多线程抓取网页功能实例详解

 ''' Created on 2012-3-13 Get Page using GET method Default using HTTP Protocol , http port 80 @author: xiaojay ''' import socket import statistics import datetime import threading socket.setdefaulttimeout(statistics.timeout) class Error404(Exception): '''Can not find the page.''' pass class ErrorOther(Exception): '''Some other exception''' def __init__(self,code): #print 'Code :',code pass class ErrorTryTooManyTimes(Exception): '''try too many times''' pass def downPage(hostname ,filename , trytimes=0): try : #To avoid too many tries .Try times can not be more than max_try_times if trytimes >= statistics.max_try_times : raise ErrorTryTooManyTimes except ErrorTryTooManyTimes : return statistics.RESULTTRYTOOMANY,hostname+filename try: s = socket.socket(socket.AF_INET,socket.SOCK_STREAM) #DNS cache if statistics.DNSCache.has_key(hostname): addr = statistics.DNSCache[hostname] else: addr = socket.gethostbyname(hostname) statistics.DNSCache[hostname] = addr #connect to http server ,default port 80 s.connect((addr,80)) msg = 'GET '+filename+' HTTP/1.0\r\n' msg += 'Host: '+hostname+'\r\n' msg += 'User-Agent:xiaojay\r\n\r\n' code = '' f = None s.sendall(msg) first = True while True: msg = s.recv(40960) if not len(msg): if f!=None: f.flush() f.close() break # Head information must be in the first recv buffer if first: first = False headpos = msg.index("\r\n\r\n") code,other = dealwithHead(msg[:headpos]) if code=='200': #statistics.fetched_url += 1 f = open('pages/'+str(abs(hash(hostname+filename))),'w') f.writelines(msg[headpos+4:]) elif code=='301' or code=='302': #if code is 301 or 302 , try down again using redirect location if other.startswith("http") : hname, fname = parse(other) downPage(hname,fname,trytimes+1)#try again else : downPage(hostname,other,trytimes+1) elif code=='404': raise Error404 else : raise ErrorOther(code) else: if f!=None :f.writelines(msg) s.shutdown(socket.SHUT_RDWR) s.close() return statistics.RESULTFETCHED,hostname+filename except Error404 : return statistics.RESULTCANNOTFIND,hostname+filename except ErrorOther: return statistics.RESULTOTHER,hostname+filename except socket.timeout: return statistics.RESULTTIMEOUT,hostname+filename except Exception, e: return statistics.RESULTOTHER,hostname+filename def dealwithHead(head): '''deal with HTTP HEAD''' lines = head.splitlines() fstline = lines[0] code =fstline.split()[1] if code == '404' : return (code,None) if code == '200' : return (code,None) if code == '301' or code == '302' : for line in lines[1:]: p = line.index(':') key = line[:p] if key=='Location' : return (code,line[p+2:]) return (code,None) def parse(url): '''Parse a url to hostname+filename''' try: u = url.strip().strip('\n').strip('\r').strip('\t') if u.startswith('http://') : u = u[7:] elif u.startswith('https://'): u = u[8:] if u.find(':80')>0 : p = u.index(':80') p2 = p + 3 else: if u.find('/')>0: p = u.index('/') p2 = p else: p = len(u) p2 = -1 hostname = u[:p] if p2>0 : filename = u[p2:] else : filename = '/' return hostname, filename except Exception ,e: print "Parse wrong : " , url print e def PrintDNSCache(): '''print DNS dict''' n = 1 for hostname in statistics.DNSCache.keys(): print n,'\t',hostname, '\t',statistics.DNSCache[hostname] n+=1 def dealwithResult(res,url): '''Deal with the result of downPage''' statistics.total_url+=1 if res==statistics.RESULTFETCHED : statistics.fetched_url+=1 print statistics.total_url , '\t fetched :', url if res==statistics.RESULTCANNOTFIND : statistics.failed_url+=1 print "Error 404 at : ", url if res==statistics.RESULTOTHER : statistics.other_url +=1 print "Error Undefined at : ", url if res==statistics.RESULTTIMEOUT : statistics.timeout_url +=1 print "Timeout ",url if res==statistics.RESULTTRYTOOMANY: statistics.trytoomany_url+=1 print e ,"Try too many times at", url if __name__=='__main__': print 'Get Page using GET method' 

 ''' Created on 2012-3-16 @author: xiaojay ''' import fetchPage import threadpool import datetime import statistics import urllib2 '''one thread''' def usingOneThread(limit): urlset = open("input.txt","r") start = datetime.datetime.now() for u in urlset: if limit <= 0 : break limit-=1 hostname , filename = parse(u) res= fetchPage.downPage(hostname,filename,0) fetchPage.dealwithResult(res) end = datetime.datetime.now() print "Start at :\t" , start print "End at :\t" , end print "Total Cost :\t" , end - start print 'Total fetched :', statistics.fetched_url '''threadpoll and GET method''' def callbackfunc(request,result): fetchPage.dealwithResult(result[0],result[1]) def usingThreadpool(limit,num_thread): urlset = open("input.txt","r") start = datetime.datetime.now() main = threadpool.ThreadPool(num_thread) for url in urlset : try : hostname , filename = fetchPage.parse(url) req = threadpool.WorkRequest(fetchPage.downPage,args=[hostname,filename],kwds={},callback=callbackfunc) main.putRequest(req) except Exception: print Exception.message while True: try: main.poll() if statistics.total_url >= limit : break except threadpool.NoResultsPending: print "no pending results" break except Exception ,e: print e end = datetime.datetime.now() print "Start at :\t" , start print "End at :\t" , end print "Total Cost :\t" , end - start print 'Total url :',statistics.total_url print 'Total fetched :', statistics.fetched_url print 'Lost url :', statistics.total_url - statistics.fetched_url print 'Error 404 :' ,statistics.failed_url print 'Error timeout :',statistics.timeout_url print 'Error Try too many times ' ,statistics.trytoomany_url print 'Error Other faults ',statistics.other_url main.stop() '''threadpool and urllib2 ''' def downPageUsingUrlib2(url): try: req = urllib2.Request(url) fd = urllib2.urlopen(req) f = open("pages3/"+str(abs(hash(url))),'w') f.write(fd.read()) f.flush() f.close() return url ,'success' except Exception: return url , None def writeFile(request,result): statistics.total_url += 1 if result[1]!=None : statistics.fetched_url += 1 print statistics.total_url,'\tfetched :', result[0], else: statistics.failed_url += 1 print statistics.total_url,'\tLost :',result[0], def usingThreadpoolUrllib2(limit,num_thread): urlset = open("input.txt","r") start = datetime.datetime.now() main = threadpool.ThreadPool(num_thread) for url in urlset : try : req = threadpool.WorkRequest(downPageUsingUrlib2,args=[url],kwds={},callback=writeFile) main.putRequest(req) except Exception ,e: print e while True: try: main.poll() if statistics.total_url >= limit : break except threadpool.NoResultsPending: print "no pending results" break except Exception ,e: print e end = datetime.datetime.now() print "Start at :\t" , start print "End at :\t" , end print "Total Cost :\t" , end - start print 'Total url :',statistics.total_url print 'Total fetched :', statistics.fetched_url print 'Lost url :', statistics.total_url - statistics.fetched_url main.stop() if __name__ =='__main__': '''too slow''' #usingOneThread(100) '''use Get method''' #usingThreadpool(3000,50) '''use urllib2''' usingThreadpoolUrllib2(3000,50)