-
Notifications
You must be signed in to change notification settings - Fork 0
/
BaiduSpider.py
75 lines (62 loc) · 2.2 KB
/
BaiduSpider.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# -*- coding:utf-8 -*-
import urllib2
import urllib
import re
class Tool:
removeImg = re.compile('<img.*?>| {7}|')
removeAddr = re.compile('<a.*?>|</a>')
replacePara = re.compile('<p.*?>')
replaceBR = re.compile('<br><br>|<br>')
def replace(self,x):
x = re.sub(self.removeImg,"",x)
x = re.sub(self.removeAddr,"",x)
x = re.sub(self.replaceBR,"\n",x)
x = re.sub(self.replacePara,"",x)
return x.strip()
class BDTB:
def __init__(self,baseUrl,seeLZ,pn):
self.baseURL = baseUrl
self.seeLZ = '?see_lz=' + str(seeLZ)
self.pageIndex ='&pn='+str(pn)
self.tool = Tool()
def getPage(self,url):
request = urllib2.Request(url)
response = urllib2.urlopen(request)
return response.read().decode('utf-8')
def getTitle(self,page):
pattern = re.compile(r"<h3.*?>(.*?)</h3>",re.S)
result = re.search(pattern,page)
if result:
return result.group(1).strip().encode('utf-8')
else:
return None
def getPageNum(self,page):
pattern = re.compile(r'<li class="l_reply_num.*?</span>.*?<span class="red">(.*?)</span>',re.S)
result = re.search(pattern,page)
if result:
return result.group(1).strip()
else:
return None
def getContent(self,page):
pattern = re.compile(r'<div id="post_content_.*?>(.*?)</div>',re.S)
items = re.findall(pattern,page)
contents = []
for item in items:
content = "\n" + self.tool.replace(item) + "\n"
contents.append(content.encode('utf-8'))
return contents
def start(self):
url = self.baseURL + self.seeLZ + self.pageIndex
page = self.getPage(url)
title = self.getTitle(page)
pageNum = self.getPageNum(page)
contents = self.getContent(page)
print "帖子标题: " + str(title)
print "帖子页数:" + str(pageNum)
for content in contents:
print content
baseURL = "http://tieba.baidu.com/p/3138733512?"
seeLZ = raw_input("是否之收看楼主")
pn = raw_input("帖子页码")
bdtbSpider = BDTB(baseURL,seeLZ,pn)
bdtbSpider.start()