python爬虫-模拟登陆新浪微+博爬取感兴趣人的所有信息return True else: return False

return True

else:

return False

def getusermessage(html):

'''

获取用户信息，包括头像，姓名。粉丝 :param html: :return:

''' user = {}

bs = bs4.BeautifulSoup(html,"lxml")

user["姓名"] = bs.find_all("h1",class_="username")[0].text

user["头像"] = bs.find_all("p",class_="photo_wrap")[0].find_all("img")[0]["src"]

user["认证"] = bs.find_all("div",class_="pf_intro")[0]["title"]

messagebox = bs.find_all("div",class_="WB_innerwrap")

messages = messagebox[0].find_all("td",class_="S_line1")

for message in messages:

title = message.find_all("span",class_="S_txt2")[0].text

content = message.find_all("strong",class_="W_f16")[0].text

user[title] = content

print(user)

return user

def getweibo(filename,html):

'''

获取一个页面上的所有信息，包括每一条微博的内容，转发，评论，点赞数量 ''' file = open(filename,"a",encoding="utf-8")

beautifulsoup = bs4.BeautifulSoup(html,"lxml")

weibos = beautifulsoup.find_all("div",class_="WB_cardwrap WB_feed_type S_bg2 ") #获得用于显示微博的父节点

weibonum = 0 #计算该页微博数量

for weibo in weibos:

#对每一个微博父节点进行处理

weibotext = weibo.find_all("div",class_ = "WB_text W_f14") #获取微博文本内容

message = weibotext[0].text.replace(" ","") #取出文本中的所有空格

weibonum+=1 #微博数量+1

file.write("\n第"+str(weibonum)+"条微博内容是:"+message+"\n")

weibopics = weibo.find_all("div",class_="WB_media_wrap clearfix") #获取用于显示图片的标签

if len(weibopics) != 0: #判断标签是否存在

pics = weibopics[0].find_all("img") #获取该标签中的所有图片

for pic in pics:

file.write(pic["src"]+"\n")

weibomessages = weibo.find_all("div", class_="WB_feed_handle") # 获取微博信息父节点

weibomessagecontents = weibomessages[0].find_all("a", class_="S_txt2") #获取用于显示微博信息的节点（包括转发，评论，点赞数量）

for weibomessagecontent in weibomessagecontents[1:]: #分别获取信息

contents = weibomessagecontent.find_all("em")

file.write(contents[1].text+" ")

file.close()

print("成功存储一个页面的微博")

return weibonum

def getpageurl(url,num):

'''

根据一个用户的第一页获取后续的url www.weibo.com/u/185800266… www.weibo.com/u/185800266…

weibo.com/leehom?refe… weibo.com/leehom?is\_… 通过分析指导，page应该是表示的页面，is_all和is_hot应该对应的是热门微博和全部微博 ''' preurl = url.split("?")

preurl = preurl[0]

pageurl = preurl+"?is_search=0&visible=0&is_all=1&is_tag=0&profile_ftype=1&page="+str(num)+"#feedtop"

return pageurl

def spider(url,username,password):

num = 0;

browser = login(**"weibo.com/login.php"*…)

print("开始获取第1页的微博")

html = getweibohtml(browser,url)

file = open("f:****\\****xuanzi.txt","a",encoding="utf-8")

user = getusermessage(html)

for key in user.keys():

file.write(key+":"+user[key]+"\n")

file.close()

num += getweibo("f:****\\****xuanzi.txt",html)

print("获取第1页的微博成功")

page = 1;

while True:

page+=1

pageurl = getpageurl(url,page)

print(pageurl)

html = getweibohtml(browser, pageurl)

if has_source(html):

print("开始获取第"+str(page)+"页的微博")

num += getweibo("f:****\\****xuanzi.txt",html)

else:

return

return num

if __name__ =="__main__":

#browser = login("weibo.com/login.php",…")

'''

for cookie in cookies: if cookie["name"] == "_s_tentry": log_cookie = cookie break ''' ''' getweibohtml("www.weibo.com/u/185800266…", browser) ''' #print(has_source(browser,"www.weibo.com/u/185800266…"))

#file = open("f:\\test5.html","rb").read() #html = file.decode("utf-8","ignore") #getweibo(html)

#url = getpageurl("weibo.com/dasima315?s…", 4) #print(url) #num = spider("www.weibo.com/u/185800266…", "15271183269", "f43312626") #print("一共获取"+str(num)+"条微博")

spider("www.weibo.com/u/185800266…")

轩子局二兔的微博一共有21页，932条，以下是一部分爬取到的文件内容

姓名:轩子巨2兔

头像:tvax3.sinaimg.cn/crop.9.0.49…

认证:斗鱼直播平台签约主播

关注:293

粉丝:503619

微博:932

第1条微博内容是:

庆祝微博40万关注~~转发这条骚气的微博，抽5人送出100元红包加一段私人定制音频~~

wx3.sinaimg.cn/orj360/6ebe…

4333 1810 3644