网站模板简易修改,电商直播,网站代码语法,建筑设计公司账务处理实例今天爬取的是b站。
如何爬取b站中的番剧呢#xff1f; 首先我们来到番剧索引中#xff0c;随便点开一部动漫#xff0c;检查代码。
每个作品对应一个链接: https://www.bilibili.com/bangumi/play/ss…#xff08;ss后面的数字称为ss号#xff09; 发现关于动漫的信息…今天爬取的是b站。
如何爬取b站中的番剧呢 首先我们来到番剧索引中随便点开一部动漫检查代码。
每个作品对应一个链接: https://www.bilibili.com/bangumi/play/ss…ss后面的数字称为ss号 发现关于动漫的信息在这里
‘script’, id“NEXT_DATA” 关键是如何获取ss号
随便乱找一下发现有media_id于是点进去看一下 于是肯定了对应的api接口api.bilibili.com /pgc/review/user?media_id…ts…
这样子就解决了如何爬取每部番剧的网址了。 获取每部番剧的title和url就行接着只要匹配获取简介的内容就行了。
其返回的是一个json字典。 我们只要选出我们感兴趣的内容比如配音演员硬币等内容。
最后保存在xlsx文件即可。 奉上代码如下所示 import numpy as np
import pandas as pdfrom bs4 import BeautifulSoup as bs
import urllib.request as ur
import urllib.parse as up
import urllib.error as ue
import http.cookiejar as hcimport re
import gzip
import jsonimport time
import os
import socketos.chdir(r...)
socket.setdefaulttimeout(30)
cookie
### 配置爬虫条件 ###
# 设置请求头
# api请求头apiheaders{Host: api.bilibili.com,Connection: keep-alive,Cache-Control: max-age0,Upgrade-Insecure-Requests: 1,Accept: application/json, text/plain, */*,Sec-Fetch-Dest: empty,User-Agent: ,Origin: https://www.bilibili.com,Sec-Fetch-Site: same-site,Sec-Fetch-User: ?1,Sec-Fetch-Mode: cors,Sec-Fetch-Dest: document,Referer: https://www.bilibili.com/anime/index/,Accept-Encoding: gzip, deflate, br,Accept-Language: zh-CN,zh;q0.9,Cookie:cookie
}
# 网页请求头
wwwheaders{Host: www.bilibili.com,Connection: keep-alive,Cache-Control: max-age0,Upgrade-Insecure-Requests: 1,User-Agent: ,Sec-Fetch-Dest: document,Accept: text/html,application/xhtmlxml,application/xml;q0.9,Sec-Fetch-Site: same-origin,Sec-Fetch-Mode: navigate,Sec-Fetch-User: ?1,Referer: https://www.bilibili.com/anime/index/,Accept-Encoding: gzip, deflate, br,Accept-Language: zh-CN,zh;q0.9,Cookie:cookie
}
# 图片请求头
imageheaders{Host: i0.hdslb.com,Connection: keep-alive,User-Agent: ,Sec-Fetch-Dest: image,Accept: image/webp,image/apng,image/*,*/*;q0.8,Sec-Fetch-Site: cross-site,Sec-Fetch-Mode: no-cors,Referer: https://www.bilibili.com/bangumi/media/md1178/?fromsearchseid17806546061422186816,Accept-Encoding: gzip, deflate, br,Accept-Language: zh-CN,zh;q0.9
}
# 创建cookiejar对象
cjhc.CookieJar()
# 根据cookiejar创建handler对象
hlur.HTTPCookieProcessor(cj)
# 根据handler创建opener对象
openerur.build_opener(hl)### 爬取ss号 ### 番剧区索引
def ssdownload():sslistlist()patternre.compile(rhttps://www.bilibili.com/bangumi/play/ss\d)pattern2re.compile(rtitle:(.*?))for i in range(1,11):urlhttps://api.bilibili.com/pgc/season/index/result?season_version-1area-1is_finish-1copyright-1season_status-1season_month-1year-1style_id-1order5st1sort0pagestr(i)season_type1pagesize20type1print(正在获取第str(i)页番剧信息)print(url)try_time0while try_time5:try:rur.Request(urlurl,headersapiheaders)responseopener.open(r) breakexcept ue.HTTPError as e:print(网页不存在...跳过)breakexcept Exception as e:try_time1print(重新尝试,try_time)else:raise Exception(下载失败!!)try:contentstr(gzip.decompress(response.read()),utf-8)except Exception as e:breakresponse.close()titlesre.findall(pattern2,content)ssurlre.findall(pattern,content)for i in range(len(ssurl)):sslist.append({title:titles[i],ssurl:ssurl[i]}) return sslist### 访问ss链接 ###
def mddownload(sslist):datajson[]for each_ssurl in sslist:print(正在下载第str(sslist.index(each_ssurl)1)个番剧)print(第str(sslist.index(each_ssurl)1)个番剧网址为each_ssurl[ssurl])try_time0httperrorFalsewhile try_time5:try:rur.Request(urleach_ssurl[ssurl],headerswwwheaders)responseopener.open(r) breakexcept ue.HTTPError as e:httperrorTrueprint(网页不存在...跳过)breakexcept Exception as e:try_time1print(重新尝试,try_time)else:raise Exception(下载失败!!)if httperror:continuecontentstr(gzip.decompress(response.read()),utf-8)response.close()soupbs(content, featureslxml)script_tag soup.find(script, id__NEXT_DATA__)if script_tag is not None:json_text script_tag.stringdata json.loads(json_text)datajson.append(data)return datajsondef getdata(datajson):data[]for each_data in datajson:each_dataeach_data[props][pageProps][dehydratedState][queries][0][state][data][seasonInfo][mediaInfo]data.append({title:each_data[title],media_id:each_data[media_id],season_id:each_data[season_id],coins:each_data[stat][coins],danmakus:each_data[stat][danmakus],favorite:each_data[stat][favorite],favorites:each_data[stat][favorites],follow_text:each_data[stat][follow_text],reply:each_data[stat][reply],likes:each_data[stat][likes],share:each_data[stat][share],views:each_data[stat][views],evaluate:each_data[evaluate],styles:each_data[styles],actors:each_data[actors],staff:each_data[staff],publish:each_data[publish],})print(成功写入有关each_data[title]的数据)return dataif __name____main__:sslistssdownload()datajsonmddownload(sslist)datagetdata(datajson)dfpd.DataFrame(data)df.to_excel(bilibili.xlsx,indexFalse)print(爬取完成) User-Agent和cookie用自己的具体可以自行搜索如何操作。
代码运行效果 最近新开了公众号请大家关注一下。
