有一个做5s壁纸的网站,MAKA网站做H5怎么压缩图片,泰州市建设工程招标网,dede手机网站标签需求介绍输入#xff1a;关键字文件#xff0c;每一行数据为一爬取单元。若一行存在多个and关系的关键字 #xff0c;则用|隔开处理#xff1a;爬取访问6个网站的推送#xff0c;获取推送内容的标题#xff0c;发布时间#xff0c;来源#xff0c;正文第一段#xff0…需求介绍输入关键字文件每一行数据为一爬取单元。若一行存在多个and关系的关键字 则用|隔开处理爬取访问6个网站的推送获取推送内容的标题发布时间来源正文第一段不是图片或者图例输出输出到csv文件ui窗口小程序能实时地跟踪爬虫进度运行要求不依赖于python环境独立运行的exe文件 分析实现的主要程序 最后pyinstaller 打包crawl.py即可 实现 uI中的线程控制 import tkinter as tk
import time
import sys
import queue
import threading
def fmtTime(timestamp):localtimetime.localtime(timestamp)datetimetime.strftime(%Y-%m-%d %H:%M:%S,localtime)return datetimeclass re_Text():def __init__(self,queue):self.qqueuedef write(self,content):self.q.put(content)class GUI(object):def __init__(self,root):self.rootrootself.qqueue.Queue()self.initGUI(root)def show_msg(self):if not self.q.empty():self.text.insert(insert,self.q.get())self.text.see(tk.END)self.root.after(100,self.show_msg)def initGUI(self,root):root.title(点击数据)root.geometry(400x200700500)bntk.Button(root,textclick,width10,commandself.show)#pack 控制排版bn.pack(sidetop)scrollBar tk.Scrollbar(root)scrollBar.pack(sideright, filly)self.text tk.Text(root, height10, width45, yscrollcommandscrollBar.set)self.text.pack(sidetop, filltk.BOTH, padx10, pady10)#动态绑定scrollBar.config(commandself.text.yview)#不要想着中断机制或者调用子函数机制把它视为另一个线程# write通信作用sys.stdoutre_Text(self.q)root.after(100,self.show_msg)root.mainloop()def _show(self):i 0for i in range(4):# 顺序执行 ui的刷新线程没有抢占cpu阻塞在这 等过了3秒后才刷新到texttime.sleep(1)# 重定向 调用writeprint(fmtTime(time.time()))def show(self):# 创建子线程 窗口程序可以不断地监听Tthreading.Thread(targetself._show)T.start()if __name____main__:roottk.Tk()GUI(root) scrapy.py 技术细节可以参考之前的文章这里就直接写spider了 import scrapy
from scrapy import Selector
from scrapy import Request, signals
import pandas as pd
import re
from x93.items import csvItem
import os
import sysclass ExampleSpider(scrapy.Spider):name spiderdef __init__(self, **kwargs):super(ExampleSpider, self).__init__(**kwargs)self.data list()self.keywskwargs.get(keywords)#print(self.keyws)print(----------)self.sites[sh93.gov.cn,93.gov.cn,shszx.gov.cn,tzb.ecnu.edu.cnecnu.edu.cn/info/1094,ecnu.edu.cn/info/1095]def start_requests(self):#keywself.keywsfor keyw in self.keyws:keywkeyw.strip()keywkeyw.split(|)keyw.join(keyw)for site in self.sites:self.logger.info(sitesite)#urlfhttps://cn.bing.com/search?qsite%3a{site}allintext%3a{keyw}first1url fhttps://cn.bing.com/search?qsite%3a{site}{keyw}first1yield Request(url, callbackself.parse, cb_kwargs{first:1,site:site,keyw:keyw,totallist:0})def parse(self, response,**kwargs):#百度网页 列表内容resSelector(textresponse.text)for a in res.xpath(//h2/a[target_blank]):title a.xpath(./text()).get()href a.xpath(./href) .get()out re.search(index,href)htm re.search(htm,href)# 排除含index列表页 json 数据页if out!None or htmNone:continuekwargs[href]hrefyield Request(href,callbackself.get_detail,cb_kwargskwargs)#翻页# if kwargs[first]1:# nubres.xpath(r//span[classsb_count]/text()).get()# nub.join(re.findall(re.compile([\d]),nub))# kwargs[totallist]int(nub)# #self.logger.info(kwargs[totallist] kwargs[totallist])# if kwargs[first]10kwargs[totallist]:# self.logger.info(fkwargs[totallist]{kwargs[totallist]})# kwargs[first] kwargs[first] 10# urlfhttps://cn.bing.com/search?qsite%3a{kwargs[site]}allintext%3a{kwargs[keyw]}first{kwargs[first]} # self.logger.info(furl{url})# yield Request(url, callbackself.parse, cb_kwargskwargs)def get_detail(self,response,**kwargs):res Selector(response)titledate contentsourceif kwargs[site]sh93.gov.cn:try:title res.xpath(//h3[contains(class,article-title)]/text()).get()date res.xpath(//div[contains(class,article-title-news)]/span/text()).get()date .join(re.findall(re.compile(r[\d-]), date))try:sourceres.xpath(//span[classml20]/text()).get()except TypeError:source九三上海市委新闻try:contentres.xpath(//div[contains(class,article-content)]/p[not (styletext-align: center;)]/text()).get().strip()except :contentres.xpath(//span[contains(style,font-family)]/text()).get().strip()except:try:title res.xpath(//td[classpix16blackl32]/text()).get()date res.xpath(//td[classpixh4_line24]/text()).get()date re.findall(re.compile(r[\d-]), date)date -.join(date)source 九三学社上海市委员会content res.xpath(//p/text()).get()except:self.logger.error(f无法解析{kwargs[href]})if kwargs[site]93.gov.cn:titleres.xpath(//div[contains(class,pageTitle)]/h2/text()).get()dateres.xpath(//div[contains(class,pageTitle)]//ul/li[1]/text()).get()date .join(re.findall(re.compile(r[\d-]), date))sourceres.xpath(//div[contains(class,pageTitle)]//ul/li[2]/text()).get()[3:]try:content res.xpath(//div[classtext]/p[not (styletext-align: center;)]/text()).get().strip()except AttributeError:#print(url:kwargs[href])content res.xpath(//div[classtext]//span[contains(style,font-family)]/text()).get().strip()#content res.xpathif kwargs[site]shszx.gov.cn:titleres.xpath(//h2[idivs_title]/text()).get()date res.xpath(//div[classfc con22 lh28 grey12]/text()).get()date.join(re.findall(re.compile(r[\d-]), date))source上海政协cnt1while content :cntcnt1contentres.xpath(f//div[idivs_content]/p[not (aligncenter) and fnot (styletext-align: center;)][{cnt}]/text()).get().strip()if kwargs[site]tzb.ecnu.edu.cn:titleres.xpath(//h1[classarti_title]/text()).get()#text() 会取第一个dateres.xpath(//span[classarti_update]/text()).get()date.join(re.findall(re.compile(r[\d-]), date))sourceres.xpath(//span[classarti_department]/text()).get()[4:]contentres.xapth(//div[classwp_articlecontent]//p[contains(style,font-size)]/text()).get().strip()if ecnu.edu.cn in kwargs[site]:titleres.xpath(//h2[classm3nTitle]/text()).get()dateres.xpath(//span[classm3ntm]/text()).get()datere.findall(re.compile(r[\d-]), date)date-.join(date)if 1094 in kwargs[site]:source华东师范大学新闻热点else:source 华东师范大学媒体关注contentres.xpath(//p/span[contains(style,font-family)]//text()|//p[contains(style,text-align:justify)]/text()).get().strip()itemcsvItem()item[keyword]kwargs[keyw]item[title]titleitem[date]dateitem[source]sourceitem[content]contentitem[url]kwargs[href]print(title, date, content)yield item ui脚本中运行scrapyscrapy 脚本运行有三种方式,实现细节可以参考官方文档cmdline.execute方式只能运行一个爬虫而其他两种方式可以同时运行多个异步异步。 scrapy爬虫比较耗时需要放在子线程工作因此选用crawlRunner(crawlProcess要求运行在主线程就不行)但是不清楚为什么刚开始正常运行后来又是报错提示signal run in main thread.可以尝试的解决方法参考博客 # !/user/bin/env Python3
# -*- coding:utf-8 -*-from scrapy import *
import scrapy
from scrapy import Selector
from scrapy import Request, signals
import pandas as pd
import re
import time
import tkinter as tk
from tkinter import filedialog, dialog
import os
import threading
import logging
import sys
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.cmdline import execute
from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from x93.spiders.spider import ExampleSpider
from scrapy.utils.project import get_project_settingslogger logging.getLogger(__name__)
file_path
file_text
class re_Text():def __init__(self,text):self.texttextdef write(self,content):self.text.insert(insert,content )self.text.see(tk.END)
class GUI():def __init__(self):self.roottk.Tk()self.root.title(ecnu数据爬取工具)self.root.geometry(400x400200200)def initGUI(self):# self.root.title(窗口标题) # 标题# self.root.geometry(500x500) # 窗口尺寸self.scrollBar tk.Scrollbar(self.root)self.scrollBar.pack(sideright, filly)self.text tk.Text(self.root, height10, width45, yscrollcommandself.scrollBar.set)self.text.pack(sidetop, filltk.BOTH, padx10, pady10)#self.scrollBar.config(commandself.text.yview) # 动态绑定 滚动条随着鼠标移动bt1 tk.Button(self.root, text打开文件, width15, height2, commandself.open_file)bt1.pack()bt2 tk.Button(self.root, text爬取数据, width15, height2, commandself._app)bt2.pack()#bt3 tk.Button(self.root, text保存文件, width15, height2, commandsave_file)#bt3.pack()def open_file(self):打开文件:return:global file_pathglobal file_textfile_path filedialog.askopenfilename(titleu选择文件, initialdir(os.path.expanduser(H:/)))print(打开文件, file_path)if file_path is not None:with open(filefile_path, moder, encodingutf-8) as file:file_text file.readlines()print(file_text)def thread_it(func, *args):将函数打包进线程# 创建t threading.Thread(targetfunc, argsargs)# 守护 !!!t.setDaemon(True)# 启动t.start()# 阻塞--卡死界面#t.join()def _app(self):tthreading.Thread(targetself.app,args())t.start()def app(self):global file_textlogger.info(ftype(file_text){type(file_text)})runner CrawlerRunner(get_project_settings())d runner.crawl(ExampleSpider,keywordsfile_text)d.addBoth(lambda _: reactor.stop())reactor.run()self.text.insert(insert, 爬取成功)# process CrawlerProcess(get_project_settings())# process.crawl(ExampleSpider,keywordsfile_text)# process.start()# cmdfscrapy crawl spider -a kw{file_text}.split()# execute(cmd)# cmd python Run.py# print(os.system(cmd))gGUI()
g.initGUI()
sys.stdoutre_Text(g.text)
g.root.mainloop() # 显示 pyinstaller 打包因为看到很多博客都说用pyinstaller打包scrapy 需要引入较多依赖修改配置文件等多余的操作一直没敢下手尝试直接用pyinstaller打包主程序转向成功案例较多的单线程运行爬虫方式后来还是败给的scrapy框架,返回失败的retry机制以及代理更换的便捷毕竟批量输入无可避免的有无法访问目标计算机问题。 打包后的程序调试问题后就上传占个位 文章转载自mingruqi 原文链接https://www.cnblogs.com/Im-Victor/p/17081823.html 体验地址引迈 - JNPF快速开发平台_低代码开发平台_零代码开发平台_流程设计器_表单引擎_工作流引擎_软件架构
