中国建设银行征信网站,网站建设徐州,重庆景点,重庆网站建设mlfart一、前言
每到年底国债逆回购的利息都会来一波高涨#xff0c;利息会比银行的T0的理财产品的利息高#xff0c;所以可以考虑写个脚本每天定时启动爬取逆回购数据#xff0c;实时查看利息#xff0c;然后在利息高位及时去下单。
二、环境搭建
详情请看《python爬虫进阶篇…一、前言
每到年底国债逆回购的利息都会来一波高涨利息会比银行的T0的理财产品的利息高所以可以考虑写个脚本每天定时启动爬取逆回购数据实时查看利息然后在利息高位及时去下单。
二、环境搭建
详情请看《python爬虫进阶篇Scrapy中使用Selenium模拟Firefox火狐浏览器爬取网页信息》
三、代码实现
items
class BondSpiderItem(scrapy.Item):# define the fields for your item here like:# name scrapy.Field()# 股票代码bond_code scrapy.Field()# 股票名称bond_name scrapy.Field()# 最新价last_price scrapy.Field()# 涨跌幅rise_fall_rate scrapy.Field()# 涨跌额rise_fall_price scrapy.Field()middlewares
def __init__(self):# ----------------firefox的设置------------------------------- #self.options firefox_options()def spider_opened(self, spider):spider.logger.info(Spider opened: %s % spider.name)spider.driver webdriver.Firefox(optionsself.options) # 指定使用的浏览器def process_request(self, request, spider):# Called for each request that goes through the downloader# middleware.# Must either:# - return None: continue processing this request# - or return a Response object# - or return a Request object# - or raise IgnoreRequest: process_exception() methods of# installed downloader middleware will be calledspider.driver.get(request.url)return Nonedef process_response(self, request, response, spider):# Called with the response returned from the downloader.# Must either;# - return a Response object# - return a Request object# - or raise IgnoreRequestresponse_body spider.driver.page_sourcereturn HtmlResponse(urlrequest.url, bodyresponse_body, encodingutf-8, requestrequest)settings设置
SPIDER_MIDDLEWARES {bond_spider.middlewares.BondSpiderSpiderMiddleware: 543,
}
DOWNLOADER_MIDDLEWARES {bond_spider.middlewares.BondSpiderDownloaderMiddleware: 543,
}
ITEM_PIPELINES {bond_spider.pipelines.BondSpiderPipeline: 300,
}middlewares中间件
from selenium.webdriver.firefox.options import Options as firefox_optionsspider.driver webdriver.Firefox(optionsfirefox_options()) # 指定使用的浏览器spider文件 def parse(self, response):# 股票代码bond_code response.css(table.table_wrapper-table tbody tr td:nth-child(2) a::text).extract()# 股票名称bond_name response.css(table.table_wrapper-table tbody tr td:nth-child(3) a::text).extract()# 最新价last_price response.css(table.table_wrapper-table tbody tr td:nth-child(4) span::text).extract()# 涨跌幅rise_fall_rate response.css(table.table_wrapper-table tbody tr td:nth-child(6) span::text).extract()# 涨跌额rise_fall_price response.css(table.table_wrapper-table tbody tr td:nth-child(5) span::text).extract()for i in range(len(bond_code)):item BondSpiderItem()item[bond_code] bond_code[i]item[bond_name] bond_name[i]item[last_price] last_price[i]item[rise_fall_rate] rise_fall_rate[i]item[rise_fall_price] rise_fall_price[i]yield itemprint()def close(self, spider):spider.driver.quit()pipelines持久化
def __init__(self):self.html htmlheadmeta charsetutf-8/headbodytableself.html self.html trself.html self.html td%s/td % 代码self.html self.html td%s/td % 名称self.html self.html td%s/td % 最新价self.html self.html td%s/td % 涨跌幅self.html self.html td%s/td % 涨跌额self.html self.html /trdef process_item(self, item, spider):self.html self.html trself.html self.html td%s/td % item[bond_code]self.html self.html td%s/td % item[bond_name]self.html self.html td%s/td % item[last_price]self.html self.html td%s/td % item[rise_fall_rate]self.html self.html td%s/td % item[rise_fall_price]self.html self.html /trreturn itemdef close_spider(self, spider):self.html self.html /table/body/htmlself.send_email(self.html)print()def send_email(self, html):# 设置邮箱账号account xxx# 设置邮箱授权码token xxx# 实例化smtp对象设置邮箱服务器端口smtp smtplib.SMTP_SSL(smtp.qq.com, 465)# 登录qq邮箱smtp.login(account, token)# 添加正文创建简单邮件对象email_content MIMEText(html, html, utf-8)# 设置发送者信息email_content[From] xxx# 设置接受者信息email_content[To] 技术总是日积月累的# 设置邮件标题email_content[Subject] 来自code_space的一封信# 发送邮件smtp.sendmail(account, xxx, email_content.as_string())# 关闭邮箱服务smtp.quit()四、测试结果
