from scrapy.http.headers import Headers
from Espider.tools.get_cookies import get_cookies
import pymongo,random
from Espider.tools.user_agents import user_agents
from fake_useragent import UserAgent
class zhipincookiemiddleware():
def __init__(self, mongodbHost, mongodbPort, mongodbName):
self.mongodbHost = mongodbHost
self.mongodbPort = mongodbPort
self.mongodbName = mongodbName
@
classmethod def from_crawler(cls, crawler):
return cls(mongodbHost=crawler.settings.get('MONGODB_HOST'), mongodbPort=crawler.settings.get('MONGODB_PORT'),
mongodbName=crawler.settings.get('MONGODB_DBNAME'))
def process_request(self, request, spider):
ua=UserAgent()
self.client = pymongo.MongoClient(self.mongodbHost, self.mongodbPort)
self.mongodb = self.client[self.mongodbName]
self.collection = self.mongodb[
spider.name + '_cookie']
self.cookies_str = self.collection.find_one()['cookie']
self.headers = {
"User-Agent":ua.random,
"cookie": random.choice(self.cookies_str)}
request.headers = Headers(self.headers)
框架里面写了一个 cookiemiddleware
我这里写了一个 random cookie,在每次请求的时候 会重新随机一下 cookie 吗?