scrapy 框架访问链接时，post 请求的几种姿势；在使用 scrapy 进行爬虫编写的时候，经常会遇到一些 post 请求，根据不同的 header 往往会有不同的参数提交方式。
# Query String Parameters
这种是相对简单的，参数可以直接拼接在 url 中，在 url 中 ? 后面的部分即为请求的参数，并以 & 分隔开来。
headers = { | |
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", | |
'Accept': 'application/json, text/javascript, */*; q=0.01', | |
} | |
def start_requests(self): | |
url = "https://s.taobao.com/search?q=鸿星尔克男鞋" | |
yield Request(url=url, method='get', headers=self.headers, | |
callback=self.parse_link) |
# From data
scrapy.FormRequest + formdata 传递，formdata 中的数字要变成字符串形式。
headers = { | |
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", | |
'Accept': 'application/json, text/javascript, */*; q=0.01', | |
'Content-Type': 'application/x-www-form-urlencoded', | |
} | |
def parse_CSRF(self, response): | |
url = "http://asdasdasdasdasdasdsd.com" | |
form_data = { | |
"offset":'0', | |
"limit": '20', | |
"site_id": '11111' | |
} | |
yield FormRequest(url=url, method='post', headers=headers, formdata=form_data, | |
callback=self.parse_link) |
# payload
Request + body 传递，body 内容要使用 json.dumps(payload) 处理一下。
headers = { | |
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", | |
'Accept': 'application/json, text/javascript, */*; q=0.01', | |
'Content-Type':'application/json' | |
} | |
def start_requests(self): | |
url = "http://www.xxxxxx.gov.cn/xxxxx/mailList" | |
start_hash = self.configure.interface.get_start_hash(channel['job']) | |
payload = { | |
"pageNum": '1', | |
"pageSize": '20', | |
"params": { | |
"phone": "", | |
"searchCode": "", | |
} | |
} | |
yield Request(url=url, method='post', headers=headers, | |
body=json.dumps(payload), | |
callback=self.parse_link) |
# json
Request+body 传递，body 要使用 json.dumps(post_data) 进行处理。
headers = { | |
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36", | |
'Accept': 'application/json, text/javascript, */*; q=0.01', | |
'Content-Type': 'application/json' | |
} | |
def start_requests(self): | |
url = "http://xxx.xxx.com/xxx" | |
post_data = { | |
"pageNum":1, | |
"pageSize":20 | |
} | |
start_hash = self.configure.interface.get_start_hash(channel['job']) | |
yield Request(url=url, method='post', headers=headers,body=json.dumps(post_data), | |
callback=self.parse_link) |
