Reptile-REQUESTS usage

Chinese document API: http://requests.kennethreitz.org/zh_CN/latest/

Installation

pip install requests 

Get webpage

# coding=utf-8

import requests



response = requests.get('http://www.baidu .com')



# The first way to get the response content

# View page encoding method

print(response.encoding)

# Modify encoding method

response.encoding = 'utf-8'

# Get response content

print(response.text)



# The second way to get the binary response content



# Get binary response content

print(response.content)

# Decoding decode('decoding method') The default is utf-8 Way

print(response.content.decode())# coding=utf-8

import requests



response = requests.get('http://www.baidu .com')



# The first way to get the response content

# View page encoding method

print(response.encoding)

# Modify encoding method

response.encoding = 'utf-8'

# Get response content

print(response.text)



# The second way to get the binary response content



# Get binary response content

print(response.content)

# Decoding decode('decoding method') The default is utf-8 Way

print(response.content.decode())

Save the picture

import requests



response = requests.get('http://requests.kennethreitz .org/zh_CN/latest/_static/requests-sidebar.png')

# save picture

with open('a.png','wb') as f:

 f.write(response.content)

Get the status code and determine whether the request is successful

 import requests

r = requests.get('http://www.baidu .com')

# Get status code

print(r.status_code) # The page number after the jump when 200 is not necessarily successful and may be obtained

# Assert to determine whether the request is successful

assert r.status_code==200 # If it succeeds, there is no response. Failure will report an error



# Get response header

print(r.headers)

# Get request header

print(r.request.headers)



# Get request url

print(r.request.url)

# Get the response url

print(r.url)

Disguise browser crawling content with header header

import requests

# Simulation header

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36' }

# Get webpage

r = requests.get('http://www.baidu.com',headers=headers)

# Get response content

print(r.text)

Crawl the post bar content

import requests



class WebSpider():

 def __init__(self, name):

 self.headers = {'User-Agent< span style="color: #800000;">': 'Mozilla/5.0 ( Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}

 self.url_temp = "http://tieba.baidu.com /f?kw="+ name +"&ie=utf-8&pn={}"

 self.name = name



 # Build an address list

 def get_url_list(self):

 return [self.url_temp.format(i*50) for< /span> i in range(1000)]



 # Get the post bar content

 def parse_url(self, url): #Crawling data

 print(url)

 r = requests.get(url, headers=self.headers)

 return r.content.decode()





 def run(self):

 # Get address list

 urls = self.get_url_list()

 # Traverse and crawl data

 for url in urls:

 html_str = self.parse_url(url)

 # Save

 page_num = urls.index(url)+1 # Get the number of yards

 file_name = self.name + "page {}.html< span style="color: #800000;">".format(page_num)

 with open(file_name, "w", encoding="utf-8< /span>") as f:

 f.write(html_str)

if __name__ == '__main__':

 r = WebSpider(input("Please enter the name of the post: span>"))

 r.run()

pip install requests

# coding=utf-8

import requests



response = requests.get('http://www.baidu .com')



# The first way to get the response content

# View page encoding method

print(response.encoding)

# Modify encoding method

response.encoding = 'utf-8'

# Get response content

print(response.text)



# The second way to get the binary response content



# Get binary response content

print(response.content)

# Decoding decode('decoding method') The default is utf-8 Way

print(response.content.decode())# coding=utf-8

import requests



response = requests.get('http://www.baidu .com')



# The first way to get the response content

# View page encoding method

print(response.encoding)

# Modify encoding method

response.encoding = 'utf-8'

# Get response content

print(response.text)



# The second way to get the binary response content



# Get binary response content

print(response.content)

# Decoding decode('decoding method') The default is utf-8 Way

print(response.content.decode())

import requests



response = requests.get('http://requests.kennethreitz .org/zh_CN/latest/_static/requests-sidebar.png')

# save picture

with open('a.png','wb') as f:

 f.write(response.content)

import requests

r = requests.get('http://www.baidu .com')

# Get status code

print(r.status_code) # The page number after the jump when 200 is not necessarily successful and may be obtained

# Assert to determine whether the request is successful

assert r.status_code==200 # If it succeeds, there is no response. Failure will report an error



# Get response header

print(r.headers)

# Get request header

print(r.request.headers)



# Get request url

print(r.request.url)

# Get the response url

print(r.url)

import requests

# Simulation header

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36' }

# Get webpage

r = requests.get('http://www.baidu.com',headers=headers)

# Get response content

print(r.text)

import requests



class WebSpider():

 def __init__(self, name):

 self.headers = {'User-Agent< span style="color: #800000;">': 'Mozilla/5.0 ( Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}

 self.url_temp = "http://tieba.baidu.com /f?kw="+ name +"&ie=utf-8&pn={}"

 self.name = name



 # Build an address list

 def get_url_list(self):

 return [self.url_temp.format(i*50) for< /span> i in range(1000)]



 # Get the post bar content

 def parse_url(self, url): #Crawling data

 print(url)

 r = requests.get(url, headers=self.headers)

 return r.content.decode()





 def run(self):

 # Get address list

 urls = self.get_url_list()

 # Traverse and crawl data

 for url in urls:

 html_str = self.parse_url(url)

 # Save

 page_num = urls.index(url)+1 # Get the number of yards

 file_name = self.name + "page {}.html< span style="color: #800000;">".format(page_num)

 with open(file_name, "w", encoding="utf-8< /span>") as f:

 f.write(html_str)

if __name__ == '__main__':

 r = WebSpider(input("Please enter the name of the post: span>"))

 r.run()