High performance related

How to achieve multiple tasks at the same time and high efficiency

Serial implementation

The lowest efficiency is the most undesirable

import requests



urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



for url in urls:

 response = requests.get(url)

 print(response)

Multithread

Multithread There is a problem of low thread utilization

import requests

import threading





urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



def task(url):

 response = requests.get(url)

 print(response)



for url in urls:

 t = threading.Thread(target=task,args=(url,))

 t.start()

Coroutine + IO switch

Gevent internally calls greenlet (implements coroutine)

Based on coroutines, it saves more resources than threads.

from gevent import monkey; monkey.patch_all()

import gevent

import requests





def func(url):

 response = requests.get(url)

 print(response)



urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]

spawn_list = []

for url in urls:

 spawn_list.append(gevent.spawn(func, url)) # Create a coroutine



gevent.joinall(spawn_list)

Event loop

Asynchronous non-blocking module based on event loop: Twisted

from twisted.web.client import getPage, defer

from twisted.internet import reactor



def stop_loop(arg):

 reactor.stop()





def get_response(contents):

 print(contents)



deferred_list = []



url_list = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



for url in url_list:

 deferred = getPage(bytes(url, encoding='utf8')) # I got the task to be crawled, but did not actually execute the crawler

 deferred.addCallback(get_response) # The callback function to be called 

 deferred_list.append(deferred) # Add all tasks to one list span>





dlist = defer.DeferredList(deferred_list) # Check whether all tasks are Are looped

dlist.addBoth(stop_loop) # Stop the loop if all tasks in the list are completed , The execution of the stopped function



reactor.run()

import requests



urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



for url in urls:

 response = requests.get(url)

 print(response)

import requests

import threading





urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



def task(url):

 response = requests.get(url)

 print(response)



for url in urls:

 t = threading.Thread(target=task,args=(url,))

 t.start()

from gevent import monkey; monkey.patch_all()

import gevent

import requests





def func(url):

 response = requests.get(url)

 print(response)



urls = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]

spawn_list = []

for url in urls:

 spawn_list.append(gevent.spawn(func, url)) # Create a coroutine



gevent.joinall(spawn_list)

from twisted.web.client import getPage, defer

from twisted.internet import reactor



def stop_loop(arg):

 reactor.stop()





def get_response(contents):

 print(contents)



deferred_list = []



url_list = [

 'http://www.baidu.com/',

 'https://www.cnblogs.com/',

 'https://www.cnblogs.com/news/< /span>',

 'https://cn.bing.com/',

 'https://stackoverflow.com/< span style="color: #800000;">',

]



for url in url_list:

 deferred = getPage(bytes(url, encoding='utf8')) # I got the task to be crawled, but did not actually execute the crawler

 deferred.addCallback(get_response) # The callback function to be called 

 deferred_list.append(deferred) # add all tasks to a list with one span>





dlist = defer.DeferredList(deferred_list) # Check whether all tasks are Are looped

dlist.addBoth(stop_loop) # Stop the loop if all tasks in the list are completed , The execution of the stopped function



reactor.run()