Python asynchronous network exploration

Asynchronous networks are said to greatly improve the connection speed of network servers, so I plan to write a topic to learn and understand asynchronous networks. Because Python has a very famous asynchronous Lib: Twisted, so I will use Python to complete it.

OK , first write a server segment of pythone socket, and open three ports: 10000, 10001, 10002. In krondo’s example, each server is bound to a port. When testing, you need to open 3 shells and run them separately. This is too If it’s troublesome, just use three Threads to run these services.

import optparse
import os
import socket
import time
from threading import Thread
import StringIO
 
txt = '''1111
2222
3333
4444
'''
 
def server(listen_socket):
    while True:
        buf = StringIO.StringIO(txt)
        sock, addr = listen_socket.accept()
        print 'Somebody at %s wants poetry!' % (addr,)
        while True:
                try:
                    line = buf.readline().strip()
                    if not line:
                        sock.close()
                        break
                    sock.sendall(line)  # this is a blocking call
                    print 'send bytes to client:%s' % line
                    #sock.close()
                except socket.error:
                    sock.close()
                    break
                time.sleep(1)  #server和client连接后，server会故意每发送一个单词后等待一秒钟后再发送另一个单词
 
 
def main():
    ports = [10000, 10001, 10002]
    for port in ports:
        listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        addres = (str('127.0.0.1'), port)
        listen_socket.bind(addres)
        listen_socket.listen(5)
        print "start listen at:%s" % (port,)
        worker = Thread(target = server, args = [listen_socket])
        worker.setDaemon(True)
        worker.start()
 
 
if __name__ == '__main__':
    main()
    while True:
        time.sleep(0.1) #如果不sleep的话,CPU会被Python完全占用了
        pass

The following is a client. If there is no asynchronous network, connect the server with the three ports:

import socket
 
 
if __name__ == '__main__':
    ports = [10000, 10001, 10002]
    for port in ports:
        address = (str('127.0.0.1'), port)
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect(address)
        poem = ''
        while True:
            data = sock.recv(4)
            if not data:
                sock.close()
                break
            poem += data
        print poem

The following is an asynchronous client to read, the code is as follows :

import datetime, errno, optparse, select, socket
 
def connect(port):
    """Connect to the given server and return a non-blocking socket."""
    address = (str('127.0.0.1'), port)
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(address)
    sock.setblocking(0)
    return sock
 
def format_address(address):
    host, port = address
    return '%s:%s' % (host or '127.0.0.1', port)
 
if __name__ == '__main__':
    ports = [10000, 10001, 10002]
    start = datetime.datetime.now()
 
    sockets = map(connect, ports)
    poems = dict.fromkeys(sockets, '') # socket -> accumulated poem 
 
    # socket -> task numbers
    sock2task = dict([(s, i + 1) for i, s in enumerate(sockets)])
    sockets = list(sockets) # make a copy
 
    while sockets:
        #运用select来确保那些可读取的异步socket可以立即开始读取IO
        #OS不停的搜索目前可以read的socket，有的话就返回rlist
        rlist, _, _ = select.select(sockets, [], [])
        for sock in rlist:
            data = ''
            while True:
                try:
                    new_data = sock.recv(1024)
                except socket.error, e:
                    if e.args[0] == errno.EWOULDBLOCK:
                        break
                    raise
                else:
                    if not new_data:
                        break
                    else:
                        print new_data
                        data += new_data
 
            task_num = sock2task[sock]
            if not data:
                sockets.remove(sock)
                sock.close()
                print 'Task %d finished' % task_num
            else:
                addr_fmt = format_address(sock.getpeername())
                msg = 'Task %d: got %d bytes of poetry from %s'
                print  msg % (task_num, len(data), addr_fmt)
 
            poems[sock] += data
 
    elapsed = datetime.datetime.now() - start
    print 'Got poems in %s' %  elapsed

The result is that it only takes 4 seconds to complete the reading task. The efficiency is three times that of the synchronized socket just now. There are two main points in the asynchronous transformation of the client:

In synchronous mode, the client creates sockets separately; in asynchronous mode, the client creates all sockets from the beginning.

Set the socket to asynchronous mode through "sock.setblocking(0)".

Return readable IO through the select two of the Unix system

The most core ones are lines 26 and 29. In particular, the select operation on line 29 returns the list of sockets to be read.