探索eventlet通信机制
一、源码解析
对python原生文件打补丁：
import eventleteventlet.monkey_patch()跟踪进入该模块方法：eventlet.patcher#monkey_patch
def monkey_patch(**on):......modules_to_patch = []for name, modules_function in [ ('os', _green_os_modules), ('select', _green_select_modules), ('socket', _green_socket_modules), ('thread', _green_thread_modules), ('time', _green_time_modules), ('MySQLdb', _green_MySQLdb), ('builtins', _green_builtins), ('subprocess', _green_subprocess_modules),]: if on[name] and not already_patched.get(name): modules_to_patch += modules_function() already_patched[name] = True ......该方法对某些系统模块进行全局打补丁，使其对Greenthread友好。关键字参数用于指定哪些模块需要打补丁，如果未提供关键字参数，则会对所有默认的模块（如代码所示）打补丁，例如： monkey_patch（socket = True，select = True） 仅对socket和select模块打补丁。大多数参数都是对同名的单个模块进行打补丁，比如操作系统，时间，选择。但是socket例外，它也会对ssl模块（如果存在）打补丁，thread用于对threading、thread、Queue打补丁。说明：多次调用monkey_patch是安全的。
以socket为例：('socket', _green_socket_modules)，进入该方法：
def _green_socket_modules(): from eventlet.green import socket try: from eventlet.green import ssl return [('socket',
socket), ('ssl', ssl)] except ImportError: return [('socket',
socket)]进入socket模块：eventlet.green.socket
__import__('eventlet.green._socket_nodns')__socket = sys.modules['eventlet.green._socket_nodns']__all__ = __socket.__all____patched__ = __socket.__patched__ + [ 'create_connection', 'getaddrinfo', 'gethostbyname', 'gethostbyname_ex', 'getnameinfo',]在进入eventlet.green._socket_nodns：
__socket = __import__('socket')__all__ = __socket.__all____patched__ = ['fromfd', 'socketpair', 'ssl', 'socket', 'timeout']可以看到是对python的原生socket模块进行了打补丁：pythonx.x/Lib/socket.py 以socket类为例：python原生的socket.socket()类并替换为了eventlet.greenio.base#GreenSocket类 该补丁类完全兼容原生socket类的API，它还可以识别关键字参数set_nonblocking = True。用来设置socket为非阻塞模式。
class GreenSocket(object):
# This placeholder is to prevent __getattr__ from creating an infinite call loop fd =
None def __init__(self, family
=socket.AF_INET, *
args, **kwargs): should_set_nonblocking = kwargs.pop(
'set_nonblocking', True) if isinstance(
family, six.integer_types): fd = _original_socket(family, *
args, **kwargs) # Notify the hub that this is a newly-opened socket. notify_opened(fd.fileno()) else: fd = family # import timeout from other socket, if it was there try: self._timeout
= fd.gettimeout() or socket.getdefaulttimeout() except AttributeError: self._timeout = socket.getdefaulttimeout() # Filter fd.fileno() != -1 so that won't call set non-blocking on # closed socket if should_set_nonblocking and fd.fileno() != -1: set_nonblocking(
fd) self.fd = fd # when client calls setblocking(0) or settimeout(0) the socket must # act non-blocking self.act_non_blocking = False ......我们再来看下ssl模块。python原生的ssl模块被替换为了evenlet.green.ssl模块 该模块提供了一个方法用来包装socket：
def wrap_socket(sock, *a, **kw): return GreenSSLSocket(sock, *a, **kw)直接进入GreenSSLSocket类:
class GreenSSLSocket(_original_sslsocket): ......可以看出该补丁模块继承了原生socket，将原生socket的api都重写了，但是基本都是直接调用原生api。注：Python3.x版本中，如果socket的另一端已关闭时，非阻塞模式的sslsocket对象不会再抛出错误（虽然它们会在另一端关闭时发出通知）。如果另一端的socket已经关闭，任何的写/读操作都会被简单地挂起。这个问题目前没有好的解决方案。它看起来是Python的sslsocket对象实现的一个限制。一个解决方法是使用命令 settimeout（）在socket上设置合理的超时时间，并在超时时关闭/重新打开连接。
下面看下原生ssl模块：pythonx.x/Lib/ssl.py
def wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, ssl_version=PROTOCOL_TLS, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None): if server_side and not certfile: raise ValueError("certfile must be specified for server-side " "operations") if keyfile and not certfile: raise ValueError("certfile must be specified") context = SSLContext(ssl_version) context.verify_mode = cert_reqs if ca_certs: context.load_verify_locations(ca_certs) if certfile: context.load_cert_chain(certfile, keyfile) if ciphers: context.set_ciphers(ciphers) return context.wrap_socket( sock=sock, server_side=server_side, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs )可以看到该调用了SSLContext.wrap_socket方法，进入该方法：
class SSLContext(_SSLContext): ...... sslsocket_class = None # SSLSocket is assigned later. sslobject_class = None # SSLObject is assigned later. ...... def wrap_socket(self, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, server_hostname=None, session=None): # SSLSocket class handles server_hostname encoding before it calls # ctx._wrap_socket() return self.sslsocket_class._create( sock=sock, server_side=server_side, do_handshake_on_connect=do_handshake_on_connect, suppress_ragged_eofs=suppress_ragged_eofs, server_hostname=server_hostname, context=self, session=session )该类中类属性sslobject_class定义如下：
# Python does not support forward declaration of types.SSLContext.sslsocket_class = SSLSocketSSLContext.sslobject_class = SSLObject进入SSLSocket类：
class SSLSocket(socket): ...... @classmethod def _create(cls, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, server_hostname=None, context=None, session=None): ...... if connected: # create the SSL object try: self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, ) if do_handshake_on_connect: timeout = self.gettimeout() if timeout == 0.0: # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() except (OSError, ValueError): self.close() raise return self最终该self._sslobj实例就是cpython中定义的对象，所有后续的所有操作都是调用的cpython方法。
二、遗留问题
问题堆栈：
Traceback (most recent call last): File "test.py", line 40, in <module> main() File "test.py", line 35, in main srv(listener) File "test.py", line 10, in srv r.readline(1<<10) File "/usr/lib/python3.7/socket.py", line 589, in readinto return self._sock.recv_into(b) File "/usr/lib/python3.7/site-packages/eventlet/green/ssl.py", line 241, in recv_into return self._base_recv(nbytes, flags, into=True, buffer_=buffer) File "/usr/lib/python3.7/site-packages/eventlet/green/ssl.py", line 256, in _base_recv read = self.read(nbytes, buffer_) File "/usr/lib/python3.7/site-packages/eventlet/green/ssl.py", line 176, in read super(GreenSSLSocket, self).read, *args, **kwargs) File "/usr/lib/python3.7/site-packages/eventlet/green/ssl.py", line 146, in _call_trampolining return func(*a, **kw) File "/usr/lib/python3.7/ssl.py", line 911, in read return self._sslobj.read(len, buffer)ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2488)从这里我们可以看到系统调用的入口是python3.7/socket.py中的readinto方法，进入该方法：
def readinto(self, b): self._checkClosed() self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") while True: try: return self._sock.recv_into(b) except timeout: self._timeout_occurred = True raise except error as e: if e.args[0] in _blocking_errnos: return None raise最多将len（b）个字节读入可写缓冲区* b 并返回读取的字节数。如果套接字是非阻塞的并且没有字节可用，则返回None。如果 b *为非空，则返回值为0表示该连接在另一端被关闭。注：如果未设置默认超时并且侦听套接字具有（非零）超时，请强制新套接字处于阻塞模式，以覆盖特定于平台的套接字标志继承。
我们根据堆栈一步步进入最终报错的地方：self._sslobj.read(len, buffer) 根据我们上面说的， self._sslobj实际上是cpython对象，那read方法是怎么就进入到了cpython实际的方法里面的呢？通过python代用C代码的机制可以找到如下代码：
#define _SSL__SSLSOCKET_READ_METHODDEF \ {"read", (PyCFunction)_ssl__SSLSocket_read, METH_VARARGS, _ssl__SSLSocket_read__doc__},static PyObject *_ssl__SSLSocket_read_impl(PySSLSocket *self, int len, int group_right_1, Py_buffer *buffer);static PyObject *_ssl__SSLSocket_read(PySSLSocket *self, PyObject *args){ PyObject *return_value = NULL; int len; int group_right_1 = 0; Py_buffer buffer = {NULL, NULL}; switch (PyTuple_GET_SIZE(args)) { case 1: if (!PyArg_ParseTuple(args, "i:read", &len)) { goto exit; } break; case 2: if (!PyArg_ParseTuple(args, "iw*:read", &len, &buffer)) { goto exit; } group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_ssl._SSLSocket.read requires 1 to 2 arguments"); goto exit; } return_value = _ssl__SSLSocket_read_impl(self, len, group_right_1, &buffer);exit: /* Cleanup for buffer */ if (buffer.obj) { PyBuffer_Release(&buffer); } return return_value;}可以看出，read是映射到了 _ssl__SSLSocket_read方法，而_ssl__SSLSocket_read则调用了 _ssl__SSLSocket_read_impl方法。我们进入_ssl__SSLSocket_read_impl的实现：
static PyObject *_ssl__SSLSocket_read_impl(PySSLSocket
*self, int len, int group_right_1, Py_buffer *buffer)/*[clinic end generated code: output=00097776cec2a0af
input=ff157eb918d0905b]*/{ ...... do { PySSL_BEGIN_ALLOW_THREADS count = SSL_read(self->ssl,
mem, len); err = _PySSL_errno(count <= 0,
self->ssl, count); PySSL_END_ALLOW_THREADS self->err = err; if (PyErr_CheckSignals()) goto error; if (has_timeout) timeout = deadline - _PyTime_GetMonotonicClock(); if (err.ssl == SSL_ERROR_WANT_READ) { sockstate = PySSL_select(sock,
0, timeout); } else
if (err.ssl == SSL_ERROR_WANT_WRITE) { sockstate = PySSL_select(sock,
1, timeout); } else if (err.ssl == SSL_ERROR_ZERO_RETURN && SSL_get_shutdown(self->ssl) == SSL_RECEIVED_SHUTDOWN) { count = 0; goto done; } else sockstate = SOCKET_OPERATION_OK; if (sockstate == SOCKET_HAS_TIMED_OUT) { PyErr_SetString(PySocketModule.timeout_error, "The read operation timed out"); goto error; } else if (sockstate == SOCKET_IS_NONBLOCKING) { break; } } while (err.ssl == SSL_ERROR_WANT_READ
|| err.ssl == SSL_ERROR_WANT_WRITE); if (count <= 0) { PySSL_SetError(self,
count, __FILE__, __LINE__); goto error; } if (self->exc_type != NULL) goto error;......}从该模块的include也可以看出，该模块就是调用了系统的openssl库进行ssl通信
/* Include OpenSSL header files */#include "openssl/rsa.h"#include "openssl/crypto.h"#include "openssl/x509.h"#include "openssl/x509v3.h"#include "openssl/pem.h"#include "openssl/ssl.h"#include "openssl/err.h"#include "openssl/rand.h"#include "openssl/bio.h"#include "openssl/dh.h"进入openssl头文件，可以看到确实有定义这个错误码SSL_ERROR_WANT_READ：
在openssl源码中我们可以找到这个定义include.openssl.ssl.h```c# define SSL_AD_NO_APPLICATION_PROTOCOL TLS1_AD_NO_APPLICATION_PROTOCOL# define SSL_ERROR_NONE 0# define
SSL_ERROR_SSL 1# define SSL_ERROR_WANT_READ 2#
define SSL_ERROR_WANT_WRITE 3# define SSL_ERROR_WANT_X509_LOOKUP 4#
define SSL_ERROR_SYSCALL 5/* look at error stack/return
下面我们来看下PySSL_SetError方法：cpython->modules._ssl.c
static PyObject *PySSL_SetError(PySSLSocket
*sslsock, int ret, const char *filename, int lineno){ PyObject *type = PySSLErrorObject; char *errstr = NULL; _PySSLError err; enum py_ssl_error p = PY_SSL_ERROR_NONE; unsigned long e = 0; assert(ret <= 0); e = ERR_peek_last_error(); if (sslsock->ssl != NULL) { err
= sslsock->err; switch (err.ssl) { case SSL_ERROR_ZERO_RETURN: errstr = "TLS/SSL connection has been closed
(EOF)"; type = PySSLZeroReturnErrorObject; p = PY_SSL_ERROR_ZERO_RETURN; break; case SSL_ERROR_WANT_READ: errstr = "The
operation did not complete (read)"; type = PySSLWantReadErrorObject; p = PY_SSL_ERROR_WANT_READ; break; case SSL_ERROR_WANT_WRITE: p = PY_SSL_ERROR_WANT_WRITE; type = PySSLWantWriteErrorObject; errstr = "The
operation did not complete (write)"; break;经过一步步跟进去，确实会发现返回了一个SSLError类型的错误。
