From c8cee18ee6382038d167aecf674c674fbe03512e Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 25 Jul 2017 01:34:22 -0700 Subject: [PATCH 1/3] Use a slightly less horrible way of documenting interfaces --- docs/source/reference-core.rst | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 84a2fe62f5..03ce1b1e6b 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -420,8 +420,8 @@ Of course, if you really want to make another blocking call in your cleanup handler, trio will let you; it's trying to prevent you from accidentally shooting yourself in the foot. Intentional foot-shooting is no problem (or at least – it's not trio's problem). To do this, -create a new scope, and set its :attr:`shield` attribute to -:data:`True`:: +create a new scope, and set its :attr:`~cancel_scope_interface.shield` +attribute to :data:`True`:: with trio.move_on_after(TIMEOUT): conn = make_connection() @@ -504,9 +504,14 @@ The primitive operation for creating a new cancellation scope is: .. autofunction:: open_cancel_scope :with: cancel_scope - Cancel scope objects provide the following interface: +Cancel scope objects provide the following interface: - .. currentmodule:: None +.. class:: cancel_scope_interface + + (Note: there is no public class called + :class:`cancel_scope_interface`; this is a convenient fiction to + make our documentation generator happy. You get a cancel scope + object by calling :func:`open_cancel_scope`.) .. attribute:: deadline @@ -575,8 +580,6 @@ The primitive operation for creating a new cancellation scope is: exception, and (2) this scope is the one that was responsible for triggering this :exc:`~trio.Cancelled` exception. - .. currentmodule:: trio - Trio also provides several convenience functions for the common situation of just wanting to impose a timeout on some code: @@ -885,9 +888,13 @@ The nursery API .. autofunction:: open_nursery :async-with: nursery - Nursery objects provide the following interface: +Nursery objects provide the following interface: - .. currentmodule:: None +.. class:: nursery_interface + + (Note: there is no public class called :class:`nursery_interface`; + this is a convenient fiction to make our documentation generator + happy. You get a nursery object by calling :func:`open_nursery`.) .. method:: spawn(async_fn, *args, name=None) @@ -970,8 +977,6 @@ The nursery API nursery.reap(task) return task.result.unwrap() - .. currentmodule:: trio - Task object API +++++++++++++++ From 4188f1fd280623e651ee4c510f9a52a0d389ce09 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 25 Jul 2017 01:49:58 -0700 Subject: [PATCH 2/3] An even better way of documenting hidden classes --- docs/source/conf.py | 7 ++++--- docs/source/local_customization.py | 25 +++++++++++++++++++++++++ docs/source/reference-core.rst | 15 +++------------ 3 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 docs/source/local_customization.py diff --git a/docs/source/conf.py b/docs/source/conf.py index 29a7f83739..aa03ed4b7f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,9 +17,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +import os +import sys +sys.path.insert(0, os.path.abspath('.')) # Warn about all references to unknown targets nitpicky = True @@ -51,6 +51,7 @@ def setup(app): 'sphinx.ext.coverage', 'sphinx.ext.napoleon', 'sphinxcontrib_trio', + 'local_customization', ] intersphinx_mapping = { diff --git a/docs/source/local_customization.py b/docs/source/local_customization.py new file mode 100644 index 0000000000..abade0ea8d --- /dev/null +++ b/docs/source/local_customization.py @@ -0,0 +1,25 @@ +from docutils.parsers.rst import directives +from sphinx import addnodes +from sphinx.domains.python import PyClasslike +from sphinx.ext.autodoc import ( + FunctionDocumenter, MethodDocumenter, ClassLevelDocumenter, Options, +) + +""" + +.. interface:: The nursery interface + + .. attribute:: blahblah + +""" + +class Interface(PyClasslike): + def handle_signature(self, sig, signode): + signode += addnodes.desc_name(sig, sig) + return sig, "" + + def get_index_text(self, modname, name_cls): + return "{} (interface in {})".format(name_cls[0], modname) + +def setup(app): + app.add_directive_to_domain("py", "interface", Interface) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 03ce1b1e6b..0c614607dd 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -420,7 +420,7 @@ Of course, if you really want to make another blocking call in your cleanup handler, trio will let you; it's trying to prevent you from accidentally shooting yourself in the foot. Intentional foot-shooting is no problem (or at least – it's not trio's problem). To do this, -create a new scope, and set its :attr:`~cancel_scope_interface.shield` +create a new scope, and set its :attr:`~The cancel scope interface.shield` attribute to :data:`True`:: with trio.move_on_after(TIMEOUT): @@ -506,12 +506,7 @@ The primitive operation for creating a new cancellation scope is: Cancel scope objects provide the following interface: -.. class:: cancel_scope_interface - - (Note: there is no public class called - :class:`cancel_scope_interface`; this is a convenient fiction to - make our documentation generator happy. You get a cancel scope - object by calling :func:`open_cancel_scope`.) +.. interface:: The cancel scope interface .. attribute:: deadline @@ -890,11 +885,7 @@ The nursery API Nursery objects provide the following interface: -.. class:: nursery_interface - - (Note: there is no public class called :class:`nursery_interface`; - this is a convenient fiction to make our documentation generator - happy. You get a nursery object by calling :func:`open_nursery`.) +.. interface:: The nursery interface .. method:: spawn(async_fn, *args, name=None) From d662056fec63e2d321d5a3104e9805eb06a69479 Mon Sep 17 00:00:00 2001 From: "Nathaniel J. Smith" Date: Tue, 25 Jul 2017 02:08:41 -0700 Subject: [PATCH 3/3] Hide the SocketType class It's not really an analogue to the stdlib SocketType (which is the raw _socket.SocketType that socket.socket subclasses), and having it be public makes it quite difficult to add fake sockets (see gh-170). Instead, we expose a new function trio.socket.is_trio_socket, and rework the docs accordingly. --- docs/source/reference-core.rst | 2 +- docs/source/reference-io.rst | 110 +++++++++++++++++++++++++++------ trio/_network.py | 10 +-- trio/socket.py | 88 ++++++-------------------- trio/tests/test_socket.py | 14 +++-- 5 files changed, 126 insertions(+), 98 deletions(-) diff --git a/docs/source/reference-core.rst b/docs/source/reference-core.rst index 0c614607dd..36f5b14a72 100644 --- a/docs/source/reference-core.rst +++ b/docs/source/reference-core.rst @@ -154,7 +154,7 @@ kind of issue looks like in real life, consider this function:: async def recv_exactly(sock, nbytes): data = bytearray() while nbytes > 0: - # SocketType.recv() reads up to 'nbytes' bytes each time + # recv() reads up to 'nbytes' bytes each time chunk += await sock.recv(nbytes) if not chunk: raise RuntimeError("socket unexpected closed") diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index 66fc348847..d4ce3b17d8 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -5,8 +5,8 @@ I/O in Trio .. note:: - Please excuse our dust! `geocities-construction-worker.gif - `__ + Please excuse our dust! `[insert geocities construction worker gif + here] `__ You're looking at the documentation for trio's development branch, which is currently about half-way through implementing a proper @@ -162,11 +162,11 @@ create a :class:`SSLStream`: :members: -Low-level sockets and networking --------------------------------- - .. module:: trio.socket +Low-level networking with :mod:`trio.socket` +--------------------------------------------- + The :mod:`trio.socket` module provides trio's basic low-level networking API. If you're doing ordinary things with stream-oriented connections over IPv4/IPv6/Unix domain sockets, then you probably want @@ -176,8 +176,8 @@ get direct access to all the quirky bits of your system's networking API, then you're in the right place. -:mod:`trio.socket`: top-level exports -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Top-level exports +~~~~~~~~~~~~~~~~~ Generally, the API exposed by :mod:`trio.socket` mirrors that of the standard library :mod:`socket` module. Most constants (like @@ -185,13 +185,26 @@ standard library :mod:`socket` module. Most constants (like are simply re-exported unchanged. But there are also some differences, which are described here. -All functions that return socket objects (e.g. :func:`socket.socket`, -:func:`socket.socketpair`, ...) are modified to return trio socket -objects instead. In addition, there is a new function to directly -convert a standard library socket into a trio socket: +.. function:: socket(...) + socketpair(...) + fromfd(...) + fromshare(...) + + Trio provides analogues to all the standard library functions that + return socket objects; their interface is identical, except that + they're modified to return trio socket objects instead. + +In addition, there is a new function to directly convert a standard +library socket into a trio socket: .. autofunction:: from_stdlib_socket +Unlike :func:`socket.socket`, :func:`trio.socket.socket` is a +function, not a class; if you want to check whether an object is a +trio socket, use: + +.. autofunction:: is_trio_socket + For name lookup, Trio provides the standard functions, but with some changes: @@ -237,7 +250,7 @@ broken features: Socket objects ~~~~~~~~~~~~~~ -.. class:: SocketType() +.. interface:: The trio socket object interface Trio socket objects are overall very similar to the :ref:`standard library socket objects `, with a few @@ -279,9 +292,27 @@ Socket objects this can be easily accomplished by calling either :meth:`resolve_local_address` or :meth:`resolve_remote_address`. - .. automethod:: resolve_local_address + .. method:: resolve_local_address(address) + + Resolve the given address into a numeric address suitable for + passing to :meth:`bind`. + + This performs the same address resolution that the standard library + :meth:`~socket.socket.bind` call would do, taking into account the + current socket's settings (e.g. if this is an IPv6 socket then it + returns IPv6 addresses). In particular, a hostname of ``None`` is + mapped to the wildcard address. - .. automethod:: resolve_remote_address + .. method:: resolve_remote_address(address) + + Resolve the given address into a numeric address suitable for + passing to :meth:`connect` or similar. + + This performs the same address resolution that the standard library + :meth:`~socket.socket.connect` call would do, taking into account the + current socket's settings (e.g. if this is an IPv6 socket then it + returns IPv6 addresses). In particular, a hostname of ``None`` is + mapped to the localhost address. **Modern defaults:** And finally, we took the opportunity to update the defaults for several socket options that were stuck in the @@ -326,14 +357,54 @@ Socket objects See `issue #72 `__ for discussion of these defaults. - The following methods are similar, but not identical, to the - equivalents in :func:`socket.socket`: + The following methods are similar to the equivalents in + :func:`socket.socket`, but have some trio-specific quirks: + + .. method:: bind - .. automethod:: bind + Bind this socket to the given address. - .. automethod:: connect + Unlike the stdlib :meth:`~socket.socket.bind`, this method + requires a pre-resolved address. See + :meth:`resolve_local_address`. - .. automethod:: sendall + .. method:: connect + :async: + + Connect the socket to a remote address. + + Similar to :meth:`socket.socket.connect`, except async and + requiring a pre-resolved address. See + :meth:`resolve_remote_address`. + + .. warning:: + + Due to limitations of the underlying operating system APIs, it is + not always possible to properly cancel a connection attempt once it + has begun. If :meth:`connect` is cancelled, and is unable to + abort the connection attempt, then it will: + + 1. forcibly close the socket to prevent accidental re-use + 2. raise :exc:`~trio.Cancelled`. + + tl;dr: if :meth:`connect` is cancelled then the socket is + left in an unknown state – possibly open, and possibly + closed. The only reasonable thing to do is to close it. + + .. method:: sendall(data, flags=0) + :async: + + Send the data to the socket, blocking until all of it has been + accepted by the operating system. + + ``flags`` are passed on to ``send``. + + Most low-level operations in trio provide a guarantee: if they raise + :exc:`trio.Cancelled`, this means that they had no effect, so the + system remains in a known state. This is **not true** for + :meth:`sendall`. If this operation raises :exc:`trio.Cancelled` (or + any other exception for that matter), then it may have sent some, all, + or none of the requested data, and there is no way to know which. .. method:: sendfile @@ -374,6 +445,7 @@ Socket objects * :meth:`~socket.socket.set_inheritable` * :meth:`~socket.socket.get_inheritable` + Asynchronous disk I/O --------------------- diff --git a/trio/_network.py b/trio/_network.py index e0bf67f3a8..6df9c666c7 100644 --- a/trio/_network.py +++ b/trio/_network.py @@ -35,8 +35,8 @@ class SocketStream(HalfCloseableStream): interface based on a raw network socket. Args: - sock (trio.socket.SocketType): The trio socket object to wrap. Must have - type ``SOCK_STREAM``, and be connected. + sock: The trio socket object to wrap. Must have type ``SOCK_STREAM``, + and be connected. By default, :class:`SocketStream` enables ``TCP_NODELAY``, and (on platforms where it's supported) enables ``TCP_NOTSENT_LOWAT`` with a @@ -50,12 +50,12 @@ class SocketStream(HalfCloseableStream): .. attribute:: socket - The :class:`trio.socket.SocketType` object that this stream wraps. + The Trio socket object that this stream wraps. """ def __init__(self, sock): - if not isinstance(sock, tsocket.SocketType): - raise TypeError("SocketStream requires trio.socket.SocketType") + if not tsocket.is_trio_socket(sock): + raise TypeError("SocketStream requires trio socket object") if sock._real_type != tsocket.SOCK_STREAM: raise ValueError("SocketStream requires a SOCK_STREAM socket") try: diff --git a/trio/socket.py b/trio/socket.py index 0ab70d2ef7..412e52f2fd 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -176,7 +176,7 @@ def from_stdlib_socket(sock): """Convert a standard library :func:`socket.socket` into a trio socket. """ - return SocketType(sock) + return _SocketType(sock) __all__.append("from_stdlib_socket") @_wraps(_stdlib_socket.fromfd, assigned=(), updated=()) @@ -192,9 +192,8 @@ def fromshare(*args, **kwargs): @_wraps(_stdlib_socket.socketpair, assigned=(), updated=()) def socketpair(*args, **kwargs): - return tuple( - from_stdlib_socket(s) - for s in _stdlib_socket.socketpair(*args, **kwargs)) + left, right = _stdlib_socket.socketpair(*args, **kwargs) + return (from_stdlib_socket(left), from_stdlib_socket(right)) __all__.append("socketpair") @_wraps(_stdlib_socket.socket, assigned=(), updated=()) @@ -204,7 +203,19 @@ def socket(*args, **kwargs): ################################################################ -# SocketType +# Type checking +################################################################ + +def is_trio_socket(obj): + """Check whether the given object is a trio socket. + + """ + return isinstance(obj, _SocketType) + +__all__.append("is_trio_socket") + +################################################################ +# _SocketType ################################################################ # sock.type gets weird stuff set in it, in particular on Linux: @@ -218,7 +229,7 @@ def socket(*args, **kwargs): getattr(_stdlib_socket, "SOCK_NONBLOCK", 0) | getattr(_stdlib_socket, "SOCK_CLOEXEC", 0)) -class SocketType: +class _SocketType: def __init__(self, sock): if type(sock) is not _stdlib_socket.socket: # For example, ssl.SSLSocket subclasses socket.socket, but we @@ -302,15 +313,9 @@ def dup(self): """Same as :meth:`socket.socket.dup`. """ - return SocketType(self._sock.dup()) + return _SocketType(self._sock.dup()) def bind(self, address): - """Bind this socket to the given address. - - Unlike the stdlib :meth:`~socket.socket.connect`, this method requires - a pre-resolved address. See :meth:`resolve_local_address`. - - """ self._check_address(address, require_resolved=True) return self._sock.bind(address) @@ -418,30 +423,10 @@ async def _resolve_address(self, address, flags): # Returns something appropriate to pass to bind() async def resolve_local_address(self, address): - """Resolve the given address into a numeric address suitable for - passing to :meth:`bind`. - - This performs the same address resolution that the standard library - :meth:`~socket.socket.bind` call would do, taking into account the - current socket's settings (e.g. if this is an IPv6 socket then it - returns IPv6 addresses). In particular, a hostname of ``None`` is - mapped to the wildcard address. - - """ return await self._resolve_address(address, AI_PASSIVE) # Returns something appropriate to pass to connect()/sendto()/sendmsg() async def resolve_remote_address(self, address): - """Resolve the given address into a numeric address suitable for - passing to :meth:`connect` or similar. - - This performs the same address resolution that the standard library - :meth:`~socket.socket.connect` call would do, taking into account the - current socket's settings (e.g. if this is an IPv6 socket then it - returns IPv6 addresses). In particular, a hostname of ``None`` is - mapped to the localhost address. - - """ return await self._resolve_address(address, 0) async def _nonblocking_helper(self, fn, args, kwargs, wait_fn): @@ -510,28 +495,10 @@ async def accept(self): ################################################################ async def connect(self, address): - """Connect the socket to a remote address. - - Similar to :meth:`socket.socket.connect`, except async and requiring a - pre-resolved address. See :meth:`resolve_remote_address`. - - .. warning:: - - Due to limitations of the underlying operating system APIs, it is - not always possible to properly cancel a connection attempt once it - has begun. If :meth:`connect` is cancelled, and is unable to - abort the connection attempt, then it will: - - 1. forcibly close the socket to prevent accidental re-use - 2. raise :exc:`~trio.Cancelled`. - - tl;dr: if :meth:`connect` is cancelled then you should throw away - that socket and make a new one. - - """ # nonblocking connect is weird -- you call it to start things # off, then the socket becomes writable as a completion - # notification. This means it isn't really cancellable... + # notification. This means it isn't really cancellable... we close the + # socket if cancelled, to avoid confusion. async with _try_sync(): self._check_address(address, require_resolved=True) # An interesting puzzle: can a non-blocking connect() return EINTR @@ -693,19 +660,6 @@ async def sendmsg(self, *args): ################################################################ async def sendall(self, data, flags=0): - """Send the data to the socket, blocking until all of it has been - accepted by the operating system. - - ``flags`` are passed on to ``send``. - - Most low-level operations in trio provide a guarantee: if they raise - :exc:`trio.Cancelled`, this means that they had no effect, so the - system remains in a known state. This is **not true** for - :meth:`sendall`. If this operation raises :exc:`trio.Cancelled` (or - any other exception for that matter), then it may have sent some, all, - or none of the requested data, and there is no way to know which. - - """ with memoryview(data) as data: if not data: await _core.yield_briefly() @@ -730,8 +684,6 @@ async def sendall(self, data, flags=0): # settimeout # timeout -__all__.append("SocketType") - ################################################################ # create_connection diff --git a/trio/tests/test_socket.py b/trio/tests/test_socket.py index 207bb64af4..9f91298208 100644 --- a/trio/tests/test_socket.py +++ b/trio/tests/test_socket.py @@ -181,8 +181,10 @@ async def test_getnameinfo(): async def test_from_stdlib_socket(): sa, sb = stdlib_socket.socketpair() + assert not tsocket.is_trio_socket(sa) with sa, sb: ta = tsocket.from_stdlib_socket(sa) + assert tsocket.is_trio_socket(ta) assert sa.fileno() == ta.fileno() await ta.sendall(b"xxx") assert sb.recv(3) == b"xxx" @@ -239,16 +241,18 @@ async def test_fromshare(): async def test_socket(): with tsocket.socket() as s: - assert isinstance(s, tsocket.SocketType) + assert isinstance(s, tsocket._SocketType) + assert tsocket.is_trio_socket(s) assert s.family == tsocket.AF_INET with tsocket.socket(tsocket.AF_INET6, tsocket.SOCK_DGRAM) as s: - assert isinstance(s, tsocket.SocketType) + assert isinstance(s, tsocket._SocketType) + assert tsocket.is_trio_socket(s) assert s.family == tsocket.AF_INET6 ################################################################ -# SocketType +# _SocketType ################################################################ async def test_SocketType_basics(): @@ -307,7 +311,7 @@ async def test_SocketType_dup(): with a, b: a2 = a.dup() with a2: - assert isinstance(a2, tsocket.SocketType) + assert isinstance(a2, tsocket._SocketType) assert a2.fileno() != a.fileno() a.close() await a2.sendall(b"xxx") @@ -519,7 +523,7 @@ async def test_SocketType_connect_paths(): with tsocket.socket() as sock, tsocket.socket() as listener: listener.bind(("127.0.0.1", 0)) listener.listen() - # Swap in our weird subclass under the trio.socket.SocketType's + # Swap in our weird subclass under the trio.socket._SocketType's # nose -- and then swap it back out again before we hit # wait_socket_writable, which insists on a real socket. class CancelSocket(stdlib_socket.socket):