-
-
Notifications
You must be signed in to change notification settings - Fork 2k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Feature/bugfix: make the HTTP client able to return HTTP chunks when chunked transfer encoding is used #2150
Changes from 21 commits
7a30dc5
ad942af
09b1da5
511a680
6d0211f
32e4273
fae40a7
f56b069
4563338
513f0ad
85ea0eb
09848db
d69d890
5e8e25f
939d386
bf31405
19b9d14
5ed76d6
e172a23
c99e06c
1136c56
39381ba
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -40,6 +40,14 @@ def __anext__(self): | |
raise StopAsyncIteration # NOQA | ||
return rv | ||
|
||
class ChunkTupleAsyncStreamIterator(AsyncStreamIterator): | ||
@asyncio.coroutine | ||
def __anext__(self): | ||
rv = yield from self.read_func() | ||
if rv == (b'', False): | ||
raise StopAsyncIteration # NOQA | ||
return rv | ||
|
||
|
||
class AsyncStreamReaderMixin: | ||
|
||
|
@@ -58,20 +66,21 @@ def iter_chunked(self, n): | |
return AsyncStreamIterator(lambda: self.read(n)) | ||
|
||
def iter_any(self): | ||
"""Returns an asynchronous iterator that yields slices of data | ||
as they come. | ||
"""Returns an asynchronous iterator that yields all the available | ||
data as soon as it is received | ||
|
||
Python-3.5 available for Python 3.5+ only | ||
""" | ||
return AsyncStreamIterator(self.readany) | ||
|
||
def iter_chunks(self): | ||
"""Returns an asynchronous iterator that yields chunks of the | ||
size as received by the server. | ||
"""Returns an asynchronous iterator that yields chunks of data | ||
as they are received by the server. The yielded objects are tuples | ||
of (bytes, bool) as returned by the StreamReader.readchunk method. | ||
|
||
Python-3.5 available for Python 3.5+ only | ||
""" | ||
return AsyncStreamIterator(self.readchunk) | ||
return ChunkTupleAsyncStreamIterator(self.readchunk) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add a test for the call. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done |
||
|
||
|
||
class StreamReader(AsyncStreamReaderMixin): | ||
|
@@ -96,6 +105,8 @@ def __init__(self, limit=DEFAULT_LIMIT, timer=None, loop=None): | |
loop = asyncio.get_event_loop() | ||
self._loop = loop | ||
self._size = 0 | ||
self._cursor = 0 | ||
self._http_chunk_splits = None | ||
self._buffer = collections.deque() | ||
self._buffer_offset = 0 | ||
self._eof = False | ||
|
@@ -200,6 +211,7 @@ def unread_data(self, data): | |
self._buffer[0] = self._buffer[0][self._buffer_offset:] | ||
self._buffer_offset = 0 | ||
self._size += len(data) | ||
self._cursor -= len(data) | ||
self._buffer.appendleft(data) | ||
|
||
def feed_data(self, data): | ||
|
@@ -218,6 +230,18 @@ def feed_data(self, data): | |
if not waiter.done(): | ||
waiter.set_result(False) | ||
|
||
def begin_http_chunk_receiving(self): | ||
if self._http_chunk_splits is None: | ||
self._http_chunk_splits = [] | ||
|
||
def end_http_chunk_receiving(self): | ||
if self._http_chunk_splits is None: | ||
raise RuntimeError("Called end_chunk_receiving without calling " | ||
"begin_chunk_receiving first") | ||
if not self._http_chunk_splits or \ | ||
self._http_chunk_splits[-1] != self.total_bytes: | ||
self._http_chunk_splits.append(self.total_bytes) | ||
|
||
@asyncio.coroutine | ||
def _wait(self, func_name): | ||
# StreamReader uses a future to link the protocol feed_data() method | ||
|
@@ -320,16 +344,34 @@ def readany(self): | |
|
||
@asyncio.coroutine | ||
def readchunk(self): | ||
"""Returns a tuple of (data, end_of_http_chunk). When chunked transfer | ||
encoding is used, end_of_http_chunk is a boolean indicating if the end | ||
of the data corresponds to the end of a HTTP chunk , otherwise it is | ||
always False. | ||
""" | ||
if self._exception is not None: | ||
raise self._exception | ||
|
||
if not self._buffer and not self._eof: | ||
if (self._http_chunk_splits and | ||
self._cursor == self._http_chunk_splits[0]): | ||
# end of http chunk without available data | ||
self._http_chunk_splits = self._http_chunk_splits[1:] | ||
return (b"", True) | ||
yield from self._wait('readchunk') | ||
|
||
if self._buffer: | ||
return self._read_nowait_chunk(-1) | ||
if not self._buffer: | ||
# end of file | ||
return (b"", False) | ||
elif self._http_chunk_splits is not None: | ||
while self._http_chunk_splits: | ||
pos = self._http_chunk_splits[0] | ||
self._http_chunk_splits = self._http_chunk_splits[1:] | ||
if pos > self._cursor: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Add a test for case if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FTR this condition was here because I was afraid of what would happen if some other calls than |
||
return (self._read_nowait(pos-self._cursor), True) | ||
return (self._read_nowait(-1), False) | ||
else: | ||
return b"" | ||
return (self._read_nowait_chunk(-1), False) | ||
|
||
@asyncio.coroutine | ||
def readexactly(self, n): | ||
|
@@ -378,6 +420,7 @@ def _read_nowait_chunk(self, n): | |
data = self._buffer.popleft() | ||
|
||
self._size -= len(data) | ||
self._cursor += len(data) | ||
return data | ||
|
||
def _read_nowait(self, n): | ||
|
@@ -438,7 +481,7 @@ def readany(self): | |
|
||
@asyncio.coroutine | ||
def readchunk(self): | ||
return b'' | ||
return (b'', False) | ||
|
||
@asyncio.coroutine | ||
def readexactly(self, n): | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
Make the HTTP client able to return HTTP chunks when chunked transfer encoding is used. |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please explain why you replaced
>=
with>
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If
required == chunk_len
, thenself._chunk_size == 0
and the "else" part does what is needed.So lines 552 and 553 looked like duplicated code, and deduplicating this allowed me to put
self.payload.end_http_chunk_receiving()
at only one place in the code.But the code will continue after the 'else' as opposed to the previous version where (False, None) was returned immediately. So the behaviour is not exactly the same even if the same result will be eventually returned.
So imo it looks better but if you want me to revert to the previous version tell me.
Also I'm going to remove the useless blank line(554)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ok, fine with the change