diff --git a/CHANGES/3054.bugfix b/CHANGES/3054.bugfix new file mode 100644 index 00000000000..8276e49266b --- /dev/null +++ b/CHANGES/3054.bugfix @@ -0,0 +1 @@ +When using a server-request body as the `data=` argument of a client request, iterate over the content with `readany` instead of `readline` to avoid `Line too long` errors. diff --git a/aiohttp/payload.py b/aiohttp/payload.py index e3161961dd0..1b7ea6aac44 100644 --- a/aiohttp/payload.py +++ b/aiohttp/payload.py @@ -13,7 +13,7 @@ from . import hdrs from .helpers import (PY_36, content_disposition_header, guess_filename, parse_mimetype, sentinel) -from .streams import DEFAULT_LIMIT +from .streams import DEFAULT_LIMIT, StreamReader __all__ = ('PAYLOAD_REGISTRY', 'get_payload', 'payload_type', 'Payload', @@ -325,6 +325,12 @@ async def write(self, writer): self._iter = None +class StreamReaderPayload(AsyncIterablePayload): + + def __init__(self, value, *args, **kwargs): + super().__init__(value.iter_any(), *args, **kwargs) + + PAYLOAD_REGISTRY = PayloadRegistry() PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview)) PAYLOAD_REGISTRY.register(StringPayload, str) @@ -334,6 +340,7 @@ async def write(self, writer): PAYLOAD_REGISTRY.register( BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom)) PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase) +PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader) # try_last for giving a chance to more specialized async interables like # multidict.BodyPartReaderPayload override the default PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable, diff --git a/tests/test_payload.py b/tests/test_payload.py index 26efd2a5221..bbb3e0ffb3c 100644 --- a/tests/test_payload.py +++ b/tests/test_payload.py @@ -1,9 +1,10 @@ from io import StringIO +from unittest import mock import pytest from async_generator import async_generator -from aiohttp import payload +from aiohttp import payload, streams @pytest.fixture @@ -111,3 +112,20 @@ def test_async_iterable_payload_not_async_iterable(): with pytest.raises(TypeError): payload.AsyncIterablePayload(object()) + + +async def test_stream_reader_long_lines(loop): + DATA = b'0' * 1024 ** 3 + + stream = streams.StreamReader(mock.Mock(), loop=loop) + stream.feed_data(DATA) + stream.feed_eof() + body = payload.get_payload(stream) + + writer = mock.Mock() + writer.write.return_value = loop.create_future() + writer.write.return_value.set_result(None) + await body.write(writer) + writer.write.assert_called_once_with(mock.ANY) + (chunk,), _ = writer.write.call_args + assert len(chunk) == len(DATA)