mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
Ruben Fiszel
13b521651b
* feat: add unit tests for S3BufferedReader.read and improve read method implementation * feat: refactor S3BufferedReader.read method and add unit tests for its functionality * feat: implement peek() on S3BufferedReader with buffered reads * fix(python-client): keep the read(size) contract and trim the test surface Drop the duplicated `TestS3BufferedReaderRead` class from `python-client/tests/wmill_client_test.py`: CI runs `pytest tests/` from `python-client/wmill`, so that legacy manual harness never executes, and the same assertions already live in `python-client/wmill/tests/test_s3_reader.py`. Narrow that file to the four behaviours a future change could break, and make the `bytes_generator` guard actually call `bytes_generator`. Align `peek()` with `io.BufferedReader.peek`, which does at most one read on the underlying stream, rather than looping until `size` bytes are buffered. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix(python-client): hold read1 to one underlying read read1 forwarded to read, so read1(-1) drained the whole object — the same unbounded buffering this branch removes from read. Now that a buffer exists, read1 can honour its own contract: fill only when the buffer is empty, then serve from it. Also treat read(None) as read(-1), per the BufferedReader contract, and pin that read(0) does not pull from the stream: that holds only because the drain sentinel is a negative size, and widening it to any falsy size would reintroduce whole-file buffering. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix(python-client): return from read1(0) without touching the stream A zero-length read has nothing to serve, so pulling a chunk to satisfy it both wastes a round trip and advances the stream. Guard it ahead of the fill, and pin it with a chunk source that counts pulls. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Tushar <tusharanshu18@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
98 lines
2.8 KiB
Python
98 lines
2.8 KiB
Python
"""Unit tests for S3BufferedReader: no network or env needed."""
|
|
|
|
from wmill.s3_reader import S3BufferedReader, bytes_generator
|
|
|
|
CHUNKS = [b"AAAAAAAAAA", b"BBBBBBBBBB", b"CCCCCCCCCC"]
|
|
|
|
|
|
class _FakeStream:
|
|
"""Stands in for the httpx streaming response the reader consumes."""
|
|
|
|
status_code = 200
|
|
|
|
def __init__(self, chunks):
|
|
self._chunks = chunks
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
pass
|
|
|
|
def iter_bytes(self):
|
|
return iter(self._chunks)
|
|
|
|
|
|
class CountingIterator:
|
|
"""Chunk source that records how many times the reader pulled from it."""
|
|
|
|
def __init__(self, chunks):
|
|
self._chunks = chunks
|
|
self.pulls = 0
|
|
|
|
def __iter__(self):
|
|
for chunk in self._chunks:
|
|
self.pulls += 1
|
|
yield chunk
|
|
|
|
|
|
class _FakeClient:
|
|
"""Stands in for the httpx client, so construction never touches the network."""
|
|
|
|
def __init__(self, chunks):
|
|
self._chunks = chunks
|
|
|
|
def stream(self, method, url, params=None, timeout=None):
|
|
return _FakeStream(self._chunks)
|
|
|
|
|
|
def make_reader(chunks):
|
|
reader = S3BufferedReader("ws", _FakeClient(chunks), "file.txt", None, None)
|
|
reader.__enter__()
|
|
return reader
|
|
|
|
|
|
def test_read_size_slices_chunks_and_keeps_the_remainder():
|
|
reader = make_reader(CHUNKS)
|
|
# read(0) must not pull from the stream: the "drain everything" sentinel is
|
|
# a negative size, and widening it to any falsy size would reintroduce the
|
|
# whole-file buffering this reader is built to avoid.
|
|
assert reader.read(0) == b""
|
|
assert reader.read(5) == b"AAAAA"
|
|
assert reader.read(5) == b"AAAAA"
|
|
assert reader.read(10) == b"BBBBBBBBBB"
|
|
assert reader.read(7) == b"CCCCCCC"
|
|
assert reader.read(5) == b"CCC"
|
|
assert reader.read(5) == b""
|
|
|
|
|
|
def test_read_all_drains_both_the_buffer_and_the_stream():
|
|
reader = make_reader(CHUNKS)
|
|
assert reader.read(5) == b"AAAAA"
|
|
assert reader.read(-1) == b"AAAAABBBBBBBBBBCCCCCCCCCC"
|
|
|
|
|
|
def test_bytes_generator_yields_50kb_slices_of_64kb_chunks():
|
|
reader = make_reader([b"x" * 65536] * 5)
|
|
sizes = [len(chunk) for chunk in bytes_generator(reader)]
|
|
assert max(sizes) <= 50 * 1024
|
|
assert sum(sizes) == 5 * 65536
|
|
|
|
|
|
def test_peek_does_not_consume():
|
|
reader = make_reader(CHUNKS)
|
|
assert reader.peek() == b"AAAAAAAAAA"
|
|
assert reader.read(10) == b"AAAAAAAAAA"
|
|
|
|
|
|
def test_read1_stops_after_one_chunk():
|
|
counting = CountingIterator(CHUNKS)
|
|
reader = make_reader(counting)
|
|
# A zero-length read must not touch the stream at all.
|
|
assert reader.read1(0) == b""
|
|
assert counting.pulls == 0
|
|
# read1(-1) must not drain the stream the way read(-1) does.
|
|
assert reader.read1(-1) == b"AAAAAAAAAA"
|
|
assert reader.read1(4) == b"BBBB"
|
|
assert counting.pulls == 2
|