mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
fix(python): back S3BufferedReader with a real BufferedReader (#12456)
S3BufferedReader subclassed BufferedReader without initializing it, so every method it did not override (readline, iteration, readinto, closed, TextIOWrapper) failed with "I/O operation on uninitialized object". The HTTP chunks now feed a RawIOBase that BufferedReader wraps, which gives the whole file API and replaces the hand-written read/read1/peek. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
2cb29f87a0
commit
9a074b1c73
@@ -1,5 +1,8 @@
|
||||
"""Unit tests for S3BufferedReader: no network or env needed."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
|
||||
from wmill.s3_reader import S3BufferedReader, bytes_generator
|
||||
|
||||
CHUNKS = [b"AAAAAAAAAA", b"BBBBBBBBBB", b"CCCCCCCCCC"]
|
||||
@@ -95,3 +98,23 @@ def test_read1_stops_after_one_chunk():
|
||||
assert reader.read1(-1) == b"AAAAAAAAAA"
|
||||
assert reader.read1(4) == b"BBBB"
|
||||
assert counting.pulls == 2
|
||||
|
||||
|
||||
CSV_CHUNKS = [b"name,val", b"ue\nalice,1\nb", b"ob,2\n"]
|
||||
|
||||
|
||||
def test_text_mode_csv_reads_rows_across_chunks():
|
||||
reader = make_reader(CSV_CHUNKS)
|
||||
rows = list(csv.reader(io.TextIOWrapper(reader, encoding="utf-8")))
|
||||
assert rows == [["name", "value"], ["alice", "1"], ["bob", "2"]]
|
||||
|
||||
|
||||
def test_line_and_buffer_reads_stop_where_asked():
|
||||
counting = CountingIterator(CSV_CHUNKS)
|
||||
reader = make_reader(counting)
|
||||
assert next(iter(reader)) == b"name,value\n"
|
||||
assert counting.pulls == 2
|
||||
target = bytearray(4)
|
||||
assert reader.readinto(target) == 4
|
||||
assert target == b"alic"
|
||||
assert list(reader) == [b"e,1\n", b"bob,2\n"]
|
||||
|
||||
@@ -1,9 +1,33 @@
|
||||
from io import BufferedReader, BytesIO
|
||||
from typing import Optional, Union
|
||||
from io import BufferedReader, BytesIO, RawIOBase
|
||||
from typing import Iterator, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class _HttpChunkStream(RawIOBase):
|
||||
"""Raw stream over the HTTP response's chunks, for BufferedReader to buffer."""
|
||||
|
||||
def __init__(self):
|
||||
self.chunks: Optional[Iterator[bytes]] = None
|
||||
self._pending = memoryview(b"")
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def readinto(self, buffer):
|
||||
if self.chunks is None:
|
||||
raise ValueError("S3BufferedReader must be opened with `with` before reading")
|
||||
while not self._pending:
|
||||
try:
|
||||
self._pending = memoryview(next(self.chunks))
|
||||
except StopIteration:
|
||||
return 0
|
||||
n = min(len(buffer), len(self._pending))
|
||||
buffer[:n] = self._pending[:n]
|
||||
self._pending = self._pending[n:]
|
||||
return n
|
||||
|
||||
|
||||
class S3BufferedReader(BufferedReader):
|
||||
"""Streaming buffered reader for S3 files via Windmill's S3 proxy.
|
||||
|
||||
@@ -15,6 +39,11 @@ class S3BufferedReader(BufferedReader):
|
||||
storage: Optional storage backend identifier
|
||||
"""
|
||||
def __init__(self, workspace: str, windmill_client: httpx.Client, file_key: str, s3_resource_path: Optional[str], storage: Optional[str]):
|
||||
# Every read, line and text-mode operation comes from BufferedReader
|
||||
# itself; overriding them one by one leaves the rest of the file API
|
||||
# failing on an uninitialized object.
|
||||
self._raw = _HttpChunkStream()
|
||||
super().__init__(self._raw)
|
||||
params = {
|
||||
"file_key": file_key,
|
||||
}
|
||||
@@ -28,7 +57,6 @@ class S3BufferedReader(BufferedReader):
|
||||
params=params,
|
||||
timeout=None,
|
||||
)
|
||||
self._buffer = bytearray()
|
||||
|
||||
def __enter__(self):
|
||||
reader = self._context_manager.__enter__()
|
||||
@@ -43,59 +71,11 @@ class S3BufferedReader(BufferedReader):
|
||||
request=reader.request,
|
||||
response=reader
|
||||
)
|
||||
self._iterator = reader.iter_bytes()
|
||||
self._raw.chunks = reader.iter_bytes()
|
||||
return self
|
||||
|
||||
def peek(self, size=0):
|
||||
"""Return buffered bytes without consuming them.
|
||||
|
||||
Reads the underlying stream at most once, so the amount returned may be
|
||||
more or less than `size`.
|
||||
"""
|
||||
if not self._buffer:
|
||||
self._fill(1)
|
||||
return bytes(self._buffer)
|
||||
|
||||
def _fill(self, limit):
|
||||
# iter_bytes() yields whole HTTP chunks (~64KB), so a caller asking for
|
||||
# `limit` bytes has to accumulate until the buffer holds that many.
|
||||
# A negative limit means drain the stream.
|
||||
while limit < 0 or len(self._buffer) < limit:
|
||||
try:
|
||||
self._buffer.extend(next(self._iterator))
|
||||
except StopIteration:
|
||||
break
|
||||
|
||||
def read(self, size=-1):
|
||||
# BufferedReader.read(None) is documented as equivalent to read(-1).
|
||||
if size is None:
|
||||
size = -1
|
||||
self._fill(size)
|
||||
if size < 0:
|
||||
result = bytes(self._buffer)
|
||||
self._buffer.clear()
|
||||
return result
|
||||
result = bytes(self._buffer[:size])
|
||||
del self._buffer[:size]
|
||||
return result
|
||||
|
||||
def read1(self, size=-1):
|
||||
"""Return up to `size` bytes, reading the underlying stream at most once.
|
||||
|
||||
Unlike `read`, a negative `size` returns only what is already buffered
|
||||
rather than draining the whole object.
|
||||
"""
|
||||
if size == 0:
|
||||
return b""
|
||||
if not self._buffer:
|
||||
self._fill(1)
|
||||
if size is None or size < 0:
|
||||
size = len(self._buffer)
|
||||
result = bytes(self._buffer[:size])
|
||||
del self._buffer[:size]
|
||||
return result
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.close()
|
||||
self._context_manager.__exit__(*args)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user