mirror of
https://github.com/stablyai/orca.git
synced 2026-10-17 16:02:55 +00:00
fix(speech): keep segmented download resumes progressing
This commit is contained in:
@@ -235,6 +235,127 @@ describe('ModelManager download resume', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('continues through more than eight advancing range segments', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
for (let offset = 1; offset <= 9; offset += 1) {
|
||||
scriptRequest((sentHeaders) => {
|
||||
expect(sentHeaders.range).toBe(`bytes=${offset}-`)
|
||||
return {
|
||||
statusCode: 206,
|
||||
headers: {
|
||||
'content-length': '1',
|
||||
'content-range': `bytes ${offset}-${offset}/${PAYLOAD.length}`
|
||||
},
|
||||
chunks: [PAYLOAD.subarray(offset, offset + 1)]
|
||||
}
|
||||
})
|
||||
}
|
||||
scriptRequest((sentHeaders) => {
|
||||
expect(sentHeaders.range).toBe('bytes=10-')
|
||||
return {
|
||||
statusCode: 206,
|
||||
headers: {
|
||||
'content-length': '10',
|
||||
'content-range': `bytes 10-19/${PAYLOAD.length}`
|
||||
},
|
||||
chunks: [PAYLOAD.subarray(10)]
|
||||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 1))
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
new AbortController().signal
|
||||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(10)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('bounds pathological advancing range segment requests', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
let bytesWritten = 0
|
||||
const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => {
|
||||
bytesWritten += 1
|
||||
writeFileSync(archivePath, Buffer.alloc(bytesWritten))
|
||||
return Promise.resolve()
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
300,
|
||||
'm',
|
||||
() => false,
|
||||
new AbortController().signal
|
||||
)
|
||||
).rejects.toThrow(/after 256 attempts/)
|
||||
|
||||
expect(downloadFileMock).toHaveBeenCalledTimes(256)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the known archive total when Content-Range omits it', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
scriptRequest((sentHeaders) => {
|
||||
expect(sentHeaders.range).toBe('bytes=10-')
|
||||
return {
|
||||
statusCode: 206,
|
||||
headers: {
|
||||
'content-length': '5',
|
||||
'content-range': 'bytes 10-14/*'
|
||||
},
|
||||
chunks: [PAYLOAD.subarray(10, 15)]
|
||||
}
|
||||
})
|
||||
scriptRequest((sentHeaders) => {
|
||||
expect(sentHeaders.range).toBe('bytes=15-')
|
||||
return {
|
||||
statusCode: 206,
|
||||
headers: {
|
||||
'content-length': '5',
|
||||
'content-range': 'bytes 15-19/*'
|
||||
},
|
||||
chunks: [PAYLOAD.subarray(15)]
|
||||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 10))
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
new AbortController().signal
|
||||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(2)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects and discards a partial when Content-Range does not match the offset', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
|
||||
@@ -51,7 +51,10 @@ const DOWNLOAD_IDLE_TIMEOUT_MS = 120_000
|
||||
// request lets those connections finish instead of failing the same way on
|
||||
// every full restart.
|
||||
const DOWNLOAD_RETRY_DELAYS_MS = [1_000, 2_000, 4_000]
|
||||
const MAX_DOWNLOAD_ATTEMPTS = 8
|
||||
const MAX_DOWNLOAD_FAILURES = 8
|
||||
// Why: successful bounded ranges may require more than the failure budget, but
|
||||
// a server returning pathological tiny segments must not create an endless loop.
|
||||
const MAX_ADVANCING_RANGE_SEGMENTS = 256
|
||||
// Why: a longer server window should be surfaced for a later manual retry,
|
||||
// rather than leaving the settings download looking stuck for minutes.
|
||||
const MAX_RETRY_AFTER_MS = 120_000
|
||||
@@ -450,11 +453,13 @@ export class ModelManager {
|
||||
isAborted: () => boolean,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
let attempt = 0
|
||||
let requestCount = 0
|
||||
let retryFailures = 0
|
||||
let advancingRangeSegments = 0
|
||||
let noProgressStreak = 0
|
||||
const totals: DownloadTotals = { totalBytes: expectedSize }
|
||||
for (;;) {
|
||||
attempt += 1
|
||||
requestCount += 1
|
||||
const offset = this.getPartialArchiveBytes(archivePath)
|
||||
// Why: the transport can fail after the final byte reached disk; the
|
||||
// caller's SHA-256 check is the authoritative completion test.
|
||||
@@ -487,18 +492,19 @@ export class ModelManager {
|
||||
const incompleteResponse = new Error(
|
||||
`Model download response ended at ${receivedBytes} of ${totals.totalBytes} bytes`
|
||||
)
|
||||
if (attempt >= MAX_DOWNLOAD_ATTEMPTS) {
|
||||
throw describeInterruptedDownload(
|
||||
incompleteResponse,
|
||||
receivedBytes,
|
||||
totals.totalBytes,
|
||||
attempt
|
||||
)
|
||||
}
|
||||
if (receivedBytes > offset) {
|
||||
// Why: some proxies return a valid but bounded range segment; request
|
||||
// the next segment immediately instead of failing checksum or waiting.
|
||||
noProgressStreak = 0
|
||||
advancingRangeSegments += 1
|
||||
if (advancingRangeSegments >= MAX_ADVANCING_RANGE_SEGMENTS) {
|
||||
throw describeInterruptedDownload(
|
||||
incompleteResponse,
|
||||
receivedBytes,
|
||||
totals.totalBytes,
|
||||
requestCount
|
||||
)
|
||||
}
|
||||
continue
|
||||
}
|
||||
const retryableIncompleteResponse = incompleteResponse as HttpStatusError
|
||||
@@ -516,11 +522,12 @@ export class ModelManager {
|
||||
if (!isRetryableDownloadError(err)) {
|
||||
throw err
|
||||
}
|
||||
retryFailures += 1
|
||||
if (
|
||||
attempt >= MAX_DOWNLOAD_ATTEMPTS ||
|
||||
retryFailures >= MAX_DOWNLOAD_FAILURES ||
|
||||
noProgressStreak > DOWNLOAD_RETRY_DELAYS_MS.length
|
||||
) {
|
||||
throw describeInterruptedDownload(err, receivedBytes, totals.totalBytes, attempt)
|
||||
throw describeInterruptedDownload(err, receivedBytes, totals.totalBytes, requestCount)
|
||||
}
|
||||
const retryAfterMs = (err as HttpStatusError).retryAfterMs
|
||||
if (retryAfterMs !== undefined && retryAfterMs > MAX_RETRY_AFTER_MS) {
|
||||
@@ -529,7 +536,11 @@ export class ModelManager {
|
||||
`HTTP ${statusCode}; server requested retry after ${Math.ceil(retryAfterMs / 1_000)} seconds`
|
||||
)
|
||||
}
|
||||
console.warn(`[speech] Model download attempt ${attempt} failed, retrying:`, modelId, err)
|
||||
console.warn(
|
||||
`[speech] Model download attempt ${requestCount} failed, retrying:`,
|
||||
modelId,
|
||||
err
|
||||
)
|
||||
await sleepUnlessAborted(
|
||||
retryAfterMs ??
|
||||
DOWNLOAD_RETRY_DELAYS_MS[
|
||||
@@ -721,12 +732,13 @@ export class ModelManager {
|
||||
// Why: a 200 despite our Range request means the server restarted the
|
||||
// transfer from byte zero, so the partial file must be overwritten.
|
||||
const progressBase = resumed ? resumeOffset : 0
|
||||
const totalSize =
|
||||
resumed && contentRange?.totalBytes !== undefined
|
||||
? contentRange.totalBytes
|
||||
: parsedLength > 0
|
||||
? progressBase + parsedLength
|
||||
: expectedSize
|
||||
// Why: Content-Length on a 206 describes only this segment; when
|
||||
// Content-Range uses '*', retain the known complete archive size.
|
||||
const totalSize = resumed
|
||||
? (contentRange?.totalBytes ?? totals?.totalBytes ?? expectedSize)
|
||||
: parsedLength > 0
|
||||
? parsedLength
|
||||
: expectedSize
|
||||
if (totals) {
|
||||
totals.totalBytes = totalSize
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user