Merge pull request #1159 from warmbly/feat/attachment-preflight-inventory

feat: add offline archive attachment preflight with resumable SHA-256-bound pages
This commit is contained in:
Matthew Meszaros
2026-11-05 13:59:06 +00:00
committed by GitHub
11 changed files with 1078 additions and 2 deletions
+65
View File
@@ -0,0 +1,65 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"github.com/warmbly/warmbly/internal/app/orgtransfer"
)
func runOrgAttachmentsPreflight(ctx context.Context, args []string) error {
fs := newFlagSet("org attachments-preflight")
file := fs.String("file", "", "immutable local workspace archive to read (required; no database or network)")
after := fs.Int("after", 0, "skip this many attachment rows, using next_after from the previous page")
limit := fs.Int("limit", 25, "maximum attachment rows to inspect (1 to 100)")
digest := fs.String("expect-sha256", "", "archive_sha256 from the previous page (required with --after)")
if err := fs.Parse(args); err != nil {
return err
}
if err := noExtraArgs(fs); err != nil {
return err
}
if *file == "" {
return errors.New("--file is required")
}
pathInfo, err := os.Stat(*file)
if err != nil {
return err
}
if !pathInfo.Mode().IsRegular() {
return errors.New("--file must name a regular local archive")
}
f, err := openInventoryFile(*file)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return errors.New("--file must name a regular local archive")
}
if !os.SameFile(pathInfo, info) {
return errors.New("archive changed before inventory; retry with an immutable file")
}
report, err := orgtransfer.InventoryAttachments(ctx, &fileArchive{f: f, size: info.Size()}, orgtransfer.AttachmentInventoryOptions{
After: *after, Limit: *limit, SHA256: *digest,
})
if err != nil {
return err
}
finalInfo, err := f.Stat()
if err != nil {
return err
}
if finalInfo.Size() != info.Size() || !finalInfo.ModTime().Equal(info.ModTime()) {
return errors.New("archive changed during inventory; discard this page")
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(report)
}
@@ -0,0 +1,9 @@
//go:build !linux && !darwin && !freebsd && !openbsd && !netbsd && !dragonfly
package main
import "os"
func openInventoryFile(path string) (*os.File, error) {
return os.Open(path)
}
@@ -0,0 +1,13 @@
//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
package main
import (
"os"
"syscall"
)
func openInventoryFile(path string) (*os.File, error) {
// Keep descriptor verification reachable if the path changes to a FIFO.
return os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0)
}
@@ -0,0 +1,39 @@
//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly
package main
import (
"os"
"path/filepath"
"syscall"
"testing"
"time"
)
func TestOrgAttachmentsPreflightRefusesFIFOWithoutWriter(t *testing.T) {
name := filepath.Join(t.TempDir(), "input.fifo")
if err := syscall.Mkfifo(name, 0600); err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() { done <- dispatch(t.Context(), []string{"org", "attachments-preflight", "--file", name}) }()
select {
case err := <-done:
if err == nil || err.Error() != "--file must name a regular local archive" {
t.Fatalf("regular-file refusal: %v", err)
}
case <-time.After(time.Second):
// Release and join the dispatch before reporting a failed bounded wait.
f, err := os.OpenFile(name, os.O_RDWR|syscall.O_NONBLOCK, 0600)
if err != nil {
t.Fatal(err)
}
defer f.Close()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("dispatch did not return after opening the FIFO writer")
}
t.Fatal("regular-file refusal must complete without a FIFO writer")
}
}
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"testing"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/app/orgtransfer"
"github.com/warmbly/warmbly/internal/models"
)
func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) {
t.Setenv("PRIMARY_DB", "invalid-unused-database")
t.Setenv("REDIS", "invalid-unused-cache")
t.Setenv("KMS_PROVIDER", "invalid-unused-provider")
path := filepath.Join(t.TempDir(), "workspace.warmbly.zip")
f, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
w := zip.NewWriter(f)
org, campaign := uuid.New(), uuid.New()
m := orgtransfer.Manifest{
Kind: orgtransfer.ArchiveKind, FormatVersion: 1, OrganizationID: org,
Tables: []orgtransfer.ManifestTable{{Name: "campaigns", Rows: 1}, {Name: "campaign_attachments", Rows: 2}},
}
var rows bytes.Buffer
for i := range 2 {
name := fmt.Sprintf("notes-%d.txt", i)
body := []byte("archived original bytes\n")
key := models.AttachmentObjectKey(campaign, name)
blobPath := fmt.Sprintf("blobs/%d", i)
m.Blobs = append(m.Blobs, orgtransfer.ManifestBlob{Key: key, Path: blobPath, Size: int64(len(body)), Table: "campaign_attachments", Column: "s3_key"})
member, err := w.Create(blobPath)
if err != nil {
t.Fatal(err)
}
if _, err := member.Write(body); err != nil {
t.Fatal(err)
}
if err := json.NewEncoder(&rows).Encode(models.CampaignAttachment{
ID: uuid.New(), CampaignID: campaign, Filename: name, S3Key: key, Size: int64(len(body)), MimeType: "text/plain",
}); err != nil {
t.Fatal(err)
}
}
campaignRow, err := json.Marshal(map[string]any{"id": campaign, "organization_id": org})
if err != nil {
t.Fatal(err)
}
for name, body := range map[string][]byte{"data/campaigns.ndjson": campaignRow, "data/campaign_attachments.ndjson": rows.Bytes()} {
member, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := member.Write(body); err != nil {
t.Fatal(err)
}
}
manifest, err := w.Create("manifest.json")
if err != nil {
t.Fatal(err)
}
if err := json.NewEncoder(manifest).Encode(m); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
out, err := os.CreateTemp(t.TempDir(), "report")
if err != nil {
t.Fatal(err)
}
defer out.Close()
stdout := os.Stdout
os.Stdout = out
defer func() { os.Stdout = stdout }()
if err := dispatch(t.Context(), []string{"org", "attachments-preflight", "--file", path, "--limit", "1"}); err != nil {
t.Fatal(err)
}
if _, err := out.Seek(0, io.SeekStart); err != nil {
t.Fatal(err)
}
var report orgtransfer.AttachmentInventory
if err := json.NewDecoder(out).Decode(&report); err != nil {
t.Fatal(err)
}
if report.Done || report.TotalRows != 2 || len(report.Results) != 1 || report.NextAfter != 1 || len(report.ArchiveSHA256) != 64 {
t.Fatalf("offline command report: %+v", report)
}
if report.Results[0].Status != "matches_upload_profile" {
t.Fatalf("valid attachment result: %+v", report.Results[0])
}
after, err := os.ReadFile(path)
if err != nil || string(after) != string(before) {
t.Fatal("command changed the archive")
}
}
+2 -1
View File
@@ -131,6 +131,7 @@ var commands = []command{
{"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"},
{"org export", "Write a whole workspace to a portable archive file", composeExec + "org export --org you@example.com --out /tmp/workspace.warmbly.zip"},
{"org import", "Apply an archive to a workspace on this instance", composeExec + "org import --org you@example.com --file /tmp/workspace.warmbly.zip"},
{"org attachments-preflight", "Read one page of private attachment type results from a local archive, offline", "warmblyctl org attachments-preflight --file ./workspace.warmbly.zip --limit 25"},
{"fleet join-token", "Issue the token a machine needs to join the fleet. Shown once", composeExec + "fleet join-token"},
{"fleet list", "Every worker and consumer: role, version, liveness and usage", composeExec + "fleet list"},
{"fleet version", "Show or set the version every node should be running", composeExec + "fleet version v1.4.2"},
@@ -191,7 +192,7 @@ Anything piped into a container needs exec -T, which is what turns the TTY off.
Anything that prompts needs a TTY, so run it without -T.
Environment:
PRIMARY_DB Postgres connection string. Every operator command except hash-password needs it.
PRIMARY_DB Postgres connection string. Not needed by hash-password or org attachments-preflight.
REDIS Redis URL. setup-link needs it, and reset-password needs it to mint a link.
APP_URL Base URL every printed link is built from.
WARMBLY_API_KEY API key (wmbly_...) for the API commands.
+3 -1
View File
@@ -31,6 +31,8 @@ func runOrg(ctx context.Context, args []string) error {
return runOrgExport(ctx, args[1:])
case "import":
return runOrgImport(ctx, args[1:])
case "attachments-preflight":
return runOrgAttachmentsPreflight(ctx, args[1:])
}
orgUsage(os.Stderr)
@@ -38,7 +40,7 @@ func runOrg(ctx context.Context, args []string) error {
}
func orgUsage(w *os.File) {
fmt.Fprint(w, "Move a workspace between instances.\n\nUsage:\n warmblyctl org <subcommand> [flags]\n\nSubcommands:\n")
fmt.Fprint(w, "Move a workspace between instances or inspect a local archive.\n\nUsage:\n warmblyctl org <subcommand> [flags]\n\nSubcommands:\n")
for _, c := range commands {
if !strings.HasPrefix(c.name, "org ") {
continue
@@ -609,6 +609,56 @@ The whole import runs in one transaction: if any part fails, nothing lands and t
Billing history, plan overrides, worker placement, mailbox sync checkpoints and warmup pool membership are exported for the record but never applied: each belongs to the instance rather than to the workspace. [Export and import](/guides/workspace-export-import/) has the full table.
## org attachments-preflight
With `jq` installed, write reports to a new private directory and resume only when the first page succeeds and its `done` field is false. The subshell keeps its restrictive permissions and error handling local to this example:
```bash
(
set -eu
umask 077
report_dir="$(mktemp -d ./attachment-preflight.XXXXXXXX)"
printf 'Reports: %s\n' "$report_dir"
warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \
--limit 25 > "$report_dir/page-1.json"
page_done="$(jq -r '.done' "$report_dir/page-1.json")"
if [ "$page_done" = false ]; then
next_after="$(jq -r '.next_after' "$report_dir/page-1.json")"
archive_sha256="$(jq -r '.archive_sha256' "$report_dir/page-1.json")"
warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \
--after "$next_after" --expect-sha256 "$archive_sha256" \
--limit 25 > "$report_dir/page-2.json"
fi
)
```
Reads private campaign attachment rows and their bytes from an **existing local workspace archive**, comparing each selected file with the [new-upload profile](/guides/campaigns/#campaign-attachments). This command works offline without `PRIMARY_DB`, a storage client, credentials or a destination workspace. It opens the archive for reading and prints JSON to stdout. Keep the archive immutable and protect reports as workspace data: they contain filenames, identifiers and content fingerprints. Obtaining an archive is a separate, authorized export operation; this command does not create or download one.
| Flag | Does |
|---|---|
| `--file` | Regular local archive file. Required |
| `--limit` | Rows inspected per page, from 1 to 100. Default 25 |
| `--after` | Rows to skip. Use the previous successful page's `next_after`, starting at zero |
| `--expect-sha256` | Expected archive fingerprint. Required when resuming with `--after` |
Resume using the same binary, archive and classifier build. Every page hashes the whole compressed archive before and after inspection and rescans campaign and attachment metadata. Only the selected attachment members are decompressed for classification. A failed or interrupted run emits no report; retry the same page. Save each successful report before advancing. `done` means all attachment **rows in this archive** have been visited, not that every file matches the profile. `classifier_build` identifies the inspecting binary; an unstamped development binary needs its source commit recorded separately. `source_version` is informational archive metadata.
| Per-file `status` | Meaning |
|---|---|
| `matches_upload_profile` | Original bytes pass this binary's upload classifier. `canonical_mime` is its result; `stored_mime` remains unchanged |
| `outside_upload_profile` | The file does not meet the current upload profile, including its size limits |
| `missing_blob` | The row has no matching available private blob in the archive |
| `metadata_mismatch` | Campaign/key scope or stored sizes do not agree |
| `unreadable_blob` | The selected member could not be read with its ZIP integrity checks |
`content_sha256` fingerprints bytes that were successfully read within the upload size limit. Missing, inconsistent, oversized and empty members do not receive one. `unreferenced_blobs` counts attachment blob references without an attachment row; their content is not classified. Review those separately. Zero exit status means the report completed, including any nonmatching or unknown results. Structural archive errors and invalid options exit nonzero.
The command accepts archive format versions supported by this binary, including v0.6.75 archives. It requires complete `campaigns` and `campaign_attachments` tables with matching row counts, refuses duplicate paths and attachment identifiers, and checks campaign ownership against the archive organization. Every member read has its CRC32 checked, including a recorded zero checksum. Archives without private blob bytes produce `missing_blob` results. Limits are 8 GiB compressed archive size, the existing 64 MiB manifest limit, 64 MiB decompressed per metadata table, 1 MiB per metadata row, and the upload profile's 15 MiB per selected attachment, processed sequentially. Cancellation is checked before archive source reads, including ZIP indexing and compressed member reads. Classifier calls remain synchronous. ZIP directory indexing and classifier work still consume resources; page size is not a process memory or runtime guarantee.
The input path must resolve to a regular file before opening, and the opened descriptor must identify the same regular file. Linux, macOS and supported BSD builds use a nonblocking open so descriptor verification also covers a path changed to a FIFO. These checks and the archive fingerprints detect ordinary file changes; they do not create a filesystem snapshot or interrupt a source read already in progress.
This is an inventory observation, not malware certification, import approval or send eligibility. It does not change archives, stored rows, private bytes, MIME metadata, duplication, restore or queued sends. It does not inspect live storage, retained queue messages, orphan objects, other archives or concurrent writes. Any later eligibility transition needs its own compatible migration stage covering old readers, retained messages, portability and rollback.
## The API commands
Everything in this section talks to the public REST API with an API key, so it runs from anywhere: your laptop, a CI job, an agent's sandbox. It needs no database access and works against the hosted service and self-hosted instances alike.
@@ -288,3 +288,5 @@ Reply and forward drafts kept in your browser are also excluded. Finish or copy
Campaign attachments travel with their original bytes and metadata, including files accepted by v0.6.75 and formats outside the [new-upload profile](/guides/campaigns/#campaign-attachments). Workspace and admin import use the same preservation policy. Preflight and import report that restoring private campaign files does not certify their content. Those members are retained rather than omitted because of their format. Review them before using imported campaigns. Public images keep their separate image-validation policy.
Archive format and row contracts stay compatible with v0.6.75. The new-upload policy adds no required archive field, validation certificate or send hold. An archive filename or source version is not evidence that every private member passed that policy.
Operators can use [`warmblyctl org attachments-preflight`](/development/warmblyctl/#org-attachments-preflight) to inspect an existing local archive in resumable pages against the current upload profile. The offline report records matches, files outside the profile and unavailable or inconsistent bytes. It preserves the archive and does not change restore or queued-send behavior. Its coverage is limited to that archive's attachment rows; it does not certify live storage or retained queue messages.
@@ -0,0 +1,316 @@
package orgtransfer
import (
"archive/zip"
"bufio"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash/crc32"
"io"
"strings"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
"github.com/warmbly/warmbly/internal/pkg/attachmenttype"
"github.com/warmbly/warmbly/internal/version"
)
const maxInventoryTableBytes = 64 << 20
// AttachmentInventoryOptions selects a page in an immutable local archive.
type AttachmentInventoryOptions struct {
After int
Limit int
SHA256 string
}
// AttachmentInventory reports observations only; it grants no send eligibility.
type AttachmentInventory struct {
ArchiveSHA256 string `json:"archive_sha256"`
ClassifierBuild version.Info `json:"classifier_build"`
OrganizationID uuid.UUID `json:"organization_id"`
SourceVersion string `json:"source_version"`
TotalRows int `json:"total_rows"`
UnreferencedBlobs int `json:"unreferenced_blobs"`
After int `json:"after"`
NextAfter int `json:"next_after"`
Done bool `json:"done"`
Results []AttachmentInventoryItem `json:"results"`
}
// AttachmentInventoryItem compares one archived row and blob with the upload profile.
type AttachmentInventoryItem struct {
Row int `json:"row"`
ID uuid.UUID `json:"id"`
CampaignID uuid.UUID `json:"campaign_id"`
Filename string `json:"filename"`
StoredMIME string `json:"stored_mime"`
StoredBytes int64 `json:"stored_bytes"`
Status string `json:"status"`
ContentSHA256 string `json:"content_sha256,omitempty"`
CanonicalMIME string `json:"canonical_mime,omitempty"`
}
// InventoryAttachments reads archive bytes without a database, storage client or importer.
func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts AttachmentInventoryOptions) (result *AttachmentInventory, resultErr error) {
defer func() {
if err := ctx.Err(); err != nil {
result, resultErr = nil, err
}
}()
if opts.After < 0 || opts.Limit < 1 || opts.Limit > 100 {
return nil, errors.New("after must be nonnegative and limit must be between 1 and 100")
}
if opts.After > 0 && opts.SHA256 == "" {
return nil, errors.New("resuming requires the previous report's archive SHA-256")
}
if opts.SHA256 != "" {
b, err := hex.DecodeString(opts.SHA256)
if err != nil || len(b) != sha256.Size {
return nil, errors.New("expected archive SHA-256 must contain 64 hexadecimal characters")
}
}
if archive.Size() <= 0 || archive.Size() > models.MaxOrgArchiveUploadBytes {
return nil, errors.New("archive must be between 1 byte and 8 GiB")
}
digest, err := inventoryDigest(ctx, archive)
if err != nil {
return nil, err
}
if opts.SHA256 != "" && !strings.EqualFold(opts.SHA256, digest) {
return nil, errors.New("archive SHA-256 differs from the requested inventory")
}
zr, err := zip.NewReader(inventoryContextReaderAt{ctx, archive}, archive.Size())
if ctx.Err() != nil {
return nil, ctx.Err()
}
if err != nil {
return nil, errors.New("archive is not a readable ZIP")
}
entries := make(map[string]*zip.File, len(zr.File))
for _, f := range zr.File {
if entries[f.Name] != nil {
return nil, errors.New("archive contains duplicate ZIP paths")
}
entries[f.Name] = f
}
manifestEntry := entries[manifestPath]
if manifestEntry == nil || manifestEntry.UncompressedSize64 > maxManifestBytes {
return nil, errors.New("inventory requires manifest.json within the 64 MiB limit")
}
mr, err := manifestEntry.Open()
if err != nil {
return nil, errors.New("cannot open archive manifest")
}
manifestBody, readErr := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, mr}, maxManifestBytes+1))
_ = mr.Close()
if readErr != nil || len(manifestBody) > maxManifestBytes || crc32.ChecksumIEEE(manifestBody) != manifestEntry.CRC32 {
return nil, errors.New("cannot read archive manifest within the 64 MiB limit")
}
var m Manifest
if err := json.Unmarshal(manifestBody, &m); err != nil {
return nil, errors.New("archive manifest is not readable JSON")
}
if err := m.Validate(); err != nil {
return nil, err
}
campaigns := map[uuid.UUID]bool{}
err = inventoryRows(ctx, entries, &m, "campaigns", func(b []byte) error {
var c struct {
ID uuid.UUID `json:"id"`
OrganizationID uuid.UUID `json:"organization_id"`
}
if err := json.Unmarshal(b, &c); err != nil || c.ID == uuid.Nil || c.OrganizationID != m.OrganizationID || campaigns[c.ID] {
return errors.New("campaign rows must uniquely belong to the archive organization")
}
campaigns[c.ID] = true
return nil
})
if err != nil {
return nil, err
}
blobs := map[string]ManifestBlob{}
for _, b := range m.Blobs {
if b.Table != "campaign_attachments" && !strings.HasPrefix(b.Key, "attachments/") {
continue
}
if _, exists := blobs[b.Key]; exists {
return nil, errors.New("archive contains duplicate attachment blob keys")
}
blobs[b.Key] = b
}
report := &AttachmentInventory{ArchiveSHA256: digest, ClassifierBuild: version.Current(), OrganizationID: m.OrganizationID,
SourceVersion: m.SourceAppVersion, After: opts.After, NextAfter: opts.After, Results: []AttachmentInventoryItem{}}
ids := map[uuid.UUID]bool{}
keys := map[string]bool{}
scope := blobKeyScope{orgID: m.OrganizationID, campaigns: campaigns}
err = inventoryRows(ctx, entries, &m, "campaign_attachments", func(b []byte) error {
var a models.CampaignAttachment
if err := json.Unmarshal(b, &a); err != nil || a.ID == uuid.Nil || ids[a.ID] || !campaigns[a.CampaignID] {
return errors.New("attachment rows must uniquely reference an archived campaign")
}
ids[a.ID] = true
keys[a.S3Key] = true
report.TotalRows++
if report.TotalRows <= opts.After || len(report.Results) == opts.Limit {
return nil
}
item := AttachmentInventoryItem{Row: report.TotalRows, ID: a.ID, CampaignID: a.CampaignID,
Filename: a.Filename, StoredMIME: a.MimeType, StoredBytes: a.Size, Status: "metadata_mismatch"}
ref, exists := blobs[a.S3Key]
switch {
case !scope.allows(a.S3Key) || !strings.HasPrefix(a.S3Key, "attachments/"+a.CampaignID.String()+"/"):
item.Status = "metadata_mismatch"
case !exists || entries[ref.Path] == nil:
item.Status = "missing_blob"
case ref.Table != "campaign_attachments" || ref.Column != "s3_key" || !strings.HasPrefix(ref.Path, blobPrefix) || ref.Size != a.Size || a.Size < 0 || uint64(a.Size) != entries[ref.Path].UncompressedSize64:
item.Status = "metadata_mismatch"
case a.Size == 0 || a.Size > attachmenttype.MaxBytes:
item.Status = "outside_upload_profile"
default:
item = inventoryContent(ctx, entries[ref.Path], item)
}
report.Results = append(report.Results, item)
report.NextAfter = report.TotalRows
return nil
})
if err != nil {
return nil, err
}
if opts.After > report.TotalRows {
return nil, errors.New("after is beyond the attachment row count")
}
for key := range blobs {
if !keys[key] {
report.UnreferencedBlobs++
}
}
report.Done = report.NextAfter == report.TotalRows
finalDigest, err := inventoryDigest(ctx, archive)
if err != nil {
return nil, err
}
if finalDigest != digest {
return nil, errors.New("archive changed during inventory; discard this page")
}
return report, ctx.Err()
}
func inventoryContent(ctx context.Context, entry *zip.File, item AttachmentInventoryItem) AttachmentInventoryItem {
item.Status = "unreadable_blob"
r, err := entry.Open()
if err != nil {
return item
}
defer r.Close()
body, err := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, r}, attachmenttype.MaxBytes+1))
if err != nil || crc32.ChecksumIEEE(body) != entry.CRC32 {
return item
}
if int64(len(body)) != item.StoredBytes {
item.Status = "metadata_mismatch"
return item
}
digest := sha256.Sum256(body)
item.ContentSHA256 = hex.EncodeToString(digest[:])
t, err := attachmenttype.Validate(item.Filename, body)
item.Status = "outside_upload_profile"
if err == nil {
item.Status = "matches_upload_profile"
item.CanonicalMIME = t.MIME
}
return item
}
func inventoryRows(ctx context.Context, entries map[string]*zip.File, m *Manifest, table string, visit func([]byte) error) error {
var mt *ManifestTable
for i := range m.Tables {
if m.Tables[i].Name == table {
if mt != nil {
return fmt.Errorf("duplicate %s table in manifest", table)
}
mt = &m.Tables[i]
}
}
if mt == nil || mt.Truncated || mt.Rows < 0 {
return fmt.Errorf("inventory requires a complete %s table in the archive", table)
}
entry := entries[dataPath(table)]
if entry == nil {
return fmt.Errorf("archive has no %s data", table)
}
if entry.UncompressedSize64 > maxInventoryTableBytes {
return fmt.Errorf("%s metadata exceeds the 64 MiB inventory limit", table)
}
r, err := entry.Open()
if err != nil {
return fmt.Errorf("cannot open %s metadata", table)
}
defer r.Close()
checksum := crc32.NewIEEE()
limited := &io.LimitedReader{R: io.TeeReader(inventoryContextReader{ctx, r}, checksum), N: maxInventoryTableBytes + 1}
scanner := bufio.NewScanner(limited)
scanner.Buffer(make([]byte, 64<<10), 1<<20)
var rows int64
for scanner.Scan() {
if len(strings.TrimSpace(scanner.Text())) == 0 {
continue
}
if err := visit(scanner.Bytes()); err != nil {
return err
}
rows++
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("cannot read %s metadata (1 MiB row limit): %w", table, err)
}
if limited.N == 0 || rows != mt.Rows {
return fmt.Errorf("%s metadata size or row count differs from the manifest", table)
}
if checksum.Sum32() != entry.CRC32 {
return fmt.Errorf("cannot verify %s metadata integrity", table)
}
return nil
}
func inventoryDigest(ctx context.Context, archive ReaderAtSizer) (string, error) {
h := sha256.New()
size := archive.Size()
n, err := io.Copy(h, inventoryContextReader{ctx, io.NewSectionReader(archive, 0, size)})
if err != nil {
return "", err
}
if n != size {
return "", errors.New("archive size changed during fingerprinting")
}
return hex.EncodeToString(h.Sum(nil)), nil
}
type inventoryContextReader struct {
ctx context.Context
r io.Reader
}
type inventoryContextReaderAt struct {
ctx context.Context
r io.ReaderAt
}
func (r inventoryContextReaderAt) ReadAt(b []byte, off int64) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.r.ReadAt(b, off)
}
func (r inventoryContextReader) Read(b []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.r.Read(b)
}
@@ -0,0 +1,467 @@
package orgtransfer
import (
"archive/zip"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"hash/crc32"
"io"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/warmbly/warmbly/internal/models"
)
type inventoryArchive struct{ *bytes.Reader }
func (a inventoryArchive) Size() int64 { return a.Reader.Size() }
func inventoryFixture(t *testing.T, edit func(*Manifest, []models.CampaignAttachment, map[string][]byte)) []byte {
t.Helper()
org, campaign := uuid.New(), uuid.New()
m := &Manifest{Kind: ArchiveKind, FormatVersion: 1, OrganizationID: org, SourceAppVersion: "v0.6.75",
Tables: []ManifestTable{{Name: "campaigns", Rows: 1}, {Name: "campaign_attachments", Rows: 4}}}
entries := map[string][]byte{}
rows := make([]models.CampaignAttachment, 4)
for i, name := range []string{"notes.txt", "legacy.unknown", "missing.txt", "mismatch.txt"} {
body := []byte("archived original bytes\n")
key := models.AttachmentObjectKey(campaign, name)
rows[i] = models.CampaignAttachment{ID: uuid.New(), CampaignID: campaign, Filename: name, Size: int64(len(body)), MimeType: "application/x-existing", S3Key: key}
ref := ManifestBlob{Key: key, Path: blobArchivePath(i, key), Size: int64(len(body)), Table: "campaign_attachments", Column: "s3_key"}
m.Blobs = append(m.Blobs, ref)
if i != 2 {
entries[ref.Path] = body
}
}
rows[3].Size++
c, _ := json.Marshal(map[string]any{"id": campaign, "organization_id": org})
entries[dataPath("campaigns")] = append(c, '\n')
if edit != nil {
edit(m, rows, entries)
}
var data bytes.Buffer
for _, row := range rows {
if err := json.NewEncoder(&data).Encode(row); err != nil {
t.Fatal(err)
}
}
entries[dataPath("campaign_attachments")] = data.Bytes()
manifest, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
entries[manifestPath] = manifest
var out bytes.Buffer
w := zip.NewWriter(&out)
for name, body := range entries {
f, err := w.Create(name)
if err != nil {
t.Fatal(err)
}
if _, err := f.Write(body); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return out.Bytes()
}
func TestAttachmentInventoryPagesPreserveArchiveAndReportContent(t *testing.T) {
b := inventoryFixture(t, nil)
before := bytes.Clone(b)
archive := inventoryArchive{bytes.NewReader(b)}
first, err := InventoryAttachments(t.Context(), archive, AttachmentInventoryOptions{Limit: 2})
if err != nil {
t.Fatal(err)
}
h := sha256.Sum256(b)
if first.ArchiveSHA256 != hex.EncodeToString(h[:]) || first.Done || first.NextAfter != 2 || first.TotalRows != 4 || len(first.Results) != 2 {
t.Fatalf("first page: %+v", first)
}
if first.Results[0].Status != "matches_upload_profile" || first.Results[0].CanonicalMIME != "text/plain" || first.Results[0].StoredMIME != "application/x-existing" || first.Results[1].Status != "outside_upload_profile" {
t.Fatalf("content observations: %+v", first.Results)
}
want := sha256.Sum256([]byte("archived original bytes\n"))
for _, item := range first.Results {
if item.ContentSHA256 != hex.EncodeToString(want[:]) {
t.Fatalf("original content fingerprint: %+v", item)
}
}
second, err := InventoryAttachments(t.Context(), archive, AttachmentInventoryOptions{After: first.NextAfter, Limit: 2, SHA256: first.ArchiveSHA256})
if err != nil {
t.Fatal(err)
}
if !second.Done || second.NextAfter != 4 || len(second.Results) != 2 || second.Results[0].Status != "missing_blob" || second.Results[1].Status != "metadata_mismatch" {
t.Fatalf("second page: %+v", second)
}
if !bytes.Equal(before, b) {
t.Fatal("inventory changed archive bytes")
}
}
func TestAttachmentInventoryRefusesUnboundResumeAndInvalidScope(t *testing.T) {
b := inventoryFixture(t, nil)
for _, opts := range []AttachmentInventoryOptions{
{After: 1, Limit: 1}, {Limit: 0}, {Limit: 101}, {After: -1, Limit: 1},
{Limit: 1, SHA256: "invalid"}, {Limit: 1, SHA256: string(bytes.Repeat([]byte("0"), 64))},
} {
if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, opts); err == nil || report != nil {
t.Fatalf("accepted invalid options: %+v", opts)
}
}
for _, tc := range []struct {
name string
edit func(*Manifest, []models.CampaignAttachment, map[string][]byte)
}{
{"foreign campaign", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.OrganizationID = uuid.New() }},
{"foreign attachment", func(_ *Manifest, a []models.CampaignAttachment, _ map[string][]byte) { a[0].CampaignID = uuid.New() }},
{"duplicate row", func(_ *Manifest, a []models.CampaignAttachment, _ map[string][]byte) { a[1].ID = a[0].ID }},
{"truncated", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.Tables[1].Truncated = true }},
{"row count", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.Tables[1].Rows++ }},
{"duplicate blob", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) {
m.Blobs = append(m.Blobs, m.Blobs[0])
}},
} {
t.Run(tc.name, func(t *testing.T) {
b := inventoryFixture(t, tc.edit)
if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil {
t.Fatal("ambiguous archive produced a page")
}
})
}
ctx, cancel := context.WithCancel(t.Context())
cancel()
if report, err := InventoryAttachments(ctx, inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil {
t.Fatal("canceled inventory produced a page")
}
}
func TestAttachmentInventoryMissingAndUnreferencedBlobsRemainUnknown(t *testing.T) {
b := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) {
m.Blobs[0].Key = "attachments/" + uuid.NewString() + "/unreferenced.txt"
})
report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1})
if err != nil {
t.Fatal(err)
}
if report.UnreferencedBlobs != 1 || report.Results[0].Status != "missing_blob" {
t.Fatalf("missing/unreferenced coverage: %+v", report)
}
b = inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) {
m.Blobs[0].Column = "other_column"
})
report, err = InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1})
if err != nil {
t.Fatal(err)
}
if report.Results[0].Status != "metadata_mismatch" || report.Results[0].ContentSHA256 != "" {
t.Fatalf("manifest relation: %+v", report.Results[0])
}
}
func TestAttachmentInventoryUsesCurrentCSVProfileWithoutChangingLegacyBytes(t *testing.T) {
body := []byte(strings.Repeat("field,", 4096) + "last\n")
b := inventoryFixture(t, func(m *Manifest, a []models.CampaignAttachment, entries map[string][]byte) {
a[0].Filename = "existing.csv"
a[0].Size = int64(len(body))
m.Blobs[0].Size = int64(len(body))
entries[m.Blobs[0].Path] = body
})
before := bytes.Clone(b)
report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1})
if err != nil {
t.Fatal(err)
}
if report.Results[0].Status != "outside_upload_profile" || report.Results[0].ContentSHA256 == "" || !bytes.Equal(before, b) {
t.Fatalf("legacy CSV observation: %+v", report.Results[0])
}
}
func TestAttachmentInventoryRejectsDuplicatePaths(t *testing.T) {
b := inventoryFixture(t, nil)
zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
t.Fatal(err)
}
var duplicate bytes.Buffer
w := zip.NewWriter(&duplicate)
for _, f := range zr.File {
if err := w.Copy(f); err != nil {
t.Fatal(err)
}
}
if err := w.Copy(zr.File[0]); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(duplicate.Bytes())}, AttachmentInventoryOptions{Limit: 1}); err == nil || err.Error() != "archive contains duplicate ZIP paths" || report != nil {
t.Fatal("duplicate ZIP paths produced a report")
}
}
type changingInventoryArchive struct {
*bytes.Reader
next *bytes.Reader
}
func (a *changingInventoryArchive) ReadAt(p []byte, off int64) (int, error) {
n, err := a.Reader.ReadAt(p, off)
if off+int64(n) == a.Size() && a.next != nil {
a.Reader, a.next = a.next, nil
}
return n, err
}
func TestAttachmentInventoryRefusesArchiveChangedDuringRead(t *testing.T) {
before := inventoryFixture(t, nil)
// ZIP comments change archive identity without changing any member or size.
first := append(bytes.Clone(before), 'a')
second := append(bytes.Clone(before), 'b')
for _, b := range [][]byte{first, second} {
b[len(before)-2] = 1
if _, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err != nil {
t.Fatalf("immutable fixture must be readable: %v", err)
}
}
a := &changingInventoryArchive{Reader: bytes.NewReader(first), next: bytes.NewReader(second)}
report, err := InventoryAttachments(t.Context(), a, AttachmentInventoryOptions{Limit: 1})
if report != nil || err == nil || err.Error() != "archive changed during inventory; discard this page" {
t.Fatalf("changed archive must not emit a page: report=%v error=%v", report, err)
}
}
func rewriteInventoryMember(t *testing.T, source []byte, target string, transform func(*zip.FileHeader, []byte) []byte) ([]byte, int64) {
t.Helper()
zr, err := zip.NewReader(bytes.NewReader(source), int64(len(source)))
if err != nil {
t.Fatal(err)
}
var out bytes.Buffer
w := zip.NewWriter(&out)
for _, f := range zr.File {
if f.Name != target {
if err := w.Copy(f); err != nil {
t.Fatal(err)
}
continue
}
r, err := f.Open()
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil {
t.Fatal(err)
}
h := f.FileHeader
h.Flags &^= 8
h.Method = zip.Store
h.Extra = nil
h.CompressedSize64 = uint64(len(body))
raw := transform(&h, body)
dst, err := w.CreateRaw(&h)
if err != nil {
t.Fatal(err)
}
if _, err := dst.Write(raw); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
b := out.Bytes()
zr, err = zip.NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
t.Fatal(err)
}
for _, f := range zr.File {
if f.Name == target {
offset, err := f.DataOffset()
if err != nil {
t.Fatal(err)
}
return b, offset
}
}
t.Fatal("member not found")
return nil, 0
}
func TestAttachmentInventoryVerifiesEveryReadMemberCRC(t *testing.T) {
var blob string
source := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { blob = m.Blobs[0].Path })
for _, target := range []string{blob, manifestPath, dataPath("campaigns"), dataPath("campaign_attachments")} {
for _, checksum := range []string{"valid", "zero", "nonzero"} {
t.Run(target+"/"+checksum, func(t *testing.T) {
b, _ := rewriteInventoryMember(t, source, target, func(h *zip.FileHeader, body []byte) []byte {
actual := crc32.ChecksumIEEE(body)
if actual == 0 || actual == 1 {
t.Fatal("fixture requires a distinct nonzero checksum")
}
switch checksum {
case "valid":
h.CRC32 = actual
case "zero":
h.CRC32 = 0
case "nonzero":
h.CRC32 = actual ^ 1
}
return body
})
report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1})
if checksum != "valid" && target != blob {
if err == nil || report != nil {
t.Fatal("metadata integrity must be established before emitting a page")
}
return
}
if err != nil {
t.Fatal(err)
}
item := report.Results[0]
if checksum == "valid" {
if item.Status != "matches_upload_profile" || item.ContentSHA256 == "" {
t.Fatalf("valid member observation: %+v", item)
}
} else if item.Status != "unreadable_blob" || item.ContentSHA256 != "" || item.CanonicalMIME != "" {
t.Fatalf("member integrity requires unreadable_blob without content observations, got %+v", item)
}
})
}
}
}
func TestAttachmentInventoryAcceptsEmptyMetadataWithZeroCRC(t *testing.T) {
m := Manifest{Kind: ArchiveKind, FormatVersion: 1, OrganizationID: uuid.New(),
Tables: []ManifestTable{{Name: "campaigns"}, {Name: "campaign_attachments"}}}
body, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
var b bytes.Buffer
w := zip.NewWriter(&b)
for name, content := range map[string][]byte{manifestPath: body, dataPath("campaigns"): nil, dataPath("campaign_attachments"): nil} {
h := &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE(content),
CompressedSize64: uint64(len(content)), UncompressedSize64: uint64(len(content))}
member, err := w.CreateRaw(h)
if err != nil {
t.Fatal(err)
}
if _, err := member.Write(content); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b.Bytes())}, AttachmentInventoryOptions{Limit: 1})
if err != nil || report == nil || !report.Done || report.TotalRows != 0 {
t.Fatalf("empty metadata with valid zero CRC: report=%v error=%v", report, err)
}
}
type inventoryCancelArchive struct {
*bytes.Reader
target int64
requestSize int
cancel context.CancelFunc
started bool
readsAfterCancel int
}
func (a *inventoryCancelArchive) ReadAt(p []byte, off int64) (int, error) {
if !a.started && off == a.target && (a.requestSize == 0 || len(p) == a.requestSize) {
a.started = true
a.cancel()
} else if a.started {
a.readsAfterCancel++
}
return a.Reader.ReadAt(p, off)
}
func TestAttachmentInventoryCancelsArchiveSourceReads(t *testing.T) {
var blob string
source := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { blob = m.Blobs[0].Path })
b, offset := rewriteInventoryMember(t, source, blob, func(h *zip.FileHeader, body []byte) []byte {
// Empty DEFLATE blocks exercise source reads without decompressed output.
raw := bytes.Repeat([]byte{0, 0, 0, 255, 255}, 200000)
n := uint16(len(body))
raw = append(raw, 1, byte(n), byte(n>>8), byte(^n), byte(^n>>8))
raw = append(raw, body...)
h.Method = zip.Deflate
h.CRC32 = crc32.ChecksumIEEE(body)
h.CompressedSize64 = uint64(len(raw))
return raw
})
if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err != nil || report.Results[0].Status != "matches_upload_profile" {
t.Fatalf("uncanceled fixture must be readable: report=%v error=%v", report, err)
}
for _, tc := range []struct {
name string
offset int64
size int
}{
{"index", int64(len(b) - 1024), 1024},
{"decompress", offset, 4096},
} {
t.Run(tc.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
a := &inventoryCancelArchive{Reader: bytes.NewReader(b), target: tc.offset, requestSize: tc.size, cancel: cancel}
report, err := InventoryAttachments(ctx, a, AttachmentInventoryOptions{Limit: 1})
if !a.started || report != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation: triggered=%v report=%v error=%v", a.started, report, err)
}
if a.readsAfterCancel != 0 {
t.Fatalf("archive must stop source reads after cancellation, got %d further reads", a.readsAfterCancel)
}
})
}
}
func TestAttachmentInventoryPreservesManifestInterruption(t *testing.T) {
source := inventoryFixture(t, nil)
b, offset := rewriteInventoryMember(t, source, manifestPath, func(_ *zip.FileHeader, body []byte) []byte { return body })
for _, stage := range []string{"header", "body"} {
for _, cause := range []string{"canceled", "deadline"} {
t.Run(stage+"/"+cause, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
interrupt := cancel
want := context.Canceled
if cause == "deadline" {
cancel()
ctx, cancel = context.WithTimeout(t.Context(), time.Second)
interrupt = func() { <-ctx.Done() }
want = context.DeadlineExceeded
}
defer cancel()
target, request := offset, 0
if stage == "header" {
target -= int64(30 + len(manifestPath))
request = 30
}
a := &inventoryCancelArchive{Reader: bytes.NewReader(b), target: target, requestSize: request, cancel: interrupt}
report, err := InventoryAttachments(ctx, a, AttachmentInventoryOptions{Limit: 1})
if !a.started || report != nil || !errors.Is(err, want) {
t.Fatalf("manifest interruption: triggered=%v report=%v error=%v, want %v", a.started, report, err, want)
}
if a.readsAfterCancel != 0 {
t.Fatalf("archive read after manifest interruption: %d", a.readsAfterCancel)
}
})
}
}
}