From 4188a90b45e2f6a0a454dbaf88f6f5e0b6f15507 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 09:29:01 +0100 Subject: [PATCH 1/5] feat: add offline private attachment preflight to warmblyctl with archive-bound resumable pages and unchanged portability --- cmd/warmblyctl/attachment_inventory.go | 55 ++++ cmd/warmblyctl/attachment_inventory_test.go | 76 +++++ cmd/warmblyctl/main.go | 3 +- cmd/warmblyctl/org.go | 4 +- docs/content/docs/development/warmblyctl.mdx | 33 ++ .../docs/guides/workspace-export-import.mdx | 2 + .../app/orgtransfer/attachment_inventory.go | 291 ++++++++++++++++++ .../orgtransfer/attachment_inventory_test.go | 234 ++++++++++++++ 8 files changed, 696 insertions(+), 2 deletions(-) create mode 100644 cmd/warmblyctl/attachment_inventory.go create mode 100644 cmd/warmblyctl/attachment_inventory_test.go create mode 100644 internal/app/orgtransfer/attachment_inventory.go create mode 100644 internal/app/orgtransfer/attachment_inventory_test.go diff --git a/cmd/warmblyctl/attachment_inventory.go b/cmd/warmblyctl/attachment_inventory.go new file mode 100644 index 000000000..243f4721f --- /dev/null +++ b/cmd/warmblyctl/attachment_inventory.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + + "github.com/warmbly/warmbly/internal/app/orgtransfer" +) + +func runOrgAttachmentsPreflight(ctx context.Context, args []string) error { + fs := newFlagSet("org attachments-preflight") + file := fs.String("file", "", "immutable local workspace archive to read (required; no database or network)") + after := fs.Int("after", 0, "skip this many attachment rows, using next_after from the previous page") + limit := fs.Int("limit", 25, "maximum attachment rows to inspect (1 to 100)") + digest := fs.String("expect-sha256", "", "archive_sha256 from the previous page (required with --after)") + if err := fs.Parse(args); err != nil { + return err + } + if err := noExtraArgs(fs); err != nil { + return err + } + if *file == "" { + return errors.New("--file is required") + } + f, err := os.Open(*file) + if err != nil { + return err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("--file must name a regular local archive") + } + report, err := orgtransfer.InventoryAttachments(ctx, &fileArchive{f: f, size: info.Size()}, orgtransfer.AttachmentInventoryOptions{ + After: *after, Limit: *limit, SHA256: *digest, + }) + if err != nil { + return err + } + finalInfo, err := f.Stat() + if err != nil { + return err + } + if finalInfo.Size() != info.Size() || !finalInfo.ModTime().Equal(info.ModTime()) { + return errors.New("archive changed during inventory; discard this page") + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(report) +} diff --git a/cmd/warmblyctl/attachment_inventory_test.go b/cmd/warmblyctl/attachment_inventory_test.go new file mode 100644 index 000000000..9457c14ed --- /dev/null +++ b/cmd/warmblyctl/attachment_inventory_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "archive/zip" + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/app/orgtransfer" +) + +func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) { + t.Setenv("PRIMARY_DB", "invalid-unused-database") + t.Setenv("REDIS", "invalid-unused-cache") + t.Setenv("KMS_PROVIDER", "invalid-unused-provider") + path := filepath.Join(t.TempDir(), "workspace.warmbly.zip") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + w := zip.NewWriter(f) + for _, name := range []string{"data/campaigns.ndjson", "data/campaign_attachments.ndjson"} { + if _, err := w.Create(name); err != nil { + t.Fatal(err) + } + } + manifest, err := w.Create("manifest.json") + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(manifest).Encode(orgtransfer.Manifest{ + Kind: orgtransfer.ArchiveKind, FormatVersion: 1, OrganizationID: uuid.New(), + Tables: []orgtransfer.ManifestTable{{Name: "campaigns"}, {Name: "campaign_attachments"}}, + }); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + out, err := os.CreateTemp(t.TempDir(), "report") + if err != nil { + t.Fatal(err) + } + defer out.Close() + stdout := os.Stdout + os.Stdout = out + defer func() { os.Stdout = stdout }() + if err := dispatch(t.Context(), []string{"org", "attachments-preflight", "--file", path, "--limit", "1"}); err != nil { + t.Fatal(err) + } + if _, err := out.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + var report orgtransfer.AttachmentInventory + if err := json.NewDecoder(out).Decode(&report); err != nil { + t.Fatal(err) + } + if !report.Done || report.TotalRows != 0 || report.Results == nil || len(report.ArchiveSHA256) != 64 { + t.Fatalf("offline command report: %+v", report) + } + after, err := os.ReadFile(path) + if err != nil || string(after) != string(before) { + t.Fatal("command changed the archive") + } +} diff --git a/cmd/warmblyctl/main.go b/cmd/warmblyctl/main.go index 671a68300..62dedc69b 100644 --- a/cmd/warmblyctl/main.go +++ b/cmd/warmblyctl/main.go @@ -131,6 +131,7 @@ var commands = []command{ {"org list", "List the workspaces on this instance with their id, owner, and size", composeExec + "org list"}, {"org export", "Write a whole workspace to a portable archive file", composeExec + "org export --org you@example.com --out /tmp/workspace.warmbly.zip"}, {"org import", "Apply an archive to a workspace on this instance", composeExec + "org import --org you@example.com --file /tmp/workspace.warmbly.zip"}, + {"org attachments-preflight", "Read one page of private attachment type results from a local archive, offline", "warmblyctl org attachments-preflight --file ./workspace.warmbly.zip --limit 25"}, {"fleet join-token", "Issue the token a machine needs to join the fleet. Shown once", composeExec + "fleet join-token"}, {"fleet list", "Every worker and consumer: role, version, liveness and usage", composeExec + "fleet list"}, {"fleet version", "Show or set the version every node should be running", composeExec + "fleet version v1.4.2"}, @@ -191,7 +192,7 @@ Anything piped into a container needs exec -T, which is what turns the TTY off. Anything that prompts needs a TTY, so run it without -T. Environment: - PRIMARY_DB Postgres connection string. Every operator command except hash-password needs it. + PRIMARY_DB Postgres connection string. Not needed by hash-password or org attachments-preflight. REDIS Redis URL. setup-link needs it, and reset-password needs it to mint a link. APP_URL Base URL every printed link is built from. WARMBLY_API_KEY API key (wmbly_...) for the API commands. diff --git a/cmd/warmblyctl/org.go b/cmd/warmblyctl/org.go index 6099b6a67..45b32cda8 100644 --- a/cmd/warmblyctl/org.go +++ b/cmd/warmblyctl/org.go @@ -31,6 +31,8 @@ func runOrg(ctx context.Context, args []string) error { return runOrgExport(ctx, args[1:]) case "import": return runOrgImport(ctx, args[1:]) + case "attachments-preflight": + return runOrgAttachmentsPreflight(ctx, args[1:]) } orgUsage(os.Stderr) @@ -38,7 +40,7 @@ func runOrg(ctx context.Context, args []string) error { } func orgUsage(w *os.File) { - fmt.Fprint(w, "Move a workspace between instances.\n\nUsage:\n warmblyctl org [flags]\n\nSubcommands:\n") + fmt.Fprint(w, "Move a workspace between instances or inspect a local archive.\n\nUsage:\n warmblyctl org [flags]\n\nSubcommands:\n") for _, c := range commands { if !strings.HasPrefix(c.name, "org ") { continue diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index 9ade7d878..3de322991 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -609,6 +609,39 @@ The whole import runs in one transaction: if any part fails, nothing lands and t Billing history, plan overrides, worker placement, mailbox sync checkpoints and warmup pool membership are exported for the record but never applied: each belongs to the instance rather than to the workspace. [Export and import](/guides/workspace-export-import/) has the full table. +## org attachments-preflight + +```bash +warmblyctl org attachments-preflight --file ./workspace.warmbly.zip --limit 25 > page-1.json +warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \ + --after 25 --expect-sha256 --limit 25 > page-2.json +``` + +Reads private campaign attachment rows and their bytes from an **existing local workspace archive**, comparing each selected file with the [new-upload profile](/guides/campaigns/#campaign-attachments). This command works offline without `PRIMARY_DB`, a storage client, credentials or a destination workspace. It opens the archive for reading and prints JSON to stdout. Keep the archive immutable and protect reports as workspace data: they contain filenames, identifiers and content fingerprints. Obtaining an archive is a separate, authorized export operation; this command does not create or download one. + +| Flag | Does | +|---|---| +| `--file` | Regular local archive file. Required | +| `--limit` | Rows inspected per page, from 1 to 100. Default 25 | +| `--after` | Rows to skip. Use the previous successful page's `next_after`, starting at zero | +| `--expect-sha256` | Expected archive fingerprint. Required when resuming with `--after` | + +Resume using the same binary, archive and classifier build. Every page hashes the whole compressed archive before and after inspection and rescans campaign and attachment metadata. Only the selected attachment members are decompressed for classification. A failed or interrupted run emits no report; retry the same page. Save each successful report before advancing. `done` means all attachment **rows in this archive** have been visited, not that every file matches the profile. `classifier_build` identifies the inspecting binary; an unstamped development binary needs its source commit recorded separately. `source_version` is informational archive metadata. + +| Per-file `status` | Meaning | +|---|---| +| `matches_upload_profile` | Original bytes pass this binary's upload classifier. `canonical_mime` is its result; `stored_mime` remains unchanged | +| `outside_upload_profile` | The file does not meet the current upload profile, including its size limits | +| `missing_blob` | The row has no matching available private blob in the archive | +| `metadata_mismatch` | Campaign/key scope or stored sizes do not agree | +| `unreadable_blob` | The selected member could not be read with its ZIP integrity checks | + +`content_sha256` fingerprints bytes that were successfully read within the upload size limit. Missing, inconsistent, oversized and empty members do not receive one. `unreferenced_blobs` counts attachment blob references without an attachment row; their content is not classified. Review those separately. Zero exit status means the report completed, including any nonmatching or unknown results. Structural archive errors and invalid options exit nonzero. + +The command accepts archive format versions supported by this binary, including v0.6.75 archives. It requires complete `campaigns` and `campaign_attachments` tables with matching row counts, refuses duplicate paths and attachment identifiers, and checks campaign ownership against the archive organization. Archives without private blob bytes produce `missing_blob` results. Limits are 8 GiB compressed archive size, the existing 64 MiB manifest limit, 64 MiB decompressed per metadata table, 1 MiB per metadata row, and the upload profile's 15 MiB per selected attachment, processed sequentially. ZIP directory indexing and classifier work still consume resources; page size is not a process memory or runtime guarantee. + +This is an inventory observation, not malware certification, import approval or send eligibility. It does not change archives, stored rows, private bytes, MIME metadata, duplication, restore or queued sends. It does not inspect live storage, retained queue messages, orphan objects, other archives or concurrent writes. Any later eligibility transition needs its own compatible migration stage covering old readers, retained messages, portability and rollback. + ## The API commands Everything in this section talks to the public REST API with an API key, so it runs from anywhere: your laptop, a CI job, an agent's sandbox. It needs no database access and works against the hosted service and self-hosted instances alike. diff --git a/docs/content/docs/guides/workspace-export-import.mdx b/docs/content/docs/guides/workspace-export-import.mdx index 999038259..8397aaa05 100644 --- a/docs/content/docs/guides/workspace-export-import.mdx +++ b/docs/content/docs/guides/workspace-export-import.mdx @@ -288,3 +288,5 @@ Reply and forward drafts kept in your browser are also excluded. Finish or copy Campaign attachments travel with their original bytes and metadata, including files accepted by v0.6.75 and formats outside the [new-upload profile](/guides/campaigns/#campaign-attachments). Workspace and admin import use the same preservation policy. Preflight and import report that restoring private campaign files does not certify their content. Those members are retained rather than omitted because of their format. Review them before using imported campaigns. Public images keep their separate image-validation policy. Archive format and row contracts stay compatible with v0.6.75. The new-upload policy adds no required archive field, validation certificate or send hold. An archive filename or source version is not evidence that every private member passed that policy. + +Operators can use [`warmblyctl org attachments-preflight`](/development/warmblyctl/#org-attachments-preflight) to inspect an existing local archive in resumable pages against the current upload profile. The offline report records matches, files outside the profile and unavailable or inconsistent bytes. It preserves the archive and does not change restore or queued-send behavior. Its coverage is limited to that archive's attachment rows; it does not certify live storage or retained queue messages. diff --git a/internal/app/orgtransfer/attachment_inventory.go b/internal/app/orgtransfer/attachment_inventory.go new file mode 100644 index 000000000..c7f4d404e --- /dev/null +++ b/internal/app/orgtransfer/attachment_inventory.go @@ -0,0 +1,291 @@ +package orgtransfer + +import ( + "archive/zip" + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" + "github.com/warmbly/warmbly/internal/pkg/attachmenttype" + "github.com/warmbly/warmbly/internal/version" +) + +const maxInventoryTableBytes = 64 << 20 + +// AttachmentInventoryOptions selects a page in an immutable local archive. +type AttachmentInventoryOptions struct { + After int + Limit int + SHA256 string +} + +// AttachmentInventory reports observations only; it grants no send eligibility. +type AttachmentInventory struct { + ArchiveSHA256 string `json:"archive_sha256"` + ClassifierBuild version.Info `json:"classifier_build"` + OrganizationID uuid.UUID `json:"organization_id"` + SourceVersion string `json:"source_version"` + TotalRows int `json:"total_rows"` + UnreferencedBlobs int `json:"unreferenced_blobs"` + After int `json:"after"` + NextAfter int `json:"next_after"` + Done bool `json:"done"` + Results []AttachmentInventoryItem `json:"results"` +} + +// AttachmentInventoryItem compares one archived row and blob with the upload profile. +type AttachmentInventoryItem struct { + Row int `json:"row"` + ID uuid.UUID `json:"id"` + CampaignID uuid.UUID `json:"campaign_id"` + Filename string `json:"filename"` + StoredMIME string `json:"stored_mime"` + StoredBytes int64 `json:"stored_bytes"` + Status string `json:"status"` + ContentSHA256 string `json:"content_sha256,omitempty"` + CanonicalMIME string `json:"canonical_mime,omitempty"` +} + +// InventoryAttachments reads archive bytes without a database, storage client or importer. +func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts AttachmentInventoryOptions) (*AttachmentInventory, error) { + if opts.After < 0 || opts.Limit < 1 || opts.Limit > 100 { + return nil, errors.New("after must be nonnegative and limit must be between 1 and 100") + } + if opts.After > 0 && opts.SHA256 == "" { + return nil, errors.New("resuming requires the previous report's archive SHA-256") + } + if opts.SHA256 != "" { + b, err := hex.DecodeString(opts.SHA256) + if err != nil || len(b) != sha256.Size { + return nil, errors.New("expected archive SHA-256 must contain 64 hexadecimal characters") + } + } + if archive.Size() <= 0 || archive.Size() > models.MaxOrgArchiveUploadBytes { + return nil, errors.New("archive must be between 1 byte and 8 GiB") + } + digest, err := inventoryDigest(ctx, archive) + if err != nil { + return nil, err + } + if opts.SHA256 != "" && !strings.EqualFold(opts.SHA256, digest) { + return nil, errors.New("archive SHA-256 differs from the requested inventory") + } + zr, err := zip.NewReader(archive, archive.Size()) + if err != nil { + return nil, errors.New("archive is not a readable ZIP") + } + entries := make(map[string]*zip.File, len(zr.File)) + for _, f := range zr.File { + if entries[f.Name] != nil { + return nil, errors.New("archive contains duplicate ZIP paths") + } + entries[f.Name] = f + } + manifestEntry := entries[manifestPath] + if manifestEntry == nil || manifestEntry.UncompressedSize64 > maxManifestBytes { + return nil, errors.New("inventory requires manifest.json within the 64 MiB limit") + } + mr, err := manifestEntry.Open() + if err != nil { + return nil, errors.New("cannot open archive manifest") + } + manifestBody, readErr := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, mr}, maxManifestBytes+1)) + _ = mr.Close() + if readErr != nil || len(manifestBody) > maxManifestBytes { + return nil, errors.New("cannot read archive manifest within the 64 MiB limit") + } + var m Manifest + if err := json.Unmarshal(manifestBody, &m); err != nil { + return nil, errors.New("archive manifest is not readable JSON") + } + if err := m.Validate(); err != nil { + return nil, err + } + campaigns := map[uuid.UUID]bool{} + err = inventoryRows(ctx, entries, &m, "campaigns", func(b []byte) error { + var c struct { + ID uuid.UUID `json:"id"` + OrganizationID uuid.UUID `json:"organization_id"` + } + if err := json.Unmarshal(b, &c); err != nil || c.ID == uuid.Nil || c.OrganizationID != m.OrganizationID || campaigns[c.ID] { + return errors.New("campaign rows must uniquely belong to the archive organization") + } + campaigns[c.ID] = true + return nil + }) + if err != nil { + return nil, err + } + blobs := map[string]ManifestBlob{} + for _, b := range m.Blobs { + if b.Table != "campaign_attachments" && !strings.HasPrefix(b.Key, "attachments/") { + continue + } + if _, exists := blobs[b.Key]; exists { + return nil, errors.New("archive contains duplicate attachment blob keys") + } + blobs[b.Key] = b + } + report := &AttachmentInventory{ArchiveSHA256: digest, ClassifierBuild: version.Current(), OrganizationID: m.OrganizationID, + SourceVersion: m.SourceAppVersion, After: opts.After, NextAfter: opts.After, Results: []AttachmentInventoryItem{}} + ids := map[uuid.UUID]bool{} + keys := map[string]bool{} + scope := blobKeyScope{orgID: m.OrganizationID, campaigns: campaigns} + err = inventoryRows(ctx, entries, &m, "campaign_attachments", func(b []byte) error { + var a models.CampaignAttachment + if err := json.Unmarshal(b, &a); err != nil || a.ID == uuid.Nil || ids[a.ID] || !campaigns[a.CampaignID] { + return errors.New("attachment rows must uniquely reference an archived campaign") + } + ids[a.ID] = true + keys[a.S3Key] = true + report.TotalRows++ + if report.TotalRows <= opts.After || len(report.Results) == opts.Limit { + return nil + } + item := AttachmentInventoryItem{Row: report.TotalRows, ID: a.ID, CampaignID: a.CampaignID, + Filename: a.Filename, StoredMIME: a.MimeType, StoredBytes: a.Size, Status: "metadata_mismatch"} + ref, exists := blobs[a.S3Key] + switch { + case !scope.allows(a.S3Key) || !strings.HasPrefix(a.S3Key, "attachments/"+a.CampaignID.String()+"/"): + item.Status = "metadata_mismatch" + case !exists || entries[ref.Path] == nil: + item.Status = "missing_blob" + case ref.Table != "campaign_attachments" || ref.Column != "s3_key" || !strings.HasPrefix(ref.Path, blobPrefix) || ref.Size != a.Size || a.Size < 0 || uint64(a.Size) != entries[ref.Path].UncompressedSize64: + item.Status = "metadata_mismatch" + case a.Size == 0 || a.Size > attachmenttype.MaxBytes: + item.Status = "outside_upload_profile" + default: + item = inventoryContent(ctx, entries[ref.Path], item) + } + report.Results = append(report.Results, item) + report.NextAfter = report.TotalRows + return nil + }) + if err != nil { + return nil, err + } + if opts.After > report.TotalRows { + return nil, errors.New("after is beyond the attachment row count") + } + for key := range blobs { + if !keys[key] { + report.UnreferencedBlobs++ + } + } + report.Done = report.NextAfter == report.TotalRows + finalDigest, err := inventoryDigest(ctx, archive) + if err != nil { + return nil, err + } + if finalDigest != digest { + return nil, errors.New("archive changed during inventory; discard this page") + } + return report, ctx.Err() +} + +func inventoryContent(ctx context.Context, entry *zip.File, item AttachmentInventoryItem) AttachmentInventoryItem { + item.Status = "unreadable_blob" + r, err := entry.Open() + if err != nil { + return item + } + defer r.Close() + body, err := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, r}, attachmenttype.MaxBytes+1)) + if err != nil { + return item + } + if int64(len(body)) != item.StoredBytes { + item.Status = "metadata_mismatch" + return item + } + digest := sha256.Sum256(body) + item.ContentSHA256 = hex.EncodeToString(digest[:]) + t, err := attachmenttype.Validate(item.Filename, body) + item.Status = "outside_upload_profile" + if err == nil { + item.Status = "matches_upload_profile" + item.CanonicalMIME = t.MIME + } + return item +} + +func inventoryRows(ctx context.Context, entries map[string]*zip.File, m *Manifest, table string, visit func([]byte) error) error { + var mt *ManifestTable + for i := range m.Tables { + if m.Tables[i].Name == table { + if mt != nil { + return fmt.Errorf("duplicate %s table in manifest", table) + } + mt = &m.Tables[i] + } + } + if mt == nil || mt.Truncated || mt.Rows < 0 { + return fmt.Errorf("inventory requires a complete %s table in the archive", table) + } + entry := entries[dataPath(table)] + if entry == nil { + return fmt.Errorf("archive has no %s data", table) + } + if entry.UncompressedSize64 > maxInventoryTableBytes { + return fmt.Errorf("%s metadata exceeds the 64 MiB inventory limit", table) + } + r, err := entry.Open() + if err != nil { + return fmt.Errorf("cannot open %s metadata", table) + } + defer r.Close() + limited := &io.LimitedReader{R: inventoryContextReader{ctx, r}, N: maxInventoryTableBytes + 1} + scanner := bufio.NewScanner(limited) + scanner.Buffer(make([]byte, 64<<10), 1<<20) + var rows int64 + for scanner.Scan() { + if len(strings.TrimSpace(scanner.Text())) == 0 { + continue + } + if err := visit(scanner.Bytes()); err != nil { + return err + } + rows++ + } + if err := scanner.Err(); err != nil { + return fmt.Errorf("cannot read %s metadata (1 MiB row limit): %w", table, err) + } + if limited.N == 0 || rows != mt.Rows { + return fmt.Errorf("%s metadata size or row count differs from the manifest", table) + } + return nil +} + +func inventoryDigest(ctx context.Context, archive ReaderAtSizer) (string, error) { + h := sha256.New() + size := archive.Size() + n, err := io.Copy(h, inventoryContextReader{ctx, io.NewSectionReader(archive, 0, size)}) + if err != nil { + return "", err + } + if n != size { + return "", errors.New("archive size changed during fingerprinting") + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +type inventoryContextReader struct { + ctx context.Context + r io.Reader +} + +func (r inventoryContextReader) Read(b []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(b) +} diff --git a/internal/app/orgtransfer/attachment_inventory_test.go b/internal/app/orgtransfer/attachment_inventory_test.go new file mode 100644 index 000000000..de9bd51f5 --- /dev/null +++ b/internal/app/orgtransfer/attachment_inventory_test.go @@ -0,0 +1,234 @@ +package orgtransfer + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/warmbly/warmbly/internal/models" +) + +type inventoryArchive struct{ *bytes.Reader } + +func (a inventoryArchive) Size() int64 { return a.Reader.Size() } + +func inventoryFixture(t *testing.T, edit func(*Manifest, []models.CampaignAttachment, map[string][]byte)) []byte { + t.Helper() + org, campaign := uuid.New(), uuid.New() + m := &Manifest{Kind: ArchiveKind, FormatVersion: 1, OrganizationID: org, SourceAppVersion: "v0.6.75", + Tables: []ManifestTable{{Name: "campaigns", Rows: 1}, {Name: "campaign_attachments", Rows: 4}}} + entries := map[string][]byte{} + rows := make([]models.CampaignAttachment, 4) + for i, name := range []string{"notes.txt", "legacy.unknown", "missing.txt", "mismatch.txt"} { + body := []byte("archived original bytes\n") + key := models.AttachmentObjectKey(campaign, name) + rows[i] = models.CampaignAttachment{ID: uuid.New(), CampaignID: campaign, Filename: name, Size: int64(len(body)), MimeType: "application/x-existing", S3Key: key} + ref := ManifestBlob{Key: key, Path: blobArchivePath(i, key), Size: int64(len(body)), Table: "campaign_attachments", Column: "s3_key"} + m.Blobs = append(m.Blobs, ref) + if i != 2 { + entries[ref.Path] = body + } + } + rows[3].Size++ + c, _ := json.Marshal(map[string]any{"id": campaign, "organization_id": org}) + entries[dataPath("campaigns")] = append(c, '\n') + if edit != nil { + edit(m, rows, entries) + } + var data bytes.Buffer + for _, row := range rows { + if err := json.NewEncoder(&data).Encode(row); err != nil { + t.Fatal(err) + } + } + entries[dataPath("campaign_attachments")] = data.Bytes() + manifest, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + entries[manifestPath] = manifest + var out bytes.Buffer + w := zip.NewWriter(&out) + for name, body := range entries { + f, err := w.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := f.Write(body); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func TestAttachmentInventoryPagesPreserveArchiveAndReportContent(t *testing.T) { + b := inventoryFixture(t, nil) + before := bytes.Clone(b) + archive := inventoryArchive{bytes.NewReader(b)} + first, err := InventoryAttachments(t.Context(), archive, AttachmentInventoryOptions{Limit: 2}) + if err != nil { + t.Fatal(err) + } + h := sha256.Sum256(b) + if first.ArchiveSHA256 != hex.EncodeToString(h[:]) || first.Done || first.NextAfter != 2 || first.TotalRows != 4 || len(first.Results) != 2 { + t.Fatalf("first page: %+v", first) + } + if first.Results[0].Status != "matches_upload_profile" || first.Results[0].CanonicalMIME != "text/plain" || first.Results[0].StoredMIME != "application/x-existing" || first.Results[1].Status != "outside_upload_profile" { + t.Fatalf("content observations: %+v", first.Results) + } + want := sha256.Sum256([]byte("archived original bytes\n")) + for _, item := range first.Results { + if item.ContentSHA256 != hex.EncodeToString(want[:]) { + t.Fatalf("original content fingerprint: %+v", item) + } + } + second, err := InventoryAttachments(t.Context(), archive, AttachmentInventoryOptions{After: first.NextAfter, Limit: 2, SHA256: first.ArchiveSHA256}) + if err != nil { + t.Fatal(err) + } + if !second.Done || second.NextAfter != 4 || len(second.Results) != 2 || second.Results[0].Status != "missing_blob" || second.Results[1].Status != "metadata_mismatch" { + t.Fatalf("second page: %+v", second) + } + if !bytes.Equal(before, b) { + t.Fatal("inventory changed archive bytes") + } +} + +func TestAttachmentInventoryRefusesUnboundResumeAndInvalidScope(t *testing.T) { + b := inventoryFixture(t, nil) + for _, opts := range []AttachmentInventoryOptions{ + {After: 1, Limit: 1}, {Limit: 0}, {Limit: 101}, {After: -1, Limit: 1}, + {Limit: 1, SHA256: "invalid"}, {Limit: 1, SHA256: string(bytes.Repeat([]byte("0"), 64))}, + } { + if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, opts); err == nil || report != nil { + t.Fatalf("accepted invalid options: %+v", opts) + } + } + for _, tc := range []struct { + name string + edit func(*Manifest, []models.CampaignAttachment, map[string][]byte) + }{ + {"foreign campaign", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.OrganizationID = uuid.New() }}, + {"foreign attachment", func(_ *Manifest, a []models.CampaignAttachment, _ map[string][]byte) { a[0].CampaignID = uuid.New() }}, + {"duplicate row", func(_ *Manifest, a []models.CampaignAttachment, _ map[string][]byte) { a[1].ID = a[0].ID }}, + {"truncated", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.Tables[1].Truncated = true }}, + {"row count", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { m.Tables[1].Rows++ }}, + {"duplicate blob", func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { + m.Blobs = append(m.Blobs, m.Blobs[0]) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + b := inventoryFixture(t, tc.edit) + if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil { + t.Fatal("ambiguous archive produced a page") + } + }) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if report, err := InventoryAttachments(ctx, inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil { + t.Fatal("canceled inventory produced a page") + } +} + +func TestAttachmentInventoryMissingAndUnreferencedBlobsRemainUnknown(t *testing.T) { + b := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { + m.Blobs[0].Key = "attachments/" + uuid.NewString() + "/unreferenced.txt" + }) + report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}) + if err != nil { + t.Fatal(err) + } + if report.UnreferencedBlobs != 1 || report.Results[0].Status != "missing_blob" { + t.Fatalf("missing/unreferenced coverage: %+v", report) + } + b = inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { + m.Blobs[0].Column = "other_column" + }) + report, err = InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}) + if err != nil { + t.Fatal(err) + } + if report.Results[0].Status != "metadata_mismatch" || report.Results[0].ContentSHA256 != "" { + t.Fatalf("manifest relation: %+v", report.Results[0]) + } +} + +func TestAttachmentInventoryUsesCurrentCSVProfileWithoutChangingLegacyBytes(t *testing.T) { + body := []byte(strings.Repeat("field,", 4096) + "last\n") + b := inventoryFixture(t, func(m *Manifest, a []models.CampaignAttachment, entries map[string][]byte) { + a[0].Filename = "existing.csv" + a[0].Size = int64(len(body)) + m.Blobs[0].Size = int64(len(body)) + entries[m.Blobs[0].Path] = body + }) + before := bytes.Clone(b) + report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}) + if err != nil { + t.Fatal(err) + } + if report.Results[0].Status != "outside_upload_profile" || report.Results[0].ContentSHA256 == "" || !bytes.Equal(before, b) { + t.Fatalf("legacy CSV observation: %+v", report.Results[0]) + } +} + +func TestAttachmentInventoryReportsMemberIntegrityAndRejectsDuplicatePaths(t *testing.T) { + b := inventoryFixture(t, nil) + zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + t.Fatal(err) + } + var m Manifest + r, err := indexEntries(zr)[manifestPath].Open() + if err != nil { + t.Fatal(err) + } + if err := json.NewDecoder(r).Decode(&m); err != nil { + t.Fatal(err) + } + r.Close() + // Alter only the selected member's recorded CRC, leaving metadata readable. + var corrupt bytes.Buffer + w := zip.NewWriter(&corrupt) + for _, f := range zr.File { + if f.Name == m.Blobs[0].Path { + f.CRC32++ + } + if err := w.Copy(f); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(corrupt.Bytes())}, AttachmentInventoryOptions{Limit: 1}) + if err != nil { + t.Fatal(err) + } + if report.Results[0].Status != "unreadable_blob" || report.Results[0].ContentSHA256 != "" { + t.Fatalf("integrity result: %+v", report.Results[0]) + } + var duplicate bytes.Buffer + w = zip.NewWriter(&duplicate) + for range 2 { + if err := w.Copy(zr.File[0]); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(duplicate.Bytes())}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil { + t.Fatal("duplicate ZIP paths produced a report") + } +} From 419b3ca231b12341dc9f6f7a75c834aab9f92dbb Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 09:51:20 +0100 Subject: [PATCH 2/5] feat: verify attachment inventory member checksums and cancellation while admitting only regular archive files --- cmd/warmblyctl/attachment_inventory.go | 12 +- cmd/warmblyctl/attachment_inventory_other.go | 9 + cmd/warmblyctl/attachment_inventory_posix.go | 13 + .../attachment_inventory_posix_test.go | 39 +++ cmd/warmblyctl/attachment_inventory_test.go | 50 +++- docs/content/docs/development/warmblyctl.mdx | 4 +- .../app/orgtransfer/attachment_inventory.go | 28 +- .../orgtransfer/attachment_inventory_test.go | 250 ++++++++++++++++-- 8 files changed, 366 insertions(+), 39 deletions(-) create mode 100644 cmd/warmblyctl/attachment_inventory_other.go create mode 100644 cmd/warmblyctl/attachment_inventory_posix.go create mode 100644 cmd/warmblyctl/attachment_inventory_posix_test.go diff --git a/cmd/warmblyctl/attachment_inventory.go b/cmd/warmblyctl/attachment_inventory.go index 243f4721f..cefe4ff96 100644 --- a/cmd/warmblyctl/attachment_inventory.go +++ b/cmd/warmblyctl/attachment_inventory.go @@ -24,7 +24,14 @@ func runOrgAttachmentsPreflight(ctx context.Context, args []string) error { if *file == "" { return errors.New("--file is required") } - f, err := os.Open(*file) + pathInfo, err := os.Stat(*file) + if err != nil { + return err + } + if !pathInfo.Mode().IsRegular() { + return errors.New("--file must name a regular local archive") + } + f, err := openInventoryFile(*file) if err != nil { return err } @@ -36,6 +43,9 @@ func runOrgAttachmentsPreflight(ctx context.Context, args []string) error { if !info.Mode().IsRegular() { return errors.New("--file must name a regular local archive") } + if !os.SameFile(pathInfo, info) { + return errors.New("archive changed before inventory; retry with an immutable file") + } report, err := orgtransfer.InventoryAttachments(ctx, &fileArchive{f: f, size: info.Size()}, orgtransfer.AttachmentInventoryOptions{ After: *after, Limit: *limit, SHA256: *digest, }) diff --git a/cmd/warmblyctl/attachment_inventory_other.go b/cmd/warmblyctl/attachment_inventory_other.go new file mode 100644 index 000000000..2e6f0e7f5 --- /dev/null +++ b/cmd/warmblyctl/attachment_inventory_other.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin && !freebsd && !openbsd && !netbsd && !dragonfly + +package main + +import "os" + +func openInventoryFile(path string) (*os.File, error) { + return os.Open(path) +} diff --git a/cmd/warmblyctl/attachment_inventory_posix.go b/cmd/warmblyctl/attachment_inventory_posix.go new file mode 100644 index 000000000..dd8b21236 --- /dev/null +++ b/cmd/warmblyctl/attachment_inventory_posix.go @@ -0,0 +1,13 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly + +package main + +import ( + "os" + "syscall" +) + +func openInventoryFile(path string) (*os.File, error) { + // Keep descriptor verification reachable if the path changes to a FIFO. + return os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) +} diff --git a/cmd/warmblyctl/attachment_inventory_posix_test.go b/cmd/warmblyctl/attachment_inventory_posix_test.go new file mode 100644 index 000000000..97e2dcee7 --- /dev/null +++ b/cmd/warmblyctl/attachment_inventory_posix_test.go @@ -0,0 +1,39 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly + +package main + +import ( + "os" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestOrgAttachmentsPreflightRefusesFIFOWithoutWriter(t *testing.T) { + name := filepath.Join(t.TempDir(), "input.fifo") + if err := syscall.Mkfifo(name, 0600); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- dispatch(t.Context(), []string{"org", "attachments-preflight", "--file", name}) }() + select { + case err := <-done: + if err == nil || err.Error() != "--file must name a regular local archive" { + t.Fatalf("regular-file refusal: %v", err) + } + case <-time.After(time.Second): + // Release and join the dispatch before reporting a failed bounded wait. + f, err := os.OpenFile(name, os.O_RDWR|syscall.O_NONBLOCK, 0600) + if err != nil { + t.Fatal(err) + } + defer f.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("dispatch did not return after opening the FIFO writer") + } + t.Fatal("regular-file refusal must complete without a FIFO writer") + } +} diff --git a/cmd/warmblyctl/attachment_inventory_test.go b/cmd/warmblyctl/attachment_inventory_test.go index 9457c14ed..c42b4f76e 100644 --- a/cmd/warmblyctl/attachment_inventory_test.go +++ b/cmd/warmblyctl/attachment_inventory_test.go @@ -2,7 +2,9 @@ package main import ( "archive/zip" + "bytes" "encoding/json" + "fmt" "io" "os" "path/filepath" @@ -11,6 +13,7 @@ import ( "github.com/google/uuid" "github.com/warmbly/warmbly/internal/app/orgtransfer" + "github.com/warmbly/warmbly/internal/models" ) func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) { @@ -23,8 +26,41 @@ func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) { t.Fatal(err) } w := zip.NewWriter(f) - for _, name := range []string{"data/campaigns.ndjson", "data/campaign_attachments.ndjson"} { - if _, err := w.Create(name); err != nil { + org, campaign := uuid.New(), uuid.New() + m := orgtransfer.Manifest{ + Kind: orgtransfer.ArchiveKind, FormatVersion: 1, OrganizationID: org, + Tables: []orgtransfer.ManifestTable{{Name: "campaigns", Rows: 1}, {Name: "campaign_attachments", Rows: 2}}, + } + var rows bytes.Buffer + for i := range 2 { + name := fmt.Sprintf("notes-%d.txt", i) + body := []byte("archived original bytes\n") + key := models.AttachmentObjectKey(campaign, name) + blobPath := fmt.Sprintf("blobs/%d", i) + m.Blobs = append(m.Blobs, orgtransfer.ManifestBlob{Key: key, Path: blobPath, Size: int64(len(body)), Table: "campaign_attachments", Column: "s3_key"}) + member, err := w.Create(blobPath) + if err != nil { + t.Fatal(err) + } + if _, err := member.Write(body); err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(&rows).Encode(models.CampaignAttachment{ + ID: uuid.New(), CampaignID: campaign, Filename: name, S3Key: key, Size: int64(len(body)), MimeType: "text/plain", + }); err != nil { + t.Fatal(err) + } + } + campaignRow, err := json.Marshal(map[string]any{"id": campaign, "organization_id": org}) + if err != nil { + t.Fatal(err) + } + for name, body := range map[string][]byte{"data/campaigns.ndjson": campaignRow, "data/campaign_attachments.ndjson": rows.Bytes()} { + member, err := w.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := member.Write(body); err != nil { t.Fatal(err) } } @@ -32,10 +68,7 @@ func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) { if err != nil { t.Fatal(err) } - if err := json.NewEncoder(manifest).Encode(orgtransfer.Manifest{ - Kind: orgtransfer.ArchiveKind, FormatVersion: 1, OrganizationID: uuid.New(), - Tables: []orgtransfer.ManifestTable{{Name: "campaigns"}, {Name: "campaign_attachments"}}, - }); err != nil { + if err := json.NewEncoder(manifest).Encode(m); err != nil { t.Fatal(err) } if err := w.Close(); err != nil { @@ -66,9 +99,12 @@ func TestOrgAttachmentsPreflightRunsOfflineThroughDispatch(t *testing.T) { if err := json.NewDecoder(out).Decode(&report); err != nil { t.Fatal(err) } - if !report.Done || report.TotalRows != 0 || report.Results == nil || len(report.ArchiveSHA256) != 64 { + if report.Done || report.TotalRows != 2 || len(report.Results) != 1 || report.NextAfter != 1 || len(report.ArchiveSHA256) != 64 { t.Fatalf("offline command report: %+v", report) } + if report.Results[0].Status != "matches_upload_profile" { + t.Fatalf("valid attachment result: %+v", report.Results[0]) + } after, err := os.ReadFile(path) if err != nil || string(after) != string(before) { t.Fatal("command changed the archive") diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index 3de322991..62571a79b 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -638,7 +638,9 @@ Resume using the same binary, archive and classifier build. Every page hashes th `content_sha256` fingerprints bytes that were successfully read within the upload size limit. Missing, inconsistent, oversized and empty members do not receive one. `unreferenced_blobs` counts attachment blob references without an attachment row; their content is not classified. Review those separately. Zero exit status means the report completed, including any nonmatching or unknown results. Structural archive errors and invalid options exit nonzero. -The command accepts archive format versions supported by this binary, including v0.6.75 archives. It requires complete `campaigns` and `campaign_attachments` tables with matching row counts, refuses duplicate paths and attachment identifiers, and checks campaign ownership against the archive organization. Archives without private blob bytes produce `missing_blob` results. Limits are 8 GiB compressed archive size, the existing 64 MiB manifest limit, 64 MiB decompressed per metadata table, 1 MiB per metadata row, and the upload profile's 15 MiB per selected attachment, processed sequentially. ZIP directory indexing and classifier work still consume resources; page size is not a process memory or runtime guarantee. +The command accepts archive format versions supported by this binary, including v0.6.75 archives. It requires complete `campaigns` and `campaign_attachments` tables with matching row counts, refuses duplicate paths and attachment identifiers, and checks campaign ownership against the archive organization. Every member read has its CRC32 checked, including a recorded zero checksum. Archives without private blob bytes produce `missing_blob` results. Limits are 8 GiB compressed archive size, the existing 64 MiB manifest limit, 64 MiB decompressed per metadata table, 1 MiB per metadata row, and the upload profile's 15 MiB per selected attachment, processed sequentially. Cancellation is checked before archive source reads, including ZIP indexing and compressed member reads. Classifier calls remain synchronous. ZIP directory indexing and classifier work still consume resources; page size is not a process memory or runtime guarantee. + +The input path must resolve to a regular file before opening, and the opened descriptor must identify the same regular file. Linux, macOS and supported BSD builds use a nonblocking open so descriptor verification also covers a path changed to a FIFO. These checks and the archive fingerprints detect ordinary file changes; they do not create a filesystem snapshot or interrupt a source read already in progress. This is an inventory observation, not malware certification, import approval or send eligibility. It does not change archives, stored rows, private bytes, MIME metadata, duplication, restore or queued sends. It does not inspect live storage, retained queue messages, orphan objects, other archives or concurrent writes. Any later eligibility transition needs its own compatible migration stage covering old readers, retained messages, portability and rollback. diff --git a/internal/app/orgtransfer/attachment_inventory.go b/internal/app/orgtransfer/attachment_inventory.go index c7f4d404e..1d70f5ccf 100644 --- a/internal/app/orgtransfer/attachment_inventory.go +++ b/internal/app/orgtransfer/attachment_inventory.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "hash/crc32" "io" "strings" @@ -79,7 +80,10 @@ func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts Attac if opts.SHA256 != "" && !strings.EqualFold(opts.SHA256, digest) { return nil, errors.New("archive SHA-256 differs from the requested inventory") } - zr, err := zip.NewReader(archive, archive.Size()) + zr, err := zip.NewReader(inventoryContextReaderAt{ctx, archive}, archive.Size()) + if ctx.Err() != nil { + return nil, ctx.Err() + } if err != nil { return nil, errors.New("archive is not a readable ZIP") } @@ -100,7 +104,7 @@ func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts Attac } manifestBody, readErr := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, mr}, maxManifestBytes+1)) _ = mr.Close() - if readErr != nil || len(manifestBody) > maxManifestBytes { + if readErr != nil || len(manifestBody) > maxManifestBytes || crc32.ChecksumIEEE(manifestBody) != manifestEntry.CRC32 { return nil, errors.New("cannot read archive manifest within the 64 MiB limit") } var m Manifest @@ -200,7 +204,7 @@ func inventoryContent(ctx context.Context, entry *zip.File, item AttachmentInven } defer r.Close() body, err := io.ReadAll(io.LimitReader(inventoryContextReader{ctx, r}, attachmenttype.MaxBytes+1)) - if err != nil { + if err != nil || crc32.ChecksumIEEE(body) != entry.CRC32 { return item } if int64(len(body)) != item.StoredBytes { @@ -243,7 +247,8 @@ func inventoryRows(ctx context.Context, entries map[string]*zip.File, m *Manifes return fmt.Errorf("cannot open %s metadata", table) } defer r.Close() - limited := &io.LimitedReader{R: inventoryContextReader{ctx, r}, N: maxInventoryTableBytes + 1} + checksum := crc32.NewIEEE() + limited := &io.LimitedReader{R: io.TeeReader(inventoryContextReader{ctx, r}, checksum), N: maxInventoryTableBytes + 1} scanner := bufio.NewScanner(limited) scanner.Buffer(make([]byte, 64<<10), 1<<20) var rows int64 @@ -262,6 +267,9 @@ func inventoryRows(ctx context.Context, entries map[string]*zip.File, m *Manifes if limited.N == 0 || rows != mt.Rows { return fmt.Errorf("%s metadata size or row count differs from the manifest", table) } + if checksum.Sum32() != entry.CRC32 { + return fmt.Errorf("cannot verify %s metadata integrity", table) + } return nil } @@ -283,6 +291,18 @@ type inventoryContextReader struct { r io.Reader } +type inventoryContextReaderAt struct { + ctx context.Context + r io.ReaderAt +} + +func (r inventoryContextReaderAt) ReadAt(b []byte, off int64) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.ReadAt(b, off) +} + func (r inventoryContextReader) Read(b []byte) (int, error) { if err := r.ctx.Err(); err != nil { return 0, err diff --git a/internal/app/orgtransfer/attachment_inventory_test.go b/internal/app/orgtransfer/attachment_inventory_test.go index de9bd51f5..8cdd3edb1 100644 --- a/internal/app/orgtransfer/attachment_inventory_test.go +++ b/internal/app/orgtransfer/attachment_inventory_test.go @@ -7,6 +7,9 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" + "hash/crc32" + "io" "strings" "testing" @@ -182,53 +185,248 @@ func TestAttachmentInventoryUsesCurrentCSVProfileWithoutChangingLegacyBytes(t *t } } -func TestAttachmentInventoryReportsMemberIntegrityAndRejectsDuplicatePaths(t *testing.T) { +func TestAttachmentInventoryRejectsDuplicatePaths(t *testing.T) { b := inventoryFixture(t, nil) zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b))) if err != nil { t.Fatal(err) } - var m Manifest - r, err := indexEntries(zr)[manifestPath].Open() - if err != nil { - t.Fatal(err) - } - if err := json.NewDecoder(r).Decode(&m); err != nil { - t.Fatal(err) - } - r.Close() - // Alter only the selected member's recorded CRC, leaving metadata readable. - var corrupt bytes.Buffer - w := zip.NewWriter(&corrupt) + var duplicate bytes.Buffer + w := zip.NewWriter(&duplicate) for _, f := range zr.File { - if f.Name == m.Blobs[0].Path { - f.CRC32++ - } if err := w.Copy(f); err != nil { t.Fatal(err) } } + if err := w.Copy(zr.File[0]); err != nil { + t.Fatal(err) + } if err := w.Close(); err != nil { t.Fatal(err) } - report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(corrupt.Bytes())}, AttachmentInventoryOptions{Limit: 1}) + if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(duplicate.Bytes())}, AttachmentInventoryOptions{Limit: 1}); err == nil || err.Error() != "archive contains duplicate ZIP paths" || report != nil { + t.Fatal("duplicate ZIP paths produced a report") + } +} + +type changingInventoryArchive struct { + *bytes.Reader + next *bytes.Reader +} + +func (a *changingInventoryArchive) ReadAt(p []byte, off int64) (int, error) { + n, err := a.Reader.ReadAt(p, off) + if off+int64(n) == a.Size() && a.next != nil { + a.Reader, a.next = a.next, nil + } + return n, err +} + +func TestAttachmentInventoryRefusesArchiveChangedDuringRead(t *testing.T) { + before := inventoryFixture(t, nil) + // ZIP comments change archive identity without changing any member or size. + first := append(bytes.Clone(before), 'a') + second := append(bytes.Clone(before), 'b') + for _, b := range [][]byte{first, second} { + b[len(before)-2] = 1 + if _, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err != nil { + t.Fatalf("immutable fixture must be readable: %v", err) + } + } + a := &changingInventoryArchive{Reader: bytes.NewReader(first), next: bytes.NewReader(second)} + report, err := InventoryAttachments(t.Context(), a, AttachmentInventoryOptions{Limit: 1}) + if report != nil || err == nil || err.Error() != "archive changed during inventory; discard this page" { + t.Fatalf("changed archive must not emit a page: report=%v error=%v", report, err) + } +} + +func rewriteInventoryMember(t *testing.T, source []byte, target string, transform func(*zip.FileHeader, []byte) []byte) ([]byte, int64) { + t.Helper() + zr, err := zip.NewReader(bytes.NewReader(source), int64(len(source))) if err != nil { t.Fatal(err) } - if report.Results[0].Status != "unreadable_blob" || report.Results[0].ContentSHA256 != "" { - t.Fatalf("integrity result: %+v", report.Results[0]) - } - var duplicate bytes.Buffer - w = zip.NewWriter(&duplicate) - for range 2 { - if err := w.Copy(zr.File[0]); err != nil { + var out bytes.Buffer + w := zip.NewWriter(&out) + for _, f := range zr.File { + if f.Name != target { + if err := w.Copy(f); err != nil { + t.Fatal(err) + } + continue + } + r, err := f.Open() + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(r) + r.Close() + if err != nil { + t.Fatal(err) + } + h := f.FileHeader + h.Flags &^= 8 + h.Method = zip.Store + h.Extra = nil + h.CompressedSize64 = uint64(len(body)) + raw := transform(&h, body) + dst, err := w.CreateRaw(&h) + if err != nil { + t.Fatal(err) + } + if _, err := dst.Write(raw); err != nil { t.Fatal(err) } } if err := w.Close(); err != nil { t.Fatal(err) } - if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(duplicate.Bytes())}, AttachmentInventoryOptions{Limit: 1}); err == nil || report != nil { - t.Fatal("duplicate ZIP paths produced a report") + b := out.Bytes() + zr, err = zip.NewReader(bytes.NewReader(b), int64(len(b))) + if err != nil { + t.Fatal(err) + } + for _, f := range zr.File { + if f.Name == target { + offset, err := f.DataOffset() + if err != nil { + t.Fatal(err) + } + return b, offset + } + } + t.Fatal("member not found") + return nil, 0 +} + +func TestAttachmentInventoryVerifiesEveryReadMemberCRC(t *testing.T) { + var blob string + source := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { blob = m.Blobs[0].Path }) + for _, target := range []string{blob, manifestPath, dataPath("campaigns"), dataPath("campaign_attachments")} { + for _, checksum := range []string{"valid", "zero", "nonzero"} { + t.Run(target+"/"+checksum, func(t *testing.T) { + b, _ := rewriteInventoryMember(t, source, target, func(h *zip.FileHeader, body []byte) []byte { + actual := crc32.ChecksumIEEE(body) + if actual == 0 || actual == 1 { + t.Fatal("fixture requires a distinct nonzero checksum") + } + switch checksum { + case "valid": + h.CRC32 = actual + case "zero": + h.CRC32 = 0 + case "nonzero": + h.CRC32 = actual ^ 1 + } + return body + }) + report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}) + if checksum != "valid" && target != blob { + if err == nil || report != nil { + t.Fatal("metadata integrity must be established before emitting a page") + } + return + } + if err != nil { + t.Fatal(err) + } + item := report.Results[0] + if checksum == "valid" { + if item.Status != "matches_upload_profile" || item.ContentSHA256 == "" { + t.Fatalf("valid member observation: %+v", item) + } + } else if item.Status != "unreadable_blob" || item.ContentSHA256 != "" || item.CanonicalMIME != "" { + t.Fatalf("member integrity requires unreadable_blob without content observations, got %+v", item) + } + }) + } + } +} + +func TestAttachmentInventoryAcceptsEmptyMetadataWithZeroCRC(t *testing.T) { + m := Manifest{Kind: ArchiveKind, FormatVersion: 1, OrganizationID: uuid.New(), + Tables: []ManifestTable{{Name: "campaigns"}, {Name: "campaign_attachments"}}} + body, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + var b bytes.Buffer + w := zip.NewWriter(&b) + for name, content := range map[string][]byte{manifestPath: body, dataPath("campaigns"): nil, dataPath("campaign_attachments"): nil} { + h := &zip.FileHeader{Name: name, Method: zip.Store, CRC32: crc32.ChecksumIEEE(content), + CompressedSize64: uint64(len(content)), UncompressedSize64: uint64(len(content))} + member, err := w.CreateRaw(h) + if err != nil { + t.Fatal(err) + } + if _, err := member.Write(content); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b.Bytes())}, AttachmentInventoryOptions{Limit: 1}) + if err != nil || report == nil || !report.Done || report.TotalRows != 0 { + t.Fatalf("empty metadata with valid zero CRC: report=%v error=%v", report, err) + } +} + +type inventoryCancelArchive struct { + *bytes.Reader + target int64 + requestSize int + cancel context.CancelFunc + started bool + readsAfterCancel int +} + +func (a *inventoryCancelArchive) ReadAt(p []byte, off int64) (int, error) { + if !a.started && off == a.target && len(p) == a.requestSize { + a.started = true + a.cancel() + } else if a.started { + a.readsAfterCancel++ + } + return a.Reader.ReadAt(p, off) +} + +func TestAttachmentInventoryCancelsArchiveSourceReads(t *testing.T) { + var blob string + source := inventoryFixture(t, func(m *Manifest, _ []models.CampaignAttachment, _ map[string][]byte) { blob = m.Blobs[0].Path }) + b, offset := rewriteInventoryMember(t, source, blob, func(h *zip.FileHeader, body []byte) []byte { + // Empty DEFLATE blocks exercise source reads without decompressed output. + raw := bytes.Repeat([]byte{0, 0, 0, 255, 255}, 200000) + n := uint16(len(body)) + raw = append(raw, 1, byte(n), byte(n>>8), byte(^n), byte(^n>>8)) + raw = append(raw, body...) + h.Method = zip.Deflate + h.CRC32 = crc32.ChecksumIEEE(body) + h.CompressedSize64 = uint64(len(raw)) + return raw + }) + if report, err := InventoryAttachments(t.Context(), inventoryArchive{bytes.NewReader(b)}, AttachmentInventoryOptions{Limit: 1}); err != nil || report.Results[0].Status != "matches_upload_profile" { + t.Fatalf("uncanceled fixture must be readable: report=%v error=%v", report, err) + } + for _, tc := range []struct { + name string + offset int64 + size int + }{ + {"index", int64(len(b) - 1024), 1024}, + {"decompress", offset, 4096}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + a := &inventoryCancelArchive{Reader: bytes.NewReader(b), target: tc.offset, requestSize: tc.size, cancel: cancel} + report, err := InventoryAttachments(ctx, a, AttachmentInventoryOptions{Limit: 1}) + if !a.started || report != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: triggered=%v report=%v error=%v", a.started, report, err) + } + if a.readsAfterCancel != 0 { + t.Fatalf("archive must stop source reads after cancellation, got %d further reads", a.readsAfterCancel) + } + }) } } From c03aea1c6b7c2c7b9eeebf8936974351f4072faa Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 10:10:47 +0100 Subject: [PATCH 3/5] feat: preserve attachment inventory interruption causes and discard incomplete reports at the read boundary --- .../app/orgtransfer/attachment_inventory.go | 7 +++- .../orgtransfer/attachment_inventory_test.go | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/app/orgtransfer/attachment_inventory.go b/internal/app/orgtransfer/attachment_inventory.go index 1d70f5ccf..ca328acb9 100644 --- a/internal/app/orgtransfer/attachment_inventory.go +++ b/internal/app/orgtransfer/attachment_inventory.go @@ -57,7 +57,12 @@ type AttachmentInventoryItem struct { } // InventoryAttachments reads archive bytes without a database, storage client or importer. -func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts AttachmentInventoryOptions) (*AttachmentInventory, error) { +func InventoryAttachments(ctx context.Context, archive ReaderAtSizer, opts AttachmentInventoryOptions) (result *AttachmentInventory, resultErr error) { + defer func() { + if err := ctx.Err(); err != nil { + result, resultErr = nil, err + } + }() if opts.After < 0 || opts.Limit < 1 || opts.Limit > 100 { return nil, errors.New("after must be nonnegative and limit must be between 1 and 100") } diff --git a/internal/app/orgtransfer/attachment_inventory_test.go b/internal/app/orgtransfer/attachment_inventory_test.go index 8cdd3edb1..78ab419c1 100644 --- a/internal/app/orgtransfer/attachment_inventory_test.go +++ b/internal/app/orgtransfer/attachment_inventory_test.go @@ -12,6 +12,7 @@ import ( "io" "strings" "testing" + "time" "github.com/google/uuid" @@ -382,7 +383,7 @@ type inventoryCancelArchive struct { } func (a *inventoryCancelArchive) ReadAt(p []byte, off int64) (int, error) { - if !a.started && off == a.target && len(p) == a.requestSize { + if !a.started && off == a.target && (a.requestSize == 0 || len(p) == a.requestSize) { a.started = true a.cancel() } else if a.started { @@ -430,3 +431,37 @@ func TestAttachmentInventoryCancelsArchiveSourceReads(t *testing.T) { }) } } + +func TestAttachmentInventoryPreservesManifestInterruption(t *testing.T) { + source := inventoryFixture(t, nil) + b, offset := rewriteInventoryMember(t, source, manifestPath, func(_ *zip.FileHeader, body []byte) []byte { return body }) + for _, stage := range []string{"header", "body"} { + for _, cause := range []string{"canceled", "deadline"} { + t.Run(stage+"/"+cause, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + interrupt := cancel + want := context.Canceled + if cause == "deadline" { + cancel() + ctx, cancel = context.WithTimeout(t.Context(), time.Second) + interrupt = func() { <-ctx.Done() } + want = context.DeadlineExceeded + } + defer cancel() + target, request := offset, 0 + if stage == "header" { + target -= int64(30 + len(manifestPath)) + request = 30 + } + a := &inventoryCancelArchive{Reader: bytes.NewReader(b), target: target, requestSize: request, cancel: interrupt} + report, err := InventoryAttachments(ctx, a, AttachmentInventoryOptions{Limit: 1}) + if !a.started || report != nil || !errors.Is(err, want) { + t.Fatalf("manifest interruption: triggered=%v report=%v error=%v, want %v", a.started, report, err, want) + } + if a.readsAfterCancel != 0 { + t.Fatalf("archive read after manifest interruption: %d", a.readsAfterCancel) + } + }) + } + } +} From 075d159a7018d70914daa427166736d88840ba40 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 10:19:23 +0100 Subject: [PATCH 4/5] feat: resume attachment preflight examples from the saved page cursor and fingerprint --- docs/content/docs/development/warmblyctl.mdx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index 62571a79b..e950cb3fa 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -611,10 +611,15 @@ Billing history, plan overrides, worker placement, mailbox sync checkpoints and ## org attachments-preflight +With `jq` installed, inspect the first page and resume only when its `done` field is false: + ```bash warmblyctl org attachments-preflight --file ./workspace.warmbly.zip --limit 25 > page-1.json -warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \ - --after 25 --expect-sha256 --limit 25 > page-2.json +if [ "$(jq -r '.done' page-1.json)" = false ]; then + warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \ + --after "$(jq -r '.next_after' page-1.json)" \ + --expect-sha256 "$(jq -r '.archive_sha256' page-1.json)" --limit 25 > page-2.json +fi ``` Reads private campaign attachment rows and their bytes from an **existing local workspace archive**, comparing each selected file with the [new-upload profile](/guides/campaigns/#campaign-attachments). This command works offline without `PRIMARY_DB`, a storage client, credentials or a destination workspace. It opens the archive for reading and prints JSON to stdout. Keep the archive immutable and protect reports as workspace data: they contain filenames, identifiers and content fingerprints. Obtaining an archive is a separate, authorized export operation; this command does not create or download one. From d5370760285dc67a3921871d57c5642c5ee35c81 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 10:25:41 +0100 Subject: [PATCH 5/5] feat: keep attachment preflight example reports private and preserve failed page status --- docs/content/docs/development/warmblyctl.mdx | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/content/docs/development/warmblyctl.mdx b/docs/content/docs/development/warmblyctl.mdx index e950cb3fa..3cf2ba462 100644 --- a/docs/content/docs/development/warmblyctl.mdx +++ b/docs/content/docs/development/warmblyctl.mdx @@ -611,15 +611,25 @@ Billing history, plan overrides, worker placement, mailbox sync checkpoints and ## org attachments-preflight -With `jq` installed, inspect the first page and resume only when its `done` field is false: +With `jq` installed, write reports to a new private directory and resume only when the first page succeeds and its `done` field is false. The subshell keeps its restrictive permissions and error handling local to this example: ```bash -warmblyctl org attachments-preflight --file ./workspace.warmbly.zip --limit 25 > page-1.json -if [ "$(jq -r '.done' page-1.json)" = false ]; then +( + set -eu + umask 077 + report_dir="$(mktemp -d ./attachment-preflight.XXXXXXXX)" + printf 'Reports: %s\n' "$report_dir" warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \ - --after "$(jq -r '.next_after' page-1.json)" \ - --expect-sha256 "$(jq -r '.archive_sha256' page-1.json)" --limit 25 > page-2.json -fi + --limit 25 > "$report_dir/page-1.json" + page_done="$(jq -r '.done' "$report_dir/page-1.json")" + if [ "$page_done" = false ]; then + next_after="$(jq -r '.next_after' "$report_dir/page-1.json")" + archive_sha256="$(jq -r '.archive_sha256' "$report_dir/page-1.json")" + warmblyctl org attachments-preflight --file ./workspace.warmbly.zip \ + --after "$next_after" --expect-sha256 "$archive_sha256" \ + --limit 25 > "$report_dir/page-2.json" + fi +) ``` Reads private campaign attachment rows and their bytes from an **existing local workspace archive**, comparing each selected file with the [new-upload profile](/guides/campaigns/#campaign-attachments). This command works offline without `PRIMARY_DB`, a storage client, credentials or a destination workspace. It opens the archive for reading and prints JSON to stdout. Keep the archive immutable and protect reports as workspace data: they contain filenames, identifiers and content fingerprints. Obtaining an archive is a separate, authorized export operation; this command does not create or download one.