feat(pipelines): record upstream snapshot ids on cascade-dispatched jobs (#9910)

* feat(pipelines): record upstream snapshot ids on cascade-dispatched jobs Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor: batch upstream-snapshot lookup and memoize per subscriber Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-09-11 08:07:15 +00:00 · 2026-07-04 00:10:18 +02:00
parent df6e511763
commit af36498432
7 changed files with 452 additions and 3 deletions
@@ -0,0 +1,77 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT DISTINCT ON (mp.asset_kind, mp.asset_path)\n                  mp.asset_kind AS \"asset_kind: AssetKind\", mp.asset_path,\n                  mp.snapshot_id AS \"snapshot_id!\", mp.partition\n             FROM materialized_partition mp\n             JOIN unnest($2::ASSET_KIND[], $3::text[]) AS u(kind, path)\n               ON mp.asset_kind = u.kind AND mp.asset_path = u.path\n            WHERE mp.workspace_id = $1\n              AND mp.status = 'materialized' AND mp.snapshot_id IS NOT NULL\n            ORDER BY mp.asset_kind, mp.asset_path, mp.snapshot_id DESC",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "asset_kind: AssetKind",
+        "type_info": {
+          "Custom": {
+            "name": "asset_kind",
+            "kind": {
+              "Enum": [
+                "s3object",
+                "resource",
+                "variable",
+                "ducklake",
+                "datatable",
+                "volume"
+              ]
+            }
+          }
+        }
+      },
+      {
+        "ordinal": 1,
+        "name": "asset_path",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "snapshot_id!",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 3,
+        "name": "partition",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        {
+          "Custom": {
+            "name": "asset_kind[]",
+            "kind": {
+              "Array": {
+                "Custom": {
+                  "name": "asset_kind",
+                  "kind": {
+                    "Enum": [
+                      "s3object",
+                      "resource",
+                      "variable",
+                      "ducklake",
+                      "datatable",
+                      "volume"
+                    ]
+                  }
+                }
+              }
+            }
+          }
+        },
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "0ee4a4dc8c6f09e7027bd99f98d8d1d346f9d67056ef272314dd9b80bee9f285"
+}
@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT DISTINCT trigger_ref AS \"trigger_ref!\"\n           FROM script_trigger\n           WHERE workspace_id = $1\n             AND runnable_path = $2\n             AND trigger_kind = 'asset'\n             AND runnable_kind = 'script'\n           ORDER BY trigger_ref",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "trigger_ref!",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "3703847d0cef9df3a480af8b6a5c21b6af282c68f06ce037a11f703284d8980d"
+}