diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 14183609af..d3e1661fde 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -22895,6 +22895,438 @@ paths: schema: type: string + /w/{workspace}/hub/publish_draft: + post: + summary: create or update a hub project draft + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubDraft + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishDraftBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts: + post: + summary: publish a script to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScript + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishScriptBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows: + post: + summary: publish a flow to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlow + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishFlowBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/apps: + post: + summary: publish an app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps: + post: + summary: publish a raw app to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawApp + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishRawAppBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/raw_apps/{id}/embed: + post: + summary: set or clear the embed url of a hub raw app + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubRawAppEmbed + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: id + in: path + required: true + description: hub id of the raw app + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RawAppEmbedBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/scripts/{ask_id}/recording: + post: + summary: attach a recording to a hub script + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubScriptRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: ask_id + in: path + required: true + description: hub ask id of the script + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/flows/{flow_id}/recording: + post: + summary: attach a recording to a hub flow + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubFlowRecording + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: flow_id + in: path + required: true + description: hub id of the flow + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RecordingBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resource_types: + post: + summary: publish a resource type to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResourceType + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourceTypeBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/resources: + post: + summary: publish resource placeholders to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubResources + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishResourcesBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/triggers: + post: + summary: publish triggers to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubTriggers + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishTriggersBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/migrations: + post: + summary: publish data table migrations to a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: publishHubMigrations + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublishMigrationsBody" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/export: + get: + summary: export a hub project + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub and returns the Hub's status code and raw response body. + The folder scope is only needed to re-export the caller's own draft; + approved projects are public, so it is optional here. + operationId: getHubProjectExport + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - name: folder + in: query + required: false + description: folder scoping the Hub project source (`{workspace}:{folder}`) + schema: + type: string + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/projects/{slug}/submit: + post: + summary: submit a hub project draft for review + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: submitHubProject + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: slug + in: path + required: true + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + schema: + type: string + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + + /w/{workspace}/hub/project: + get: + summary: get the hub project linked to a workspace folder + description: | + Requires the caller to be a workspace admin. Forwards the request to the + configured Hub scoped to the `{workspace}:{folder}` source and returns + the Hub's status code and raw response body. + operationId: getHubProjectBySource + tags: + - hubPublish + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/HubPublishFolder" + responses: + "200": + description: raw Hub response body (status code is passed through from the Hub) + content: + text/plain: + schema: + type: string + components: securitySchemes: bearerAuth: @@ -22943,6 +23375,16 @@ components: required: true schema: type: string + HubPublishFolder: + name: folder + in: query + required: true + description: | + workspace folder scoping the Hub publication: a workspace can publish + one Hub project per folder and the Hub-side source key is + `{workspace}:{folder}` + schema: + type: string PublicationName: name: publication in: path @@ -31839,3 +32281,276 @@ components: - name - owner - private + + HubProjectSlug: + type: string + description: hub project slug (3-50 chars, lowercase alphanumeric and hyphens, no leading/trailing hyphen) + minLength: 3 + maxLength: 50 + pattern: "^[a-z0-9][a-z0-9-]*[a-z0-9]$" + + PublishDraftBody: + type: object + properties: + slug: + $ref: "#/components/schemas/HubProjectSlug" + name: + type: string + summary: + type: string + readme: + type: string + required: + - slug + - name + - summary + + PublishScriptBody: + type: object + properties: + summary: + type: string + app: + type: string + description: + type: string + kind: + type: string + content: + type: string + language: + type: string + schema: + type: object + lockfile: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - summary + - app + - content + - language + - project_slug + + PublishFlowInner: + type: object + properties: + summary: + type: string + description: + type: string + value: + type: object + schema: + type: object + required: + - summary + - value + + PublishFlowBody: + type: object + properties: + flow: + $ref: "#/components/schemas/PublishFlowInner" + apps: + type: array + items: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - flow + - apps + - project_slug + + PublishAppBody: + type: object + properties: + app: + type: object + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - app + - apps + - summary + - project_slug + + PublishRawAppBody: + type: object + properties: + raw: + type: string + apps: + type: array + items: + type: string + description: + type: string + summary: + type: string + path: + type: string + source_path: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - raw + - apps + - summary + - project_slug + + RawAppEmbedBody: + type: object + properties: + external_embed_url: + type: string + nullable: true + description: explicit `null` clears the embed (unpublish) + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + RecordingBody: + type: object + properties: + recording: + type: object + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - project_slug + + PublishResourceTypeBody: + type: object + properties: + name: + type: string + schema: + type: object + description: + type: string + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - name + - project_slug + + PublishResourceBody: + type: object + properties: + path: + type: string + resource_type: + type: string + required: + - path + - resource_type + + PublishResourcesBody: + type: object + properties: + resources: + type: array + items: + $ref: "#/components/schemas/PublishResourceBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - resources + - project_slug + + PublishTriggerBody: + type: object + properties: + path: + type: string + kind: + type: string + summary: + type: string + nullable: true + description: + type: string + nullable: true + config: + type: object + script_ask_id: + type: integer + format: int64 + nullable: true + flow_id: + type: integer + format: int64 + nullable: true + required: + - path + - kind + - config + + PublishTriggersBody: + type: object + properties: + triggers: + type: array + items: + $ref: "#/components/schemas/PublishTriggerBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - triggers + - project_slug + + PublishMigrationBody: + type: object + description: one best-effort data table migration attached to a project (per data table) + properties: + datatable_name: + type: string + sql: + type: string + sql_down: + type: string + description: defaults to an empty string when omitted + enabled: + type: boolean + required: + - datatable_name + - sql + - enabled + + PublishMigrationsBody: + type: object + properties: + migrations: + type: array + items: + $ref: "#/components/schemas/PublishMigrationBody" + project_slug: + $ref: "#/components/schemas/HubProjectSlug" + required: + - migrations + - project_slug diff --git a/backend/windmill-api/src/hub_publish.rs b/backend/windmill-api/src/hub_publish.rs new file mode 100644 index 0000000000..e0baaecccf --- /dev/null +++ b/backend/windmill-api/src/hub_publish.rs @@ -0,0 +1,521 @@ +use crate::auth::Tokened; +use crate::db::ApiAuthed; +use crate::HTTP_CLIENT; +use axum::{ + extract::{FromRequestParts, Json, Path, Query, RawPathParams}, + http::{request::Parts, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use serde::{Deserialize, Deserializer, Serialize}; +use windmill_common::{ + error::{to_anyhow, Error}, + utils::require_admin, + HUB_BASE_URL, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/publish_draft", post(publish_draft)) + .route("/scripts", post(publish_script)) + .route("/flows", post(publish_flow)) + .route("/apps", post(publish_app)) + .route("/raw_apps", post(publish_raw_app)) + .route("/raw_apps/{id}/embed", post(publish_raw_app_embed)) + .route( + "/scripts/{ask_id}/recording", + post(publish_script_recording), + ) + .route("/flows/{flow_id}/recording", post(publish_flow_recording)) + .route("/resource_types", post(publish_resource_type)) + .route("/resources", post(publish_resources)) + .route("/triggers", post(publish_triggers)) + .route("/migrations", post(publish_migrations)) + .route("/projects/{slug}/export", get(get_project_export)) + .route("/projects/{slug}/submit", post(submit_project)) + .route("/project", get(get_project_by_source)) +} + +#[derive(Deserialize)] +struct HubScope { + folder: Option, +} + +fn validate_folder(folder: &str) -> Result<(), Error> { + let ok = !folder.is_empty() + && folder.len() <= 255 + && folder + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid folder: {folder}"))) + } +} + +fn source_key(workspace: &str, folder: &str) -> Result { + validate_folder(folder)?; + Ok(format!("{workspace}:{folder}")) +} + +fn validate_project_slug(slug: &str) -> Result<(), Error> { + let ok = slug.len() >= 3 + && slug.len() <= 50 + && !slug.starts_with('-') + && !slug.ends_with('-') + && slug + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); + if ok { + Ok(()) + } else { + Err(Error::BadRequest(format!("invalid project slug: {slug}"))) + } +} + +/// A Hub project slug that is valid by construction: deserialization (from a +/// request body or a path segment) is the only way to obtain one and it runs +/// `validate_project_slug`, so no handler can forward or interpolate an +/// unvalidated slug into a Hub URL. +#[derive(Serialize)] +#[serde(transparent)] +struct ProjectSlug(String); + +impl<'de> Deserialize<'de> for ProjectSlug { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + validate_project_slug(&s).map_err(serde::de::Error::custom)?; + Ok(ProjectSlug(s)) + } +} + +impl std::fmt::Display for ProjectSlug { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// The single gate every Hub endpoint goes through: an admin caller, their +/// token (the Hub authenticates it back against this instance's whoami), and +/// the validated `workspace_id:folder` source key scoping ownership Hub-side. +/// Handlers can only reach the Hub via this extractor's methods, so a new +/// endpoint cannot forget the admin check or folder validation. +/// +/// A workspace can publish one Hub project per folder. The stable, never-mutated +/// link key is `workspace_id:folder_name` (folder name is the path segment and is +/// never renamed — only display_name changes). `:` is safe: neither workspace ids +/// nor folder names (alphanumeric, underscore, hyphen) contain it. +struct HubPublishCtx { + source_id: Option, + token: String, +} + +impl FromRequestParts for HubPublishCtx +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + let authed = ApiAuthed::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let tokened = Tokened::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let params = RawPathParams::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let workspace = params + .iter() + .find(|(k, _)| *k == "workspace_id") + .map(|(_, v)| v.to_owned()); + let Query(scope) = Query::::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let build = || -> Result { + require_admin(authed.is_admin, &authed.username)?; + let workspace = workspace.ok_or_else(|| { + Error::internal_err( + "hub publish route must be nested under /w/{workspace_id}".to_string(), + ) + })?; + let source_id = scope + .folder + .as_deref() + .map(|f| source_key(&workspace, f)) + .transpose()?; + Ok(HubPublishCtx { source_id, token: tokened.token }) + }; + build().map_err(IntoResponse::into_response) + } +} + +impl HubPublishCtx { + fn require_source(&self) -> Result<&str, Error> { + self.source_id + .as_deref() + .ok_or_else(|| Error::BadRequest("missing folder query param".to_string())) + } + + async fn post( + &self, + path: &str, + body: &T, + ) -> Result<(StatusCode, String), Error> { + forward_to_hub(path, self.require_source()?, &self.token, body).await + } + + async fn get(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.require_source()?, &self.token).await + } + + /// GET without requiring a folder scope. Only for reads the Hub allows + /// publicly (e.g. exporting an approved project); the empty source id makes + /// the Hub skip the ownership match. + async fn get_maybe_unscoped(&self, path: &str) -> Result<(StatusCode, String), Error> { + get_from_hub(path, self.source_id.as_deref().unwrap_or(""), &self.token).await + } +} + +#[derive(Deserialize, Serialize)] +struct PublishDraftBody { + slug: ProjectSlug, + name: String, + summary: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + readme: Option, +} + +async fn publish_draft( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/projects", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishScriptBody { + summary: String, + app: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + kind: Option, + content: String, + language: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + lockfile: Option, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_script( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/scripts/add", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowInner { + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + value: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishFlowBody { + flow: PublishFlowInner, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_flow( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/flows", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishAppBody { + app: serde_json::Value, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct PublishRawAppBody { + raw: String, + apps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + summary: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + source_path: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post("/raw_apps", &body).await +} + +#[derive(Deserialize, Serialize)] +struct RawAppEmbedBody { + // No skip_serializing_if: `null` must reach the Hub to clear the embed (unpublish). + external_embed_url: Option, + project_slug: ProjectSlug, +} + +async fn publish_raw_app_embed( + ctx: HubPublishCtx, + Path((_workspace, id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/raw_apps/{}/embed", id), &body).await +} + +#[derive(Deserialize, Serialize)] +struct RecordingBody { + #[serde(skip_serializing_if = "Option::is_none")] + recording: Option, + project_slug: ProjectSlug, +} + +async fn publish_script_recording( + ctx: HubPublishCtx, + Path((_workspace, ask_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/scripts/{}/recording", ask_id), &body) + .await +} + +async fn publish_flow_recording( + ctx: HubPublishCtx, + Path((_workspace, flow_id)): Path<(String, i64)>, + Json(body): Json, +) -> Result { + ctx.post(&format!("/flows/{}/recording", flow_id), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceTypeBody { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + project_slug: ProjectSlug, +} + +async fn publish_resource_type( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/resource_types", body.project_slug), + &body, + ) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishResourceBody { + path: String, + resource_type: String, +} + +#[derive(Deserialize, Serialize)] +struct PublishResourcesBody { + resources: Vec, + project_slug: ProjectSlug, +} + +async fn publish_resources( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/resources", body.project_slug), &body) + .await +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggerBody { + path: String, + kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + summary: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + config: serde_json::Value, + #[serde(skip_serializing_if = "Option::is_none")] + script_ask_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + flow_id: Option, +} + +#[derive(Deserialize, Serialize)] +struct PublishTriggersBody { + triggers: Vec, + project_slug: ProjectSlug, +} + +async fn publish_triggers( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post(&format!("/projects/{}/triggers", body.project_slug), &body) + .await +} + +// One best-effort data table migration attached to a project (per data table). +#[derive(Deserialize, Serialize)] +struct PublishMigrationBody { + datatable_name: String, + sql: String, + #[serde(default)] + sql_down: String, + enabled: bool, +} + +#[derive(Deserialize, Serialize)] +struct PublishMigrationsBody { + migrations: Vec, + project_slug: ProjectSlug, +} + +async fn publish_migrations( + ctx: HubPublishCtx, + Json(body): Json, +) -> Result { + ctx.post( + &format!("/projects/{}/migrations", body.project_slug), + &body, + ) + .await +} + +// Export is owner-scoped only when re-exporting your own draft; approved +// projects are public, so the folder scope is optional here. +async fn get_project_export( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.get_maybe_unscoped(&format!("/projects/{}/export", slug)) + .await +} + +async fn get_project_by_source(ctx: HubPublishCtx) -> Result { + ctx.get("/projects/by_source").await +} + +async fn submit_project( + ctx: HubPublishCtx, + Path((_workspace, slug)): Path<(String, ProjectSlug)>, +) -> Result { + ctx.post( + &format!("/projects/{}/submit", slug), + &serde_json::json!({}), + ) + .await +} + +// The Hub has no auth of its own: it validates bearer tokens by calling this +// instance's /api/users/whoami. Forwarding the caller's own token logs them in +// on the Hub as themselves (account auto-created on first use). +async fn get_from_hub( + path: &str, + source_id: &str, + token: &str, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let res = HTTP_CLIENT + .get(&url) + .query(&[("source_id", source_id)]) + .bearer_auth(token) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} + +async fn forward_to_hub( + path: &str, + source_id: &str, + token: &str, + body: &T, +) -> Result<(StatusCode, String), Error> { + let url = format!("{}{}", **HUB_BASE_URL.load(), path); + + let mut payload = serde_json::to_value(body).map_err(to_anyhow)?; + let obj = payload + .as_object_mut() + .ok_or_else(|| Error::internal_err("hub publish body must be a JSON object".to_string()))?; + obj.insert( + "source_id".to_string(), + serde_json::Value::String(source_id.to_string()), + ); + + let res = HTTP_CLIENT + .post(&url) + .bearer_auth(token) + .json(&payload) + .send() + .await + .map_err(|e| Error::InternalErr(format!("hub request failed: {e}")))?; + + let status = StatusCode::from_u16(res.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY); + let text = res + .text() + .await + .map_err(|e| Error::InternalErr(format!("hub response read failed: {e}")))?; + + Ok((status, text)) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index d56928a712..ce66ec22da 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -94,6 +94,7 @@ mod granular_acls; mod group_history; mod groups; mod health; +mod hub_publish; #[cfg(feature = "private")] pub mod indexer_ee; mod indexer_oss; @@ -656,6 +657,7 @@ pub async fn run_server( .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) + .nest("/hub", hub_publish::workspaced_service()) .nest( "/data_metrics", windmill_api_workspaces::data_metrics::workspaced_service(), diff --git a/frontend/src/lib/components/DatatableSchemaDiff.svelte b/frontend/src/lib/components/DatatableSchemaDiff.svelte index 9eccda8195..ac215747e0 100644 --- a/frontend/src/lib/components/DatatableSchemaDiff.svelte +++ b/frontend/src/lib/components/DatatableSchemaDiff.svelte @@ -1,195 +1,11 @@ - - + +{#if deployHub.session} + {#key deployHub.session} + {@const s = deployHub.session} +
+ + {#snippet header()} + {@const stepNum = + s.phase === 'predeploy' + ? 1 + : s.phase === 'draft' + ? 2 + : s.phase === 'under_review' + ? 3 + : 4} +
+
    + + How to publish your project to the Hub + +
  1. 1 ? 'opacity-60' : ''}> + {stepNum > 1 ? '✓' : '1.'} + Bundle your project — creates a draft + on the Hub with every selected script, flow, app and resource from this folder. +
    2. +
  2. 2 ? 'opacity-60' : 'opacity-40'} + > + {stepNum > 2 ? '✓' : '2.'} + Generate iframes & recordings — share + public apps as iframes and capture one execution per script/flow. +
    3. +
  3. 3 ? 'opacity-60' : 'opacity-40'} + > + {stepNum > 3 ? '✓' : '3.'} + Submit for review — send the bundle for + approval. +
    4. +
+
+ {#if s.phase === 'predeploy'} + + Step 1: Bundle your project + + {:else if s.phase === 'draft'} + + Step 2: Generate iframes & recordings + + {:else if s.phase === 'under_review'} + Step 3: Awaiting review + {:else} + Live on the Hub + {/if} + {#if s.phase !== 'predeploy'} + + + on Hub: + {s.hubName || s.hubSlug} + + + Open in Hub + + {/if} +
+ {#if s.phase === 'predeploy'} + + {:else if s.phase === 'draft'} + + {:else if s.phase === 'under_review'} + + {/if} +
+
+ {#if s.phase === 'predeploy'} +
+ + Bundling creates a draft project on the Hub from the selected scripts, flows and + apps of {s.selectedFolder}/. + {s.selectedItems.length} of {s.filteredWorkspaceItems.length} items selected. + +
+
+ + Resource dependencies + {#if s.detectingResources} + + {:else} + ({s.dependencyTypes.length}) + {/if} + + Resource types the selected items depend on (whether passed as inputs or + referenced by a hardcoded path). Synced to the Hub so a fork knows what + credentials it needs to fill. + + + {#if s.dependencyTypes.length === 0} + + No resource references detected in the current selection. + + {:else} + {#each s.dependencyTypes as r (r.resource_type)} + + {r.resource_type} + + {/each} + + {/if} +
+
+ + Data table dependencies + {#if s.detectingDatatables} + + {:else} + ({s.datatableUsage.size}) + {/if} + + Data tables the selected items read or write. A best-effort CREATE TABLE + migration for these is generated in the bundle step and shipped with the + project, so a fork can recreate the tables it needs. + + + {#if s.datatableUsage.size === 0} + + No data table usage detected in the current selection. + + {:else} + {#each [...s.datatableUsage] as [dt, tables] (dt)} + + {dt} + {#if tables.size > 0} + ×{tables.size} + {/if} + + {/each} + {/if} +
+ {/if} + {#if s.phase === 'draft'} +
+ + A recording captures one real run of a script or flow — inputs, logs, step outputs + and result — replayable on the Hub so visitors see it work before forking. Public + apps can also be shared as live iframes. Optional, but recommended. + +
+ {/if} + {#if s.phase === 'predeploy'} +
+ + Triggers + {#if s.triggersLoading} + + {:else} + ({s.relevantTriggers.length}) + {/if} + + {#if s.triggerDiscoveryFailed} + + Some trigger kinds could not be listed — publishing is disabled so triggers + aren't silently left out of the bundle. + + + {:else if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} + + {TRIGGER_KINDS[kind].badge} + ×{triggers.length} + + {/each} + + {/if} +
+ {/if} + {#if s.phase === 'under_review'} +
+ +
+ Locked while under review + + The Windmill team is reviewing this submission. Editing, recording, and sharing + actions are disabled. Estimated turnaround: 1-2 business days. + +
+
+ {/if} + {#if s.phase === 'draft'} + {@const recordedCount = s.recordableItems.filter((i) => i.rec === 'recorded').length} + {@const pct = s.recordableItems.length + ? Math.round((recordedCount / s.recordableItems.length) * 100) + : 0} +
+ {recordedCount}/{s.recordableItems.length} +
+
+
+ + {s.allRecorded ? 'Full recordings' : 'Recordings recommended'} + +
+ {/if} +
+ {/snippet} + + {#snippet itemSummary(item)} + {@const it = item as DeployItem} + + {it.summary?.trim() || it.path} + + {/snippet} + + {#snippet itemActions(item)} + {@const it = item as DeployItem} + {#if s.phase !== 'predeploy' && canRecord(it.kind)} + {#if it.rec === 'recorded'} + + Recorded + + {#if s.recordings[it.key]} + + See recording + + {/if} + {#if s.phase === 'draft'} + + {/if} + {:else if s.phase === 'draft'} + No recording + + {:else} + No recording + {/if} + {/if} + {#if s.phase !== 'predeploy' && canShareAsIframe(it)} + {#if it.published} + + Public + + {#if it.publicUrl} + + Open + + + {:else if s.phase !== 'under_review'} + + + {/if} + {#if s.phase !== 'under_review'} + + {/if} + {:else if s.phase !== 'under_review'} + + {/if} + {/if} + {/snippet} + + {#snippet footer()} +
+ {#if s.phase === 'predeploy'} + + Select the items to include — all selected by default. + + {:else if s.phase === 'draft'} + + {#if s.allRecorded} + All scripts and flows have a recording — best chance of approval and featuring. + {:else} + {s.recordableItems.filter((i) => i.rec === 'recorded').length} of {s + .recordableItems.length} + recorded. Bundles with full recordings get approved faster and featured on the public + Hub. + {/if} + + {:else if s.phase === 'under_review'} + + Waiting for the Windmill team to review the submission. + + {:else} + Iterate further by starting a new draft. + {/if} +
+ {/snippet} + +
+ + + recordDrawer?.closeDrawer()} + > +
+

+ Run this {s.recordTarget?.kind} once with the inputs below. The full execution — args, logs, + intermediate step outputs and final result — is saved as a replayable recording + shown on the Hub page. Visitors can step through it to see how the {s.recordTarget + ?.kind} works without running anything themselves. +

+ + {#if s.runState !== 'idle'} +
+
+ {#if s.runState === 'running'} + + Running… + {:else if s.runState === 'success'} + + + Execution succeeded + + {:else} + + Execution failed + {/if} + {#if s.runJobId} + + Open job + + {/if} +
+ {#if s.runState === 'success' && s.runResult !== undefined} +
+ Result preview: + 
{JSON.stringify(s.runResult, null, 2)}
+
+ {:else if s.runState === 'failed' && s.runError} + 
{s.runError}
+ {/if} + {#if s.runState === 'success'} +
+ + Looks good? Save this run as the Hub recording. + + +
+ {:else if s.runState === 'failed'} + + Fix inputs and try again. Only successful runs can be saved as a recording. + + {/if} +
+ {/if} + + {#if s.recordSchemaLoading} + Loading schema… + {:else} + + {/if} +
+ {#snippet actions()} + {#if s.runState === 'success'} + + + {:else} + + {/if} + {/snippet} + + + + + publishDrawer?.closeDrawer()} + > +
+

+ Expose {s.publishTarget?.path} at a public URL + so it can be embedded as an iframe (e.g. on the Hub, a docs page, or your own site). Anyone + with the URL will be able to interact with it. +

+ +
+
+ + Rate limit (workspace-wide) + + Caps public app executions per minute per server. Applies to all public apps in this + workspace. + +
+ {#if s.workspaceRateLimit && s.workspaceRateLimit > 0} + + Currently {s.workspaceRateLimit} executions + / minute / server. + + {:else} + + No rate limit configured — anyone with the URL can hit this app at any rate. + + {/if} + publishDrawer?.closeDrawer()} + > + Edit in Workspace settings → Apps + +
+
+ {#snippet actions()} + + + {/snippet} + + + + + resourceDrawer?.closeDrawer()}> +
+

+ Resource types the selected items depend on. Each is synced to the Hub so a fork knows + what credentials it needs to fill. Input means the + item takes the resource as a parameter; + hardcoded path means the item pins a specific resource + path in its code. +

+ {#if s.dependencyTypes.length === 0} + No resource references in the current selection. + {:else} + {#each s.dependencyTypes as r (r.resource_type)} +
+
+ + {r.resource_type} + + + {r.usages.length} usage{r.usages.length > 1 ? 's' : ''} + +
+
+ {#each r.usages as u, ui (ui)} + {#if u.role === 'trigger'} +
+ + {u.label} + + {TRIGGER_KINDS[u.triggerKind].badge} trigger + +
+ {:else} + {@const itemUrl = s.itemUrl(u.kind, u.itemPath)} +
+
+ {#if u.kind === 'script'} + + {:else if u.kind === 'flow'} + + {:else} + + {/if} + {u.label} + + {u.role === 'hardcoded' ? 'hardcoded path' : 'input'} + {#if u.role === 'hardcoded'} + + + {#snippet text()} +
+ + This {u.kind} references the resource by a hardcoded path + $res:{u.path}. + + + For portability, prefer taking the resource as an input — a + fork won't have this exact path. It's relocated into the + project on publish, but converting it to an input keeps the + item reusable. + +
+ {/snippet} + + {/if} + + {#if itemUrl} + + + + {/if} +
+
+ {/if} + {/each} +
+
+ {/each} + {/if} +
+ + + + + triggerDrawer?.closeDrawer()}> +
+

+ Triggers attached to the selected scripts and flows. Synced to the Hub as + disabled stubs. Recipients review and enable each one + manually after importing. External hooks (Slack/Discord webhooks, message-queue + subscriptions, etc.) must be re-registered against the importing instance. +

+ {#if s.relevantTriggers.length === 0} + No triggers reference the selected items. + {:else} + {#each s.triggersByKind as [kind, triggers] (kind)} +
+
+ + {TRIGGER_KINDS[kind].badge} + + + {triggers.length} trigger{triggers.length > 1 ? 's' : ''} + + {#if TRIGGER_KINDS[kind].note} + + + + {#snippet text()} +
+ {TRIGGER_KINDS[kind].note} +
+ {/snippet} + + + {/if} +
+
+ {#each triggers as t (t.path)} + {@const runnableSummary = s.runnableSummaryByPath.get( + `${t.is_flow ? 'flow' : 'script'}:${t.script_path}` + )} + {@const details = triggerDetails(t)} + {@const cfg = t.config as any} + {@const previewKey = + t.kind === 'schedule' ? `${cfg.schedule}|${cfg.timezone}` : ''} + {@const preview = + t.kind === 'schedule' ? s.schedulePreviews[previewKey] : undefined} + {@const triggerUrl = s.triggerListUrl(t.kind)} +
+
+ {#if t.is_flow} + + {:else} + + {/if} + + {runnableSummary || t.script_path} + + + {t.is_flow ? 'flow' : 'script'} + + {#if triggerUrl} + + + + {/if} +
+ {#if details.length > 0} +
+ {#each details as d (d.label)} +
{d.label}
+
{d.value}
+ {/each} + {#if t.kind === 'schedule'} +
Next runs
+
+ {#if preview && preview.length > 0} +
+ {#each preview as date (date)} + {displayDate(date)} + {/each} +
+ {:else if preview && preview.length === 0} + No upcoming run + {:else} + Loading… + {/if} +
+ {/if} +
+ {/if} +
+ {/each} +
+
+ {/each} + {/if} +
+ + + + + bundleDrawer?.closeDrawer()}> +
+

+ Name and document your bundle. The readme can be updated later, but a clear one speeds + up the Windmill team's review. +

+ {#if s.bundlePreview && s.bundlePreview.unresolved.length > 0} +
+ {s.bundlePreview.unresolved.length} unresolved reference(s) — cannot publish + + These items or resources couldn't be resolved, so the bundle would ship broken + references. Deselect or fix them, then retry: + +
    + {#each s.bundlePreview.unresolved as u (u)} +
  • {u}
    • + {/each} +
+
+ {/if} + +
+ Project slug + + {s.effectiveSlug || sanitizeSlug(s.hubName) || '—'} + + + {#if s.effectiveSlug} + Locked — items live under f/{s.effectiveSlug}/. + {:else if s.hubName.trim() && !isValidSlug(sanitizeSlug(s.hubName))} + + The name yields an invalid slug. Use at least 3 letters/digits. + + {:else} + Auto-generated from the name. Once project forked, items will live under + f/{sanitizeSlug(s.hubName) || ''}/. + {/if} + +
+ + +
+
+ + Data table migrations +
+ {#if s.migrationsGenerating} +
+ + Detecting data tables used by this project… +
+ {:else if s.migrationDrafts.length === 0} + + No data table usage detected in this project's scripts, flows, or raw apps. + + {:else} + + We detected these data tables. When included, the migration recreates their tables + on import. Best-effort — review and edit before publishing. + + {#each s.migrationDrafts as m (m.datatable_name)} +
+
+ {m.datatable_name} + +
+ +
+ {/each} + {/if} +
+
+ {#snippet actions()} + + {/snippet} + + + {/key} +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte new file mode 100644 index 0000000000..76e3da3e07 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/MigrationSqlEditor.svelte @@ -0,0 +1,40 @@ + + +
+
+ {#each [{ id: 'up', label: 'Up' }, { id: 'down', label: 'Down' }] as t (t.id)} + + {/each} +
+ {#key generation} +
+ {#if tab === 'up'} + + {:else} + + {/if} +
+ {/key} +
diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts new file mode 100644 index 0000000000..11926c804b --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.svelte.ts @@ -0,0 +1,1635 @@ +import { untrack } from 'svelte' +import { base } from '$lib/base' +import { + AppService, + FlowService, + JobService, + RawAppService, + ResourceService, + ScriptService, + WorkspaceService, + ScheduleService +} from '$lib/gen' +import { sendUserToast } from '$lib/toast' +import { sleep, emptySchema } from '$lib/utils' +import { computeSecretUrl } from '$lib/components/apps/editor/appDeploy.svelte' +import { + buildProjectBundle, + buildPathMap, + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + rewriteTriggerConfig, + rewriteVarRefsInValue, + type BundleDeps, + type BundledItem, + type FetchedItem, + type ItemKind, + type ItemRef, + type ProjectBundle +} from './projectBundle' +import { + detectDatatableTables, + generateDatatableMigrations, + type GeneratedMigration +} from './projectMigrations' +import type { Kind } from '$lib/utils_deployable' +import { + TRIGGER_KINDS, + listAllWorkspaceTriggers, + triggerResourcePath, + triggerHandlerRefs, + portableTriggerConfig, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' + +export type Phase = 'predeploy' | 'draft' | 'under_review' | 'live' +export type RecStatus = 'none' | 'recorded' +export interface DeployItem { + key: string + path: string + kind: Kind + summary?: string + rec: RecStatus + published?: boolean + publicUrl?: string + [k: string]: unknown +} + +export const canRecord = (k: Kind) => k === 'script' || k === 'flow' +// Legacy raw apps live only in the `raw_app` table, but the iframe share flow +// drives AppService (the `app` table), so it can only target apps stored there. +export const canShareAsIframe = (it: DeployItem): boolean => + it.kind === 'app' || (it.kind === 'raw_app' && it.appTable === true) + +// Hub rehydration only carries draft membership, not the live share state of an +// app. Copy the public-execution flag, public URL, and app-table origin from the +// loaded workspace items onto matching draft items so a still-public app keeps its +// Public badge, Unpublish, and iframe controls after its draft is reopened. Returns +// the original array unchanged when nothing needs merging (stable reference). +export function mergeShareState( + draftItems: DeployItem[], + workspaceItems: DeployItem[] +): DeployItem[] { + if (draftItems.length === 0 || workspaceItems.length === 0) return draftItems + const byKey = new Map(workspaceItems.map((w) => [w.key, w])) + let changed = false + const merged = draftItems.map((d) => { + const w = byKey.get(d.key) + if (!w) return d + if (w.published !== d.published || w.publicUrl !== d.publicUrl || w.appTable !== d.appTable) { + changed = true + return { ...d, published: w.published, publicUrl: w.publicUrl, appTable: w.appTable } + } + return d + }) + return changed ? merged : draftItems +} + +export function sanitizeSlug(s: string): string { + return s + .toLowerCase() + .replace(/[_\s]+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50) + .replace(/-+$/g, '') +} +const SLUG_RE = /^[a-z0-9][a-z0-9-]{1,48}[a-z0-9]$/ +export function isValidSlug(s: string): boolean { + return SLUG_RE.test(s) +} + +export type RunState = 'idle' | 'running' | 'success' | 'failed' + +const ITEM_KIND_ROUTE: Record = { + script: 'scripts/get', + flow: 'flows/get', + app: 'apps/get', + raw_app: 'apps_raw/get' +} + +const HIDDEN_RESOURCE_TYPES = new Set(['app_theme', 'state', 'cache']) + +function typesFromSchema(schema: any): string[] { + const out = new Set() + const props = schema?.properties + if (props && typeof props === 'object') { + for (const key of Object.keys(props)) { + const fmt = props[key]?.format + if (typeof fmt === 'string' && fmt.startsWith('resource-')) { + out.add(fmt.slice('resource-'.length)) + } + } + } + return [...out] +} + +type DependencyUsage = + | { role: 'input'; label: string; kind: ItemKind; itemPath: string } + | { role: 'hardcoded'; label: string; kind: ItemKind; path: string; itemPath: string } + | { role: 'trigger'; label: string; triggerKind: WorkspaceTriggerKind; path: string } +export interface DependencyType { + resource_type: string + hasHardcoded: boolean + usages: DependencyUsage[] +} + +interface SessionDeps { + hasEeLicense: () => boolean +} + +/** + * All state and async operations for one Deploy-to-Hub surface, bound to an + * immutable (workspace, folder) pair. A workspace or folder change never mutates + * a session — `useDeployToHubSession` replaces the instance, so in-flight async + * work keeps writing to the discarded object and cannot leak into the new scope. + * The only invalidation tokens left are intra-session (competing calls on the + * same session), not lifecycle guards. + */ +export class DeployToHubSession { + readonly workspace: string + readonly folder: string + /** `f/`-prefixed folder path the project is scoped to. */ + readonly selectedFolder: string + + #disposed = false + #deps: SessionDeps + + phase = $state('predeploy') + workspaceItems = $state([]) + draftItems = $state([]) + workspaceTriggers = $state([]) + triggersLoading = $state(false) + // True when a trigger kind's discovery failed (not a feature-gated 404): + // the trigger list may be incomplete, so publishing is blocked until a + // retry succeeds. + triggerDiscoveryFailed = $state(false) + schedulePreviews = $state>({}) + manualDeselected = $state>(new Set()) + loading = $state(false) + workspaceRateLimit = $state(undefined) + deploymentStatus = $state< + Record + >({}) + deploying = $state(false) + + recordTarget = $state() + recordArgs = $state>({}) + recordValid = $state(true) + recordSchema = $state>(emptySchema()) + recordSchemaLoading = $state(false) + runState = $state('idle') + runJobId = $state(undefined) + runResult = $state(undefined) + runError = $state(undefined) + recordings = $state>({}) + + publishTarget = $state() + publishing = $state(false) + + hubName = $state('') + hubSummary = $state('') + hubReadme = $state('') + effectiveSlug = $state('') + hubItemIds = $state>({}) + + // Best-effort data table migrations for the bundle, editable in the drawer and + // pushed on deploy. Regenerated when the bundle drawer opens. + migrationDrafts = $state([]) + migrationsGenerating = $state(false) + // Bumped whenever the drafts are (re)generated, to re-key the Monaco editors so + // they pick up the fresh SQL (Monaco doesn't sync external `code` changes). + migrationsGeneration = $state(0) + + bundlePreview = $state(undefined) + detectingResources = $state(false) + // Data tables (→ tables) the current selection reads/writes, detected off the + // same bundle preview. Drives the predeploy "Data table dependencies" summary; + // the editable migration itself is generated in the bundle drawer. + datatableUsage = $state>>(new Map()) + detectingDatatables = $state(false) + + submitting = $state(false) + syncing = $state(false) + + // Intra-session tokens: latest call wins among competing calls on this session. + #triggerLoadTok = 0 + #recordRunTok = 0 + #migrationsTok = 0 + #schedulePreviewsInFlight = new Set() + // Preview-only cache: toggling checkboxes re-runs the closure walk, but item + // contents don't change mid-session. deployAll bypasses this and fetches fresh. + #previewItemCache = new Map>() + #previewTypeCache = new Map>() + + constructor(workspace: string, folder: string, deps: SessionDeps) { + this.workspace = workspace + this.folder = folder + this.selectedFolder = `f/${folder}` + this.#deps = deps + } + + dispose() { + this.#disposed = true + } + + load() { + void this.#loadWorkspace() + void this.#loadTriggers() + void this.rehydrateFromHub() + } + + filteredWorkspaceItems = $derived( + this.workspaceItems.filter((i) => i.path.startsWith(this.selectedFolder + '/')) + ) + // Derived (not merged at load time) so it settles regardless of which of the + // racing loads (#loadWorkspace / rehydrateFromHub) finishes last. + draftItemsWithLocalState = $derived(mergeShareState(this.draftItems, this.workspaceItems)) + items = $derived( + this.phase === 'predeploy' ? this.filteredWorkspaceItems : this.draftItemsWithLocalState + ) + selectedItems = $derived( + this.phase === 'predeploy' + ? this.filteredWorkspaceItems.filter((i) => !this.manualDeselected.has(i.key)) + : [] + ) + selectedItemKeys = $derived(this.selectedItems.map((i) => i.key)) + allSelected = $derived( + this.phase === 'predeploy' && + this.selectedItemKeys.length === this.filteredWorkspaceItems.length + ) + recordableItems = $derived(this.items.filter((i) => canRecord(i.kind))) + allRecorded = $derived( + this.recordableItems.length > 0 && this.recordableItems.every((i) => i.rec === 'recorded') + ) + hubSlug = $derived(this.effectiveSlug || sanitizeSlug(this.hubName)) + + relevantTriggers = $derived.by(() => { + const selectedScripts = new Set( + this.selectedItems.filter((i) => i.kind === 'script').map((i) => i.path) + ) + const selectedFlows = new Set( + this.selectedItems.filter((i) => i.kind === 'flow').map((i) => i.path) + ) + return this.workspaceTriggers.filter((t) => + t.is_flow ? selectedFlows.has(t.script_path) : selectedScripts.has(t.script_path) + ) + }) + + triggersByKind = $derived.by(() => { + const out = new Map() + for (const t of this.relevantTriggers) { + const arr = out.get(t.kind) ?? [] + arr.push(t) + out.set(t.kind, arr) + } + return Array.from(out.entries()).sort((a, b) => a[0].localeCompare(b[0])) + }) + + runnableSummaryByPath = $derived.by(() => { + const m = new Map() + for (const it of this.workspaceItems) { + if (it.kind === 'script' || it.kind === 'flow') { + m.set(`${it.kind}:${it.path}`, it.summary) + } + } + return m + }) + + // `hasHardcoded` = pinned via $res: path (relocated as a stub); else input-only. + dependencyTypes = $derived.by(() => { + const b = this.bundlePreview + if (!b) return [] as DependencyType[] + const stubByNewPath = new Map(b.resourceStubs.map((s) => [s.newPath, s])) + const byType = new Map() + const ensure = (rt: string) => { + let e = byType.get(rt) + if (!e) { + e = { resource_type: rt, hasHardcoded: false, usages: [] } + byType.set(rt, e) + } + return e + } + for (const it of b.items) { + const label = (it.summary?.trim() || it.path) ?? it.path + const refs = + it.kind === 'flow' + ? extractFlowRefs(it.value).filter((r) => r.kind === 'resource') + : it.kind === 'app' + ? extractAppRefs(it.value) + : extractScriptRefs(it.content ?? '') + for (const r of refs) { + const stub = stubByNewPath.get(r.path) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + const e = ensure(stub.resource_type) + e.hasHardcoded = true + e.usages.push({ + role: 'hardcoded', + label, + kind: it.kind, + path: stub.originalPath, + itemPath: it.path + }) + } + for (const t of typesFromSchema(it.schema)) { + if (HIDDEN_RESOURCE_TYPES.has(t)) continue + ensure(t).usages.push({ role: 'input', label, kind: it.kind, itemPath: it.path }) + } + } + // Resources referenced only by a trigger (no item uses them in code) — + // its kind resource field or any `$res:` token in its config. + const stubByOriginal = new Map(b.resourceStubs.map((s) => [s.originalPath, s])) + for (const t of this.relevantTriggers) { + const refs = new Set( + extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config)) + ) + const rp = triggerResourcePath(t) + if (rp) refs.add(rp) + for (const ref of refs) { + const stub = stubByOriginal.get(ref) + if (!stub || HIDDEN_RESOURCE_TYPES.has(stub.resource_type)) continue + ensure(stub.resource_type).usages.push({ + role: 'trigger', + label: t.summary?.trim() || t.path, + triggerKind: t.kind, + path: stub.originalPath + }) + } + } + return [...byType.values()].sort((a, b) => a.resource_type.localeCompare(b.resource_type)) + }) + + toggleItem = (item: { key: string }) => { + const next = new Set(this.manualDeselected) + if (next.has(item.key)) next.delete(item.key) + else next.add(item.key) + this.manualDeselected = next + } + selectAll = () => { + this.manualDeselected = new Set() + } + deselectAll = () => { + this.manualDeselected = new Set(this.filteredWorkspaceItems.map((i) => i.key)) + } + + #folderQs(): string { + return `?folder=${encodeURIComponent(this.folder)}` + } + + itemUrl(kind: ItemKind, path: string): string | undefined { + if (!path) return undefined + return `${base}/${ITEM_KIND_ROUTE[kind]}/${path}?workspace=${this.workspace}` + } + triggerListUrl(kind: WorkspaceTriggerKind): string { + return `${base}/${TRIGGER_KINDS[kind].route}?workspace=${this.workspace}` + } + + #patchItem(key: string, patch: Partial) { + this.workspaceItems = this.workspaceItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + this.draftItems = this.draftItems.map((i) => (i.key === key ? { ...i, ...patch } : i)) + } + + async #listAllPages( + fetcher: (params: { perPage: number; page: number }) => Promise + ): Promise { + const perPage = 100 + const out: T[] = [] + for (let page = 1; page <= 1000; page++) { + const batch = await fetcher({ perPage, page }) + out.push(...batch) + if (batch.length < perPage) return out + } + return out + } + + async #loadWorkspace() { + const workspace = this.workspace + this.loading = true + try { + const [apps, rawApps, flows, scripts, settings] = await Promise.all([ + this.#listAllPages((p) => AppService.listApps({ workspace, ...p })), + this.#listAllPages((p) => RawAppService.listRawApps({ workspace, ...p })), + this.#listAllPages((p) => FlowService.listFlows({ workspace, ...p })), + this.#listAllPages((p) => ScriptService.listScripts({ workspace, ...p })), + WorkspaceService.getSettings({ workspace }).catch(() => undefined) + ]) + if (this.#disposed) return + + this.workspaceRateLimit = settings?.public_app_execution_limit_per_minute + + const next: DeployItem[] = [] + const publicApps = apps.filter((a) => a.execution_mode === 'anonymous') + const publicUrls = await Promise.all(publicApps.map((a) => this.#resolvePublicUrl(a.path))) + const publicUrlByPath = new Map(publicApps.map((a, i) => [a.path, publicUrls[i]])) + for (const a of apps) { + const isPublic = a.execution_mode === 'anonymous' + // Raw apps live in the `app` table (value = files/runnables) but must be + // published to the Hub as raw apps, not low-code apps. + const isRaw = (a as any).raw_app === true + next.push({ + key: `${isRaw ? 'raw_app' : 'app'}:${a.path}`, + path: a.path, + kind: isRaw ? 'raw_app' : 'app', + appTable: isRaw || undefined, + summary: a.summary, + rec: 'none', + published: isPublic, + publicUrl: isPublic ? publicUrlByPath.get(a.path) : undefined + }) + } + for (const a of rawApps) { + next.push({ + key: `raw_app:${a.path}`, + path: a.path, + kind: 'raw_app', + summary: a.summary, + rec: 'none' + }) + } + for (const f of flows) { + next.push({ + key: `flow:${f.path}`, + path: f.path, + kind: 'flow', + summary: f.summary, + rec: 'none' + }) + } + for (const s of scripts) { + next.push({ + key: `script:${s.path}`, + path: s.path, + kind: 'script', + summary: s.summary, + rec: 'none' + }) + } + if (this.#disposed) return + this.workspaceItems = next + } catch (e: any) { + if (!this.#disposed) { + sendUserToast(`Failed to load project items: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed) this.loading = false + } + } + + /** Re-fetch triggers, e.g. after the EE license hydrates late. */ + reloadTriggers() { + void this.#loadTriggers() + } + + async #loadTriggers() { + const tok = ++this.#triggerLoadTok + this.triggersLoading = true + try { + const { triggers, failedKinds } = await listAllWorkspaceTriggers(this.workspace, { + includeEeOnly: this.#deps.hasEeLicense(), + onError: (message) => { + if (!this.#disposed) sendUserToast(message, true) + } + }) + if (this.#disposed || tok !== this.#triggerLoadTok) return + this.workspaceTriggers = triggers + this.triggerDiscoveryFailed = failedKinds.length > 0 + } finally { + if (!this.#disposed && tok === this.#triggerLoadTok) this.triggersLoading = false + } + } + + async #resolvePublicUrl(path: string): Promise { + try { + const secret = await AppService.getPublicSecretOfApp({ workspace: this.workspace, path }) + return computeSecretUrl(secret) + } catch { + return undefined + } + } + + async rehydrateFromHub() { + try { + const res = await fetch(`/api/w/${this.workspace}/hub/project${this.#folderQs()}`, { + credentials: 'include', + headers: { accept: 'application/json' } + }) + if (this.#disposed) return + if (!res.ok) return // 404 = no project published for this folder yet + const p = JSON.parse(await res.text()) + if (this.#disposed || !p?.slug) return + this.effectiveSlug = p.slug + this.hubName = p.name ?? '' + this.hubSummary = p.summary ?? '' + this.hubReadme = p.readme ?? '' + this.phase = + p.status === 'live' ? 'live' : p.status === 'under_review' ? 'under_review' : 'draft' + const ids: Record = {} + this.draftItems = (p.items ?? []).map((it: any) => { + const wpath = it.source_path ?? it.path + const key = `${it.kind}:${wpath}` + if (typeof it.hub_id === 'number') ids[key] = it.hub_id + return { + key, + path: wpath, + kind: it.kind as Kind, + summary: it.summary ?? undefined, + rec: it.has_recording ? 'recorded' : 'none' + } satisfies DeployItem + }) + this.hubItemIds = ids + } catch {} + } + + /** Kick off schedule-preview fetches for any relevant schedule trigger missing one. */ + ensureSchedulePreviews() { + for (const t of this.relevantTriggers) { + if (t.kind !== 'schedule') continue + const c = t.config as any + const key = `${c.schedule}|${c.timezone}` + if (this.schedulePreviews[key] || this.#schedulePreviewsInFlight.has(key)) continue + this.#schedulePreviewsInFlight.add(key) + ScheduleService.previewSchedule({ + requestBody: { + schedule: c.schedule, + timezone: c.timezone, + cron_version: c.cron_version ?? 'v2' + } + }) + .then((dates) => { + this.schedulePreviews = { ...this.schedulePreviews, [key]: dates.slice(0, 3) } + }) + .catch(() => {}) + .finally(() => this.#schedulePreviewsInFlight.delete(key)) + } + } + + /** + * Rebuild the predeploy bundle preview (resource + data table dependency + * summaries), debounced so rapid checkbox toggles coalesce into one walk. + * Reads its reactive inputs synchronously and returns a cancel function, so + * it can be driven from an `$effect` with proper cleanup. + */ + queueBundlePreview(): (() => void) | undefined { + if (this.phase !== 'predeploy') { + this.bundlePreview = undefined + this.datatableUsage = new Map() + return undefined + } + this.detectingResources = true + this.detectingDatatables = true + const slug = this.hubSlug + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, slug) + ] + const triggerResources = this.#triggerResourcePaths(this.relevantTriggers) + const triggerVars = this.#triggerVarPaths(this.relevantTriggers) + let cancelled = false + const timer = setTimeout(() => { + buildProjectBundle(seed, slug, this.#cachedBundleDeps(), triggerResources, triggerVars) + .then((b) => { + if (cancelled) return + this.bundlePreview = b + // Detect data table usage off the same fetched items. + detectDatatableTables(b.items) + .then((usage) => { + if (!cancelled) this.datatableUsage = usage + }) + .finally(() => { + if (!cancelled) this.detectingDatatables = false + }) + }) + .finally(() => { + if (!cancelled) this.detectingResources = false + }) + }, 250) + return () => { + cancelled = true + clearTimeout(timer) + } + } + + #buildBundleDeps(): BundleDeps { + const workspace = this.workspace + return { + fetchItem: async (ref: ItemRef): Promise => { + try { + if (ref.kind === 'script') { + const s = await ScriptService.getScriptByPath({ workspace, path: ref.path }) + return { + kind: 'script', + path: ref.path, + summary: s.summary, + description: s.description ?? undefined, + content: s.content, + language: s.language, + schema: s.schema, + lock: s.lock ?? undefined, + scriptKind: typeof s.kind === 'string' ? s.kind.toLowerCase() : 'script' + } + } else if (ref.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace, path: ref.path }) + return { + kind: 'flow', + path: ref.path, + summary: f.summary, + description: f.description ?? undefined, + value: f.value, + schema: f.schema + } + } else if (ref.kind === 'app') { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + return { kind: 'app', path: ref.path, summary: a.summary, value: a.value } + } else if (ref.kind === 'raw_app') { + // Modern raw apps live in the `app` table: fetch source files + + // runnables + the compiled bundle, and shape them into the `raw` + // payload the Hub's RawAppView expects (JSON is valid YAML). + const isModern = this.workspaceItems.some( + (i) => i.kind === 'raw_app' && i.path === ref.path && i.appTable + ) + if (isModern) { + const a = await AppService.getAppByPath({ workspace, path: ref.path }) + const secret = await AppService.getPublicSecretOfLatestVersionOfApp({ + workspace, + path: ref.path + }) + // The compiled JS bundle is required; a missing one means the app + // was never built/deployed, so fail loudly instead of pushing a blank app. + const [jsRes, cssRes] = await Promise.all([ + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.js`, { + credentials: 'include' + }), + fetch(`/api/w/${workspace}/apps/get_data/v/${secret}.css`, { + credentials: 'include' + }) + ]) + if (!jsRes.ok) { + throw new Error(`raw app ${ref.path} has no compiled bundle — deploy it first`) + } + const js = await jsRes.text() + const css = cssRes.ok ? await cssRes.text() : '' + const v: any = a.value ?? {} + const content = JSON.stringify({ + files: { ...(v.files ?? {}), '/bundle.js': js, '/bundle.css': css }, + runnables: v.runnables ?? {}, + // Preserve the full-code app's explicit data table declaration so it + // survives publish/import and feeds migration detection. + ...(v.data !== undefined ? { data: v.data } : {}), + ...(v.datatables !== undefined ? { datatables: v.datatables } : {}) + }) + return { kind: 'raw_app', path: ref.path, summary: a.summary, content } + } + const r = await fetch(`/api/w/${workspace}/raw_apps/get_data/0/${ref.path}`, { + credentials: 'include' + }) + if (!r.ok) return undefined + return { kind: 'raw_app', path: ref.path, content: await r.text() } + } + } catch (e: any) { + return undefined + } + return undefined + }, + resolveResourceType: async (path: string): Promise => { + try { + const r = await ResourceService.getResource({ workspace, path }) + return r.resource_type ?? undefined + } catch (e: any) { + return undefined + } + } + } + } + + #cachedBundleDeps(): BundleDeps { + const deps = this.#buildBundleDeps() + // Memoize only successful lookups: a miss (undefined) is likely transient, so + // evict it once it resolves. Otherwise a fixed/retried dependency can never + // clear `bundlePreview.unresolved` until the whole session is recreated. + const memoize = ( + cache: Map>, + key: string, + run: () => Promise + ) => { + let p = cache.get(key) + if (!p) { + p = run() + cache.set(key, p) + void p.then((r) => { + if (r === undefined && cache.get(key) === p) cache.delete(key) + }) + } + return p + } + return { + fetchItem: (ref) => + memoize(this.#previewItemCache, `${ref.kind}:${ref.path}`, () => deps.fetchItem(ref)), + resolveResourceType: (path) => + memoize(this.#previewTypeCache, path, () => deps.resolveResourceType(path)) + } + } + + async #postHub(path: string, body: unknown): Promise | undefined> { + const res = await fetch(`/api/w/${this.workspace}${path}${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify(body) + }) + const text = await res.text() + if (!res.ok) throw new Error(text) + try { + return JSON.parse(text) + } catch { + return undefined + } + } + + async regenerateMigrations() { + const tok = ++this.#migrationsTok + this.migrationsGenerating = true + try { + // Same handler-augmented seed as deployAll: a data table used only by a + // bundled trigger handler must still get its migration. + const seed: ItemRef[] = [ + ...this.selectedItems + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(this.relevantTriggers, this.hubSlug || 'project') + ] + // Detection is independent of the final slug (data table refs aren't + // relocated), so any placeholder slug works for this throwaway bundle. + const bundle = await buildProjectBundle( + seed, + this.hubSlug || 'project', + this.#buildBundleDeps(), + [] + ) + const usage = await detectDatatableTables(bundle.items) + const drafts = await generateDatatableMigrations(this.workspace, usage) + if (this.#disposed || tok !== this.#migrationsTok) return + this.migrationDrafts = drafts + this.migrationsGeneration++ + } catch (e: any) { + if (!this.#disposed && tok === this.#migrationsTok) { + this.migrationDrafts = [] + this.migrationsGeneration++ + // Toast so a genuine failure isn't mistaken for "no data table usage". + sendUserToast(`Could not generate data table migrations: ${e?.message ?? e}`, true) + } + } finally { + if (!this.#disposed && tok === this.#migrationsTok) this.migrationsGenerating = false + } + } + + /** Prefill bundle metadata and start migration detection (bundle drawer opening). */ + prepareBundle() { + this.hubName = this.hubName || this.folder + void this.regenerateMigrations() + } + + /** + * Create the Hub draft then push the full bundle. `deploying` is set + * synchronously before the first request so a double-click cannot start a + * second publish, and the whole run is refused while triggers are still + * loading — an incomplete `relevantTriggers` snapshot would permanently + * omit triggers (and their handlers and migrations) from the draft. + * `onDraftCreated` fires once the draft exists (the bundle drawer closes + * there while items continue publishing). + */ + async publishBundle(onDraftCreated?: () => void): Promise { + if (this.deploying || this.triggersLoading || this.triggerDiscoveryFailed) return + this.deploying = true + try { + if (!(await this.#createDraft())) return + onDraftCreated?.() + await this.#deployAll() + } finally { + this.deploying = false + } + } + + /** + * Create the Hub draft project. Returns true when the draft exists and + * publishing can proceed. + */ + async #createDraft(): Promise { + this.hubName = this.hubName.trim() + this.hubSummary = this.hubSummary.trim() + this.hubReadme = this.hubReadme.trim() + try { + const res = await fetch(`/api/w/${this.workspace}/hub/publish_draft${this.#folderQs()}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + slug: this.hubSlug, + name: this.hubName, + summary: this.hubSummary || this.hubName, + readme: this.hubReadme || undefined + }) + }) + const text = await res.text() + if (!res.ok) { + sendUserToast(`Hub draft creation failed: ${text}`, true) + return false + } + // Abort if Hub didn't echo a slug — guessing here lands items under + // a folder the Hub never locked. + let returnedSlug: string | undefined + try { + const parsed = JSON.parse(text) + if (typeof parsed?.slug === 'string') returnedSlug = parsed.slug + } catch {} + if (!returnedSlug) { + sendUserToast(`Hub did not return a slug. Aborting publish to avoid path drift.`, true) + return false + } + // Session replaced mid-request (workspace/folder switch): publishing now + // would push another scope's items into this draft. Abort. + if (this.#disposed) { + sendUserToast(`Workspace changed during publish — aborted to avoid mixing items.`, true) + return false + } + this.effectiveSlug = returnedSlug + return true + } catch (e: any) { + sendUserToast(`Hub draft creation failed: ${e?.message ?? e}`, true) + return false + } + } + + async #pushBundledItem(slug: string, it: BundledItem): Promise { + const key = `${it.kind}:${it.path}` + if (it.kind === 'script') { + const resp = await this.#postHub('/hub/scripts', { + summary: it.summary || it.newPath, + app: slug, + description: it.description ?? '', + kind: it.scriptKind ?? 'script', + content: it.content, + language: it.language, + schema: it.schema ?? undefined, + lockfile: it.lock ?? undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'flow') { + const resp = await this.#postHub('/hub/flows', { + flow: { + summary: it.summary || it.newPath, + description: it.description ?? undefined, + value: it.value, + schema: it.schema ?? undefined + }, + apps: [], + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } else if (it.kind === 'app') { + await this.#postHub('/hub/apps', { + app: it.value, + apps: [], + summary: it.summary || it.newPath, + description: undefined, + path: it.newPath, + source_path: it.path, + project_slug: slug + }) + } else if (it.kind === 'raw_app') { + const resp = await this.#postHub('/hub/raw_apps', { + raw: it.content ?? '', + apps: [], + summary: it.summary || it.newPath, + path: it.newPath, + source_path: it.path, + description: undefined, + project_slug: slug + }) + if (typeof resp?.id === 'number') this.hubItemIds = { ...this.hubItemIds, [key]: resp.id } + } + } + + // Handler runnables (trigger error handlers, schedule on_* handlers) ship + // with the bundle like the primary runnables do; hub refs stay external. + #triggerHandlerSeed(triggers: WorkspaceTrigger[], slug: string): ItemRef[] { + return triggers.flatMap(triggerHandlerRefs).filter((r) => classifyPath(r.path, slug) !== 'hub') + } + + // Every resource a trigger's exported config references: the kind-specific + // broker/auth field plus any `$res:` token nested in it (schedule args, + // handler extra args, …) — all must enter the bundle path map. + #triggerResourcePaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + const rp = triggerResourcePath(t) + if (rp) out.add(rp) + for (const p of extractTriggerConfigResourceRefs(portableTriggerConfig(t.kind, t.config))) { + out.add(p) + } + } + return [...out] + } + + // Every whole-string `$var:`/`$jsonvar:` value a trigger's config resolves (SQS + // queue_url, schedule args, …) — relocated through the bundle map like item vars. + #triggerVarPaths(triggers: WorkspaceTrigger[]): string[] { + const out = new Set() + for (const t of triggers) { + for (const p of extractVarRefsFromValue(portableTriggerConfig(t.kind, t.config))) out.add(p) + } + return [...out] + } + + async #pushTriggers( + slug: string, + resourcePathMap: Map, + relevant: WorkspaceTrigger[] + ): Promise { + const pathMap = buildPathMap( + relevant.map((t) => t.path), + slug + ) + const triggers: Array> = [] + const skipped: string[] = [] + for (const t of relevant) { + const itemKind: ItemKind = t.is_flow ? 'flow' : 'script' + const runnableKey = `${itemKind}:${t.script_path}` + const hubId = this.hubItemIds[runnableKey] + if (!hubId) { + skipped.push(t.path) + continue + } + // Full-config remap: resource paths, error-handler paths, schedule on_* + // handler refs and whole-string `$var:` values all relocate through the map. + const config = rewriteVarRefsInValue( + rewriteTriggerConfig(portableTriggerConfig(t.kind, t.config), resourcePathMap), + resourcePathMap + ) + triggers.push({ + path: pathMap.get(t.path) ?? t.path, + kind: t.kind, + summary: t.summary ?? null, + description: (t.config as any)?.description ?? null, + config, + script_ask_id: t.is_flow ? null : hubId, + flow_id: t.is_flow ? hubId : null + }) + } + if (skipped.length > 0) { + sendUserToast( + `Skipped ${skipped.length} trigger(s) whose runnable did not publish: ${skipped.join(', ')}`, + true + ) + } + // Full-set sync: always push (an empty list clears the Hub's triggers on a + // re-deploy), so removing every trigger doesn't leave stale ones on the Hub. + await this.#postHub('/hub/triggers', { triggers, project_slug: slug }) + } + + // Builtin types (git_repository, ...) aren't in resource_type — push with empty schema. + async #pushResourceTypes(slug: string, types: string[]): Promise { + const results = await Promise.all( + types.map(async (name) => { + let schema: unknown = undefined + let description: string | undefined = undefined + try { + const rt = await ResourceService.getResourceType({ + workspace: this.workspace, + path: name + }) + schema = rt.schema ?? undefined + description = rt.description ?? undefined + } catch (e: any) {} + try { + await this.#postHub('/hub/resource_types', { + name, + schema, + description, + project_slug: slug + }) + return 0 + } catch (e: any) { + sendUserToast(`Resource type ${name} push failed: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + return results.reduce((a: number, b) => a + b, 0) + } + + async #deployAll() { + const slug = this.hubSlug + // Snapshot the selection up-front: `selectedItems`/`relevantTriggers` are + // derived from live workspace data and `migrationDrafts` is edited in the + // drawer — the deploy must publish exactly what the user confirmed. + const itemsSnapshot = this.selectedItems.slice() + const triggersSnapshot = this.relevantTriggers.slice() + const migrationsSnapshot = this.migrationDrafts.slice() + this.hubItemIds = {} + this.deploymentStatus = {} + let failures = 0 + try { + const seed: ItemRef[] = [ + ...itemsSnapshot + .filter((i) => i.kind !== 'resource') + .map((i) => ({ kind: i.kind as ItemRef['kind'], path: i.path })), + ...this.#triggerHandlerSeed(triggersSnapshot, slug) + ] + const triggerResources = this.#triggerResourcePaths(triggersSnapshot) + const triggerVars = this.#triggerVarPaths(triggersSnapshot) + const bundle = await buildProjectBundle( + seed, + slug, + this.#buildBundleDeps(), + triggerResources, + triggerVars + ) + // Full path map (incl. unresolved) so a trigger's resource path is always + // relocated — never leaks the publisher's original private path to the Hub. + const resourcePathMap = bundle.pathMap + + // A dangling reference (a selected root or transitive runnable that failed + // to fetch, or a resource whose type can't be resolved) means the bundle + // doesn't close: the root would silently vanish, or a published item would + // still point at the publisher's private source-workspace path. Refuse to + // publish until every reference resolves rather than ship a broken project. + if (bundle.unresolved.length > 0) { + sendUserToast( + `Cannot publish: ${bundle.unresolved.length} unresolved reference(s): ${bundle.unresolved.join(', ')}. Deselect or fix them, then retry.`, + true + ) + return + } + + // Bundle building is slow — bail before the first Hub write if the session + // was replaced (workspace/folder switch) in the meantime. + if (this.#disposed) return + + // Types come from $res: stubs AND schema inputs (resource-). + const inputTypes = bundle.items + .flatMap((i) => typesFromSchema(i.schema)) + .filter((t) => !HIDDEN_RESOURCE_TYPES.has(t)) + const types = [ + ...new Set([...bundle.resourceStubs.map((s) => s.resource_type), ...inputTypes]) + ] + const depFailures = await this.#pushResourceTypes(slug, types) + + // Input-type deps with no path get a conventional f// stub. + const stubsByPath = new Map() + for (const s of bundle.resourceStubs) + stubsByPath.set(s.newPath, { path: s.newPath, resource_type: s.resource_type }) + for (const t of inputTypes) { + const path = `f/${slug}/${t}` + if (!stubsByPath.has(path)) stubsByPath.set(path, { path, resource_type: t }) + } + const stubs = [...stubsByPath.values()] + if (stubs.length > 0) { + try { + await this.#postHub('/hub/resources', { resources: stubs, project_slug: slug }) + } catch (e: any) { + sendUserToast(`Resource sync failed: ${e?.message ?? e}`, true) + failures++ + } + } + failures += depFailures + if (failures > 0) { + sendUserToast( + `Resource dependency sync failed — items not published to avoid broken references.`, + true + ) + return + } + + for (const it of bundle.items) { + // Stop writing item status / Hub IDs once the session is replaced — + // continuing would publish into a project the user has moved away from. + if (this.#disposed) return + const key = `${it.kind}:${it.path}` + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'loading' } } + try { + await this.#pushBundledItem(slug, it) + this.deploymentStatus = { ...this.deploymentStatus, [key]: { status: 'deployed' } } + } catch (e: any) { + failures++ + this.deploymentStatus = { + ...this.deploymentStatus, + [key]: { status: 'failed', error: e?.message ?? String(e) } + } + } + } + // A re-bundle clears the Hub-side embed (idempotent replace), so re-push it + // for any raw app that is already public — keeps the live iframe in sync + // without forcing an unpublish/share round-trip. Updates by hub id, safe in parallel. + const embedResults = await Promise.all( + bundle.items + .filter((it) => it.kind === 'raw_app') + .map(async (it) => { + const hubId = this.hubItemIds[`${it.kind}:${it.path}`] + const src = itemsSnapshot.find((i) => i.kind === 'raw_app' && i.path === it.path) + if (!hubId || !src?.published) return 0 + // The re-bundle cleared the embed; a public raw app with no resolved URL + // can't have its iframe restored, so it's an incomplete publish too — + // count it (like a push failure) so the draft can't become submit-ready. + if (!src.publicUrl) { + sendUserToast(`Cannot restore the iframe for ${it.path}: missing public URL`, true) + return 1 + } + try { + await this.#pushRawAppEmbed(hubId, src.publicUrl) + return 0 + } catch (e: any) { + sendUserToast(`Failed to sync iframe for ${it.path}: ${e?.message ?? e}`, true) + return 1 + } + }) + ) + failures += embedResults.reduce((a: number, b) => a + b, 0) + if (this.#disposed) return + try { + await this.#pushTriggers(slug, resourcePathMap, triggersSnapshot) + } catch (e: any) { + sendUserToast(`Trigger sync failed: ${e?.message ?? e}`, true) + failures++ + } + + // Full-set sync: always push (an empty list clears the Hub's migrations on + // a re-deploy). The Hub drops empty-SQL entries, so disabled placeholders + // don't persist. + try { + await this.#postHub('/hub/migrations', { + migrations: migrationsSnapshot.map((m) => ({ + datatable_name: m.datatable_name, + sql: m.sql, + sql_down: m.sql_down, + enabled: m.enabled + })), + project_slug: slug + }) + } catch (e: any) { + sendUserToast(`Data table migration sync failed: ${e?.message ?? e}`, true) + failures++ + } + + await sleep(150) + if (this.#disposed) return + // An incomplete push must never become submittable: a failed transitive item + // can leave a pushed runnable pointing at content that never landed. Stay in + // predeploy (deploymentStatus keeps the failed items visible) so re-publishing + // retries every write — createDraft and the item pushes are idempotent. + if (failures > 0) { + sendUserToast( + `Publish incomplete: ${failures} write(s) failed. Nothing was submitted — fix them and re-publish.`, + true + ) + return + } + this.deploymentStatus = {} + this.recordings = {} + // Deterministic baseline so a transient Hub read failure can't leave the + // UI stuck in `predeploy`; rehydrate then upgrades to authoritative state. + this.draftItems = itemsSnapshot.map((i) => ({ ...i, rec: 'none' })) + this.phase = 'draft' + await this.rehydrateFromHub() + sendUserToast(`Draft created on the Hub. Add recordings before submitting for review.`) + } finally { + this.deploying = false + } + } + + submitForReview = async () => { + const slug = this.hubSlug + if (!slug) return + this.submitting = true + try { + const res = await fetch( + `/api/w/${this.workspace}/hub/projects/${encodeURIComponent(slug)}/submit${this.#folderQs()}`, + { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: '{}' + } + ) + if (!res.ok) { + sendUserToast(`Submit for review failed: ${await res.text()}`, true) + return + } + this.phase = 'under_review' + sendUserToast('Submitted for review by the Windmill team.') + } finally { + this.submitting = false + } + } + + syncWithHub = async () => { + this.syncing = true + try { + if (this.phase === 'draft') { + await this.#loadWorkspace() + const prev = new Map(this.draftItems.map((i) => [i.key, { rec: i.rec }])) + this.draftItems = this.workspaceItems + .filter((i) => prev.has(i.key)) + .map((i) => ({ ...i, rec: prev.get(i.key)?.rec ?? 'none' })) + } else { + // under_review / live: re-fetch the Hub project to pick up an + // admin status change (under_review -> live). + const before = this.phase + await this.rehydrateFromHub() + sendUserToast( + this.phase === before + ? 'Still waiting for review.' + : this.phase === 'live' + ? 'Approved — your project is now live.' + : `Status updated: ${this.phase}.` + ) + } + } catch (e: any) { + sendUserToast(`Sync failed: ${e?.message ?? e}`, true) + } finally { + this.syncing = false + } + } + + startNewDraft = () => { + this.draftItems = [] + this.recordings = {} + this.phase = 'predeploy' + } + + /** Reset record-drawer state and load the target's schema. */ + async openRecord(it: DeployItem) { + const tok = ++this.#recordRunTok + this.recordTarget = it + this.recordArgs = {} + this.recordValid = true + this.recordSchema = emptySchema() + this.recordSchemaLoading = true + this.runState = 'idle' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + if (it.kind === 'script') { + const s = await ScriptService.getScriptByPath({ + workspace: this.workspace, + path: it.path + }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (s.schema as Record) ?? emptySchema() + } else if (it.kind === 'flow') { + const f = await FlowService.getFlowByPath({ workspace: this.workspace, path: it.path }) + if (tok !== this.#recordRunTok) return + this.recordSchema = (f.schema as Record) ?? emptySchema() + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + sendUserToast(`Failed to load schema: ${e?.message ?? e}`, true) + } finally { + if (tok === this.#recordRunTok) this.recordSchemaLoading = false + } + } + + /** Invalidate any in-flight record run/poll (record drawer closed). */ + cancelRecordRun = () => { + this.#recordRunTok++ + } + + runJob = async () => { + const it = this.recordTarget + if (!it) return + const tok = ++this.#recordRunTok + this.runState = 'running' + this.runJobId = undefined + this.runResult = undefined + this.runError = undefined + try { + let jobId: string + if (it.kind === 'script') { + jobId = await JobService.runScriptByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else if (it.kind === 'flow') { + jobId = await JobService.runFlowByPath({ + workspace: this.workspace, + path: it.path, + requestBody: this.recordArgs + }) + } else { + if (tok === this.#recordRunTok) this.runState = 'idle' + return + } + if (tok !== this.#recordRunTok) return + this.runJobId = jobId + await this.#pollJobUntilComplete(jobId, tok) + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Failed to start: ${e?.message ?? e}` + } + } + + async #pollJobUntilComplete(jobId: string, tok: number) { + // First check immediately (fast scripts complete in ms), then back off to 2s. + const deadline = Date.now() + 5 * 60_000 + let interval = 250 + while (Date.now() < deadline) { + if (tok !== this.#recordRunTok) return + try { + const r = await JobService.getCompletedJobResultMaybe({ + workspace: this.workspace, + id: jobId + }) + if (tok !== this.#recordRunTok) return + if (r.completed) { + this.runResult = r.result + if (r.success) { + this.runState = 'success' + } else { + this.runState = 'failed' + this.runError = typeof r.result === 'string' ? r.result : JSON.stringify(r.result) + } + return + } + } catch (e: any) { + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = `Polling failed: ${e?.message ?? e}` + return + } + await sleep(interval) + interval = Math.min(interval * 2, 2000) + } + if (tok !== this.#recordRunTok) return + this.runState = 'failed' + this.runError = 'Timed out after 5 minutes' + } + + async #buildScriptRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const s = await ScriptService.getScriptByPath({ workspace, path: it.path }) + const job = await JobService.getCompletedJob({ workspace, id: jobId }) + const initial_job = { ...(job as any), type: 'CompletedJob' } + const events = [{ t: 0, data: { completed: true, job: initial_job } }] + const duration = (initial_job.duration_ms as number) ?? 0 + return { + version: 1, + type: 'script' as const, + recorded_at: new Date().toISOString(), + script_path: it.path, + total_duration_ms: duration, + code: s.content, + language: s.language, + args: (job.args ?? {}) as Record, + schema: s.schema, + job: { initial_job, events } + } + } + + async #buildFlowRecording(it: DeployItem, jobId: string) { + const workspace = this.workspace + const f = await FlowService.getFlowByPath({ workspace, path: it.path }) + const root = (await JobService.getCompletedJob({ workspace, id: jobId })) as any + const jobs: Record = {} + const collect = async (j: any) => { + const stamped = { ...j, type: 'CompletedJob' } + jobs[j.id] = { + initial_job: stamped, + events: [{ t: 0, data: { completed: true, job: stamped } }] + } + const modules = (j.flow_status?.modules ?? []).filter( + (m: any) => m.job && typeof m.job === 'string' + ) + // Sub-jobs at the same level are independent reads. + await Promise.all( + modules.map(async (m: any) => { + try { + const sub = (await JobService.getCompletedJob({ workspace, id: m.job })) as any + await collect(sub) + } catch { + /* sub-job missing — skip */ + } + }) + ) + } + await collect(root) + return { + version: 1, + recorded_at: new Date().toISOString(), + flow_path: it.path, + total_duration_ms: (root.duration_ms as number) ?? 0, + flow: { + path: it.path, + value: f.value, + schema: f.schema ?? { type: 'object', properties: {}, required: [] }, + summary: f.summary ?? '', + archived: false, + edited_at: '', + edited_by: '', + extra_perms: {} + }, + jobs + } + } + + /** Save the current successful run as the Hub recording. Returns true on success. */ + async saveRecording(): Promise { + const it = this.recordTarget + if (!it || !this.runJobId || this.runState !== 'success') return false + const hubId = this.hubItemIds[it.key] + if (!hubId) { + sendUserToast(`Push the bundle to the Hub first before saving recordings`, true) + return false + } + if (it.kind !== 'script' && it.kind !== 'flow') { + sendUserToast(`Recordings only supported for script/flow`, true) + return false + } + try { + const recording = + it.kind === 'script' + ? await this.#buildScriptRecording(it, this.runJobId) + : await this.#buildFlowRecording(it, this.runJobId) + const path = it.kind === 'script' ? 'scripts' : 'flows' + await this.#postHub(`/hub/${path}/${hubId}/recording`, { + recording, + project_slug: this.hubSlug + }) + this.recordings = { ...this.recordings, [it.key]: this.runJobId } + this.#patchItem(it.key, { rec: 'recorded' }) + sendUserToast(`Recording saved — job ${this.runJobId}`) + return true + } catch (e: any) { + sendUserToast(`Failed to save recording: ${e?.message ?? e}`, true) + return false + } + } + + // Set the Hub raw app's live-iframe URL (or clear it with null). The Hub renders + // from external_embed_url; project_slug scopes ownership. + async #pushRawAppEmbed(hubId: number, url: string | null) { + await this.#postHub(`/hub/raw_apps/${hubId}/embed`, { + external_embed_url: url, + project_slug: this.hubSlug + }) + } + + // Flip an app/raw app between public (anonymous) and private (publisher) and keep + // the Hub raw-app iframe in sync. Returns the resolved public URL when shared. + async #setAppShared(it: DeployItem, shared: boolean): Promise { + const workspace = this.workspace + const hubId = it.kind === 'raw_app' ? this.hubItemIds[it.key] : undefined + // Sharing a raw app as an iframe needs its Hub item to wire the embed. Fail + // before flipping the app public so it can't be left anonymous with no embed. + if (shared && it.kind === 'raw_app' && !hubId) { + throw new Error('Push the bundle to the Hub first to share the live iframe') + } + const app = await AppService.getAppByPath({ workspace, path: it.path }) + const prevMode = (app.policy?.execution_mode ?? 'publisher') as 'anonymous' | 'publisher' + const nextMode = (shared ? 'anonymous' : 'publisher') as 'anonymous' | 'publisher' + const setMode = (mode: 'anonymous' | 'publisher', message: string) => + AppService.updateApp({ + workspace, + path: it.path, + requestBody: { + policy: { ...(app.policy ?? {}), execution_mode: mode }, + deployment_message: message + } + }) + // Undo the policy flip so the app's public state stays consistent when a later + // step of the share fails. Best-effort: a revert failure must not mask the cause. + const rollback = () => setMode(prevMode, 'Revert iframe share').catch(() => {}) + await setMode(nextMode, shared ? 'Share as iframe' : 'Unshare iframe') + const url = shared ? ((await this.#resolvePublicUrl(it.path)) ?? null) : null + // A share with no resolvable public URL is incomplete (no embeddable link, no + // Unpublish control); don't leave the app anonymous while reporting success. + if (shared && url === null) { + await rollback() + throw new Error(`Could not resolve the public URL for ${it.path}`) + } + if (hubId && it.kind === 'raw_app' && (!shared || url)) { + try { + await this.#pushRawAppEmbed(hubId, shared ? url : null) + } catch (e) { + await rollback() + throw e + } + } + return url + } + + /** Make the publish target public. Returns true on success. */ + async confirmPublish(): Promise { + const it = this.publishTarget + if (!it || !canShareAsIframe(it)) return false + this.publishing = true + try { + const url = await this.#setAppShared(it, true) + this.#patchItem(it.key, { published: true, publicUrl: url ?? undefined }) + sendUserToast(`${it.path} is now public`) + return true + } catch (e: any) { + sendUserToast(`Failed to publish: ${e?.message ?? e}`, true) + return false + } finally { + this.publishing = false + } + } + + unpublishApp = async (it: DeployItem) => { + if (!canShareAsIframe(it)) return + try { + await this.#setAppShared(it, false) + this.#patchItem(it.key, { published: false, publicUrl: undefined }) + sendUserToast('App unpublished') + } catch (e: any) { + sendUserToast(`Failed to unpublish: ${e?.message ?? e}`, true) + } + } +} + +/** + * Owns the session lifecycle: a new `DeployToHubSession` is created whenever the + * (workspace, folder) identity actually changes — a spurious same-value store + * emit reuses the live session — and the previous one is disposed, which is the + * single mechanism invalidating its in-flight work. Also hosts the reactive + * plumbing the session itself can't (license-hydration reload, schedule + * previews, debounced bundle preview). + */ +export function useDeployToHubSession(args: { + workspace: () => string | undefined + folder: () => string + hasEeLicense: () => boolean +}) { + let session = $state() + + $effect(() => { + const workspace = args.workspace() + const folder = args.folder() + if (!workspace) return + untrack(() => { + if (session && session.workspace === workspace && session.folder === folder) return + session?.dispose() + const next = new DeployToHubSession(workspace, folder, { + hasEeLicense: args.hasEeLicense + }) + session = next + next.load() + }) + }) + + // The EE license hydrates async; if it lands after a license-less trigger load, + // EE kinds stay empty. Re-fetch on false→true (the session reads the license + // getter at call time). + let prevHadLicense: boolean | undefined = undefined + $effect(() => { + const hasLicense = args.hasEeLicense() + untrack(() => { + if (hasLicense && prevHadLicense === false) session?.reloadTriggers() + prevHadLicense = hasLicense + }) + }) + + // Leaving/entering predeploy invalidates manual selection tweaks. + $effect(() => { + const s = session + if (!s) return + s.phase + untrack(() => { + s.manualDeselected = new Set() + }) + }) + + // Schedule previews for relevant schedule triggers (deduped in the session). + $effect(() => { + session?.ensureSchedulePreviews() + }) + + // Debounced predeploy bundle preview; the session reads its reactive inputs + // synchronously and returns the cancel function used as effect cleanup. + $effect(() => { + const s = session + if (!s) return + return s.queueBundlePreview() + }) + + return { + get session() { + return session + } + } +} diff --git a/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts new file mode 100644 index 0000000000..7b61251fcc --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/deployToHubSession.test.ts @@ -0,0 +1,54 @@ +import { describe, it, expect } from 'vitest' +import { canShareAsIframe, mergeShareState, type DeployItem } from './deployToHubSession.svelte' + +function item(over: Partial & Pick): DeployItem { + return { rec: 'none', ...over } +} + +describe('canShareAsIframe', () => { + it('allows low-code apps and app-table raw apps', () => { + expect(canShareAsIframe(item({ key: 'app:f/a', path: 'f/a', kind: 'app' }))).toBe(true) + expect( + canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })) + ).toBe(true) + }) + it('hides the action for legacy raw apps (raw_app table only)', () => { + // Legacy entries from RawAppService carry no appTable flag; AppService can't load them. + expect(canShareAsIframe(item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' }))).toBe(false) + }) + it('never offers the action for flows or scripts', () => { + expect(canShareAsIframe(item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' }))).toBe(false) + }) +}) + +describe('mergeShareState', () => { + it('carries live public-share state from workspace items onto matching drafts', () => { + const drafts = [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })] + const workspace = [ + item({ + key: 'app:f/a', + path: 'f/a', + kind: 'app', + published: true, + publicUrl: 'https://x/app' + }) + ] + const merged = mergeShareState(drafts, workspace) + expect(merged[0].published).toBe(true) + expect(merged[0].publicUrl).toBe('https://x/app') + }) + it('restores the app-table origin so app-table raw apps stay shareable', () => { + const drafts = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app' })] + const workspace = [item({ key: 'raw_app:f/r', path: 'f/r', kind: 'raw_app', appTable: true })] + expect(canShareAsIframe(mergeShareState(drafts, workspace)[0])).toBe(true) + }) + it('returns the same reference when nothing changes', () => { + const drafts = [item({ key: 'flow:f/f', path: 'f/f', kind: 'flow' })] + expect(mergeShareState(drafts, drafts)).toBe(drafts) + }) + it('leaves drafts without a workspace match untouched', () => { + const drafts = [item({ key: 'app:f/gone', path: 'f/gone', kind: 'app' })] + const merged = mergeShareState(drafts, [item({ key: 'app:f/a', path: 'f/a', kind: 'app' })]) + expect(merged).toBe(drafts) + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts new file mode 100644 index 0000000000..669c0a43de --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.test.ts @@ -0,0 +1,869 @@ +import { describe, it, expect } from 'vitest' +import { + classifyPath, + extractScriptRefs, + extractFlowRefs, + extractAppRefs, + buildPathMap, + rewriteContent, + rewriteTriggerConfig, + rewriteFlowValue, + rewriteAppValue, + extractRawAppRefs, + rewriteRawAppContent, + buildProjectBundle, + retargetProjectExport, + collectExportVarPaths, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + type ProjectExport, + type FetchedItem, + type ItemRef +} from './projectBundle' + +describe('classifyPath', () => { + it('internal for paths under the project folder', () => { + expect(classifyPath('f/proj/db', 'proj')).toBe('internal') + expect(classifyPath('f/proj', 'proj')).toBe('internal') + }) + it('hub for hub paths', () => { + expect(classifyPath('hub/16043/discord/send', 'proj')).toBe('hub') + }) + it('external for user and other folders', () => { + expect(classifyPath('u/admin/db', 'proj')).toBe('external') + expect(classifyPath('f/other/db', 'proj')).toBe('external') + }) + it('does not treat a prefix-only match as internal', () => { + expect(classifyPath('f/project2/db', 'proj')).toBe('external') + }) +}) + +describe('extractScriptRefs', () => { + it('finds $res: and res:// resource refs, deduped', () => { + const c = `const a = "$res:u/admin/db"; const b = "res://f/x/api"; const c2 = "$res:u/admin/db"` + expect(extractScriptRefs(c)).toEqual([ + { kind: 'resource', path: 'u/admin/db' }, + { kind: 'resource', path: 'f/x/api' } + ]) + }) + it('returns nothing when no refs', () => { + expect(extractScriptRefs('export async function main() {}')).toEqual([]) + }) +}) + +describe('extractFlowRefs', () => { + it('finds inline-code, static-input, and script-path refs', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { + other: { type: 'static', value: '$res:f/shared/api' }, + expr1: { type: 'javascript', expr: 'flow_input.x' } + } + } + }, + { + id: 'b', + value: { + type: 'branchone', + branches: [ + { + modules: [ + { id: 'c', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'd', value: { type: 'script', path: 'hub/123/x/y' } } + ] + } + ], + default: [{ id: 'e', value: { type: 'rawscript', content: 'no refs' } }] + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'f/shared/api' }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/my_script' }) + expect(refs).toContainEqual({ kind: 'script', path: 'hub/123/x/y' }) + // a javascript expr (flow_input) is not a hardcoded ref + expect(refs.filter((r) => r.path === 'flow_input.x')).toEqual([]) + }) + it('finds sub-flow refs from type: flow steps', () => { + const value = { + modules: [ + { id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }, + { id: 'b', value: { type: 'flow', path: 'hub/9/x/y' } } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sub_flow' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'hub/9/x/y' }) + }) +}) + +describe('buildPathMap', () => { + it('reparents into the project folder keeping the leaf name', () => { + const m = buildPathMap(['u/admin/db', 'f/other/api'], 'proj') + expect(m.get('u/admin/db')).toBe('f/proj/db') + expect(m.get('f/other/api')).toBe('f/proj/api') + }) + it('suffixes collisions deterministically', () => { + const m = buildPathMap(['u/alice/db', 'f/shared/db', 'u/bob/db'], 'proj') + // sorted: f/shared/db, u/alice/db, u/bob/db + expect(m.get('f/shared/db')).toBe('f/proj/db') + expect(m.get('u/alice/db')).toBe('f/proj/db_2') + expect(m.get('u/bob/db')).toBe('f/proj/db_3') + }) + it('maps internal paths to themselves, preserving subfolder depth', () => { + const m = buildPathMap(['f/proj/api', 'f/proj/sub/deep/script'], 'proj') + expect(m.get('f/proj/api')).toBe('f/proj/api') + expect(m.get('f/proj/sub/deep/script')).toBe('f/proj/sub/deep/script') + }) + it('does not flatten two internal items sharing a leaf name', () => { + const m = buildPathMap(['f/proj/a/x', 'f/proj/b/x'], 'proj') + expect(m.get('f/proj/a/x')).toBe('f/proj/a/x') + expect(m.get('f/proj/b/x')).toBe('f/proj/b/x') + }) + it('relocates an external onto a suffix when its leaf collides with an internal path', () => { + const m = buildPathMap(['f/proj/db', 'u/admin/db'], 'proj') + expect(m.get('f/proj/db')).toBe('f/proj/db') + expect(m.get('u/admin/db')).toBe('f/proj/db_2') + }) +}) + +describe('rewriteContent', () => { + it('rewrites mapped refs and leaves unmapped ones', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + expect(rewriteContent('x = "$res:u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "res://u/admin/db"', map)).toBe('x = "$res:f/proj/db"') + expect(rewriteContent('x = "$res:hub/1/a/b"', map)).toBe('x = "$res:hub/1/a/b"') + }) + it('does not partial-match a longer path', () => { + const map = new Map([['u/admin/db', 'f/proj/db']]) + // u/admin/db2 must not be rewritten by the u/admin/db entry + expect(rewriteContent('x = "$res:u/admin/db2"', map)).toBe('x = "$res:u/admin/db2"') + }) +}) + +describe('rewriteTriggerConfig', () => { + const map = new Map([ + ['f/proj/kafka', 'f/target/kafka'], + ['f/proj/script', 'f/target/script'] + ]) + it('remaps plain resource path fields', () => { + expect( + rewriteTriggerConfig({ kafka_resource_path: 'f/proj/kafka', group_id: 'g1' }, map) + ).toEqual({ kafka_resource_path: 'f/target/kafka', group_id: 'g1' }) + }) + it('remaps nested objects, arrays, and $res: tokens', () => { + expect( + rewriteTriggerConfig( + { + nested: { path: 'f/proj/script' }, + list: ['f/proj/kafka', 'unrelated'], + code: 'x = "$res:f/proj/kafka"' + }, + map + ) + ).toEqual({ + nested: { path: 'f/target/script' }, + list: ['f/target/kafka', 'unrelated'], + code: 'x = "$res:f/target/kafka"' + }) + }) + it('leaves non-matching strings and non-string values untouched', () => { + const config = { url: 'wss://example.com', port: 9092, enabled: true, extra: null } + expect(rewriteTriggerConfig(config, map)).toEqual(config) + }) +}) + +describe('rewriteFlowValue', () => { + it('rewrites inline code, static inputs, and script paths; clones input', () => { + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['f/shared/api', 'f/proj/api'], + ['u/admin/my_script', 'f/proj/my_script'] + ]) + const value = { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + content: 'const db = "$res:u/admin/pg"', + input_transforms: { other: { type: 'static', value: '$res:f/shared/api' } } + } + }, + { id: 'b', value: { type: 'script', path: 'u/admin/my_script' } }, + { id: 'c', value: { type: 'script', path: 'hub/1/keep/me' } } + ] + } + const out = rewriteFlowValue(value, map) + expect(out.modules[0].value.content).toBe('const db = "$res:f/proj/pg"') + expect(out.modules[0].value.input_transforms.other.value).toBe('$res:f/proj/api') + expect(out.modules[1].value.path).toBe('f/proj/my_script') + expect(out.modules[2].value.path).toBe('hub/1/keep/me') + // original untouched (deep clone) + expect(value.modules[0].value.content).toBe('const db = "$res:u/admin/pg"') + }) +}) + +// A trimmed app value: a runnable-by-path component, a hub runnable, a $res in an +// inline script, and incidental `f/...` text that must NOT be rewritten. +const appValue = () => ({ + grid: [ + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'script', path: 'u/admin/charts' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'flow', path: 'f/shared/sync' } + } + } + }, + { + data: { + componentInput: { + runnable: { type: 'runnableByPath', runType: 'hubscript', path: 'hub/1/keep' } + } + } + } + ], + hiddenInlineScripts: [ + { name: 'h', inlineScript: { content: 'x = "$res:u/admin/pg"', language: 'deno' } } + ], + someLabel: 'see docs at f/shared/sync for details' +}) + +describe('extractAppRefs', () => { + it('extracts runnable-by-path scripts/flows and $res resources, skips hub', () => { + const refs = extractAppRefs(appValue()) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/charts' }) + expect(refs).toContainEqual({ kind: 'flow', path: 'f/shared/sync' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) +}) + +describe('rewriteAppValue', () => { + it('relocates runnable paths and $res, leaves hub refs and incidental text intact', () => { + const map = new Map([ + ['u/admin/charts', 'f/proj/charts'], + ['f/shared/sync', 'f/proj/sync'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const value = appValue() + const out = rewriteAppValue(value, map) + expect(out.grid[0].data.componentInput.runnable.path).toBe('f/proj/charts') + expect(out.grid[1].data.componentInput.runnable.path).toBe('f/proj/sync') + expect(out.grid[2].data.componentInput.runnable.path).toBe('hub/1/keep') + expect(out.hiddenInlineScripts[0].inlineScript.content).toBe('x = "$res:f/proj/pg"') + // incidental text untouched + expect(out.someLabel).toBe('see docs at f/shared/sync for details') + // original untouched (deep clone) + expect(value.grid[0].data.componentInput.runnable.path).toBe('u/admin/charts') + }) +}) + +describe('raw app (value.raw JSON string)', () => { + const rawContent = () => + JSON.stringify({ + runnables: { + a: { type: 'path', runType: 'flow', path: 'u/admin/sync' }, + b: { type: 'path', runType: 'script', path: 'f/shared/calc' }, + c: { type: 'path', runType: 'hubscript', path: 'hub/1/keep' } + }, + files: { '/bundle.js': 'const conn = "$res:u/admin/pg"' } + }) + + it('extractRawAppRefs sees nested runnables and $res, skips hub', () => { + const refs = extractRawAppRefs(rawContent()) + expect(refs).toContainEqual({ kind: 'flow', path: 'u/admin/sync' }) + expect(refs).toContainEqual({ kind: 'script', path: 'f/shared/calc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs.some((r) => r.path === 'hub/1/keep')).toBe(false) + }) + + it('rewriteRawAppContent relocates nested runnable paths and $res', () => { + const map = new Map([ + ['u/admin/sync', 'f/proj/sync'], + ['f/shared/calc', 'f/proj/calc'], + ['u/admin/pg', 'f/proj/pg'] + ]) + const out = JSON.parse(rewriteRawAppContent(rawContent(), map)) + expect(out.runnables.a.path).toBe('f/proj/sync') + expect(out.runnables.b.path).toBe('f/proj/calc') + expect(out.runnables.c.path).toBe('hub/1/keep') + expect(out.files['/bundle.js']).toBe('const conn = "$res:f/proj/pg"') + }) + + it('falls back to $res scan on non-JSON content', () => { + expect(extractRawAppRefs('x = "$res:u/admin/pg"')).toContainEqual({ + kind: 'resource', + path: 'u/admin/pg' + }) + expect( + rewriteRawAppContent('x = "$res:u/admin/pg"', new Map([['u/admin/pg', 'f/proj/pg']])) + ).toBe('x = "$res:f/proj/pg"') + }) +}) + +describe('buildProjectBundle', () => { + // A flow that calls an external script which itself hardcodes a resource. + const flow: FetchedItem = { + kind: 'flow', + path: 'u/admin/my_flow', + summary: 'Flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/helper' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:f/shared/api"', + input_transforms: {} + } + } + ] + } + } + const helper: FetchedItem = { + kind: 'script', + path: 'u/admin/helper', + summary: 'Helper', + language: 'bun', + content: 'const db = "$res:u/admin/pg"; export async function main(){}' + } + + const deps = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/my_flow') return flow + if (ref.path === 'u/admin/helper') return helper + return undefined + }, + resolveResourceType: async (path: string) => { + if (path === 'u/admin/pg') return 'postgresql' + if (path === 'f/shared/api') return 'http_api' + return undefined + } + } + + it('pulls in referenced scripts + resources and rewrites everything under the folder', async () => { + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/my_flow' }], + 'proj', + deps + ) + + // flow + transitively-pulled helper script are both bundled + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + expect(Object.keys(byPath).sort()).toEqual(['u/admin/helper', 'u/admin/my_flow']) + + // items relocated under f/proj/ + expect(byPath['u/admin/my_flow'].newPath).toBe('f/proj/my_flow') + expect(byPath['u/admin/helper'].newPath).toBe('f/proj/helper') + + // flow's script-path ref rewritten to the helper's new path + expect(byPath['u/admin/my_flow'].value.modules[0].value.path).toBe('f/proj/helper') + // flow inline + helper code resource refs rewritten + expect(byPath['u/admin/my_flow'].value.modules[1].value.content).toBe( + 'const x = "$res:f/proj/api"' + ) + expect(byPath['u/admin/helper'].content).toContain('"$res:f/proj/pg"') + + // resource stubs created at new paths with resolved types + const stubs = Object.fromEntries(bundle.resourceStubs.map((s) => [s.originalPath, s])) + expect(stubs['u/admin/pg'].newPath).toBe('f/proj/pg') + expect(stubs['u/admin/pg'].resource_type).toBe('postgresql') + expect(stubs['f/shared/api'].resource_type).toBe('http_api') + + expect(bundle.unresolved).toEqual([]) + }) + + it('pulls in a sub-flow referenced by a type: flow step and rewrites its path', async () => { + const parent: FetchedItem = { + kind: 'flow', + path: 'u/admin/parent_flow', + value: { modules: [{ id: 'a', value: { type: 'flow', path: 'u/admin/sub_flow' } }] } + } + const sub: FetchedItem = { + kind: 'flow', + path: 'u/admin/sub_flow', + value: { + modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/keep/me' } }] + } + } + const d = { + fetchItem: async (ref: ItemRef) => { + if (ref.path === 'u/admin/parent_flow') return parent + if (ref.path === 'u/admin/sub_flow') return sub + return undefined + }, + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'u/admin/parent_flow' }], + 'proj', + d + ) + const byPath = Object.fromEntries(bundle.items.map((i) => [i.path, i])) + // both flows bundled + expect(Object.keys(byPath).sort()).toEqual(['u/admin/parent_flow', 'u/admin/sub_flow']) + // parent's type: flow ref rewritten to the sub-flow's new path + expect(byPath['u/admin/parent_flow'].value.modules[0].value.path).toBe('f/proj/sub_flow') + expect(byPath['u/admin/sub_flow'].newPath).toBe('f/proj/sub_flow') + // hub ref inside the sub-flow left untouched + expect(byPath['u/admin/sub_flow'].value.modules[0].value.path).toBe('hub/1/keep/me') + expect(bundle.unresolved).toEqual([]) + }) + + it('leaves hub script references untouched and does not fetch them', async () => { + const hubFlow: FetchedItem = { + kind: 'flow', + path: 'u/admin/hub_flow', + value: { modules: [{ id: 'a', value: { type: 'script', path: 'hub/1/x/y' } }] } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/hub_flow' ? hubFlow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/hub_flow' }], 'proj', d) + expect(bundle.items.map((i) => i.path)).toEqual(['u/admin/hub_flow']) + expect(bundle.items[0].value.modules[0].value.path).toBe('hub/1/x/y') + expect(bundle.unresolved).toEqual([]) + }) + + it('reports a missing item and an unresolvable resource as unresolved', async () => { + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/gone' } }, + { + id: 'b', + value: { + type: 'rawscript', + content: 'const x = "$res:u/admin/untyped"', + input_transforms: {} + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved.sort()).toEqual(['u/admin/gone', 'u/admin/untyped']) + }) + + it('relocates $var:/$jsonvar: refs into the slug when it differs from the source folder', async () => { + const flow: FetchedItem = { + kind: 'flow', + path: 'f/source_folder/main', + value: { + flow_env: { CFG: '$jsonvar:f/source_folder/cfg' }, + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + // Whole-value ref is relocated; the inline literal is not. + content: 'return "$var:f/source_folder/key"', + input_transforms: { k: { type: 'static', value: '$var:f/source_folder/key' } } + } + } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'f/source_folder/main' ? flow : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle( + [{ kind: 'flow', path: 'f/source_folder/main' }], + 'kit', + d + ) + const v = bundle.items[0].value + expect(v.modules[0].value.input_transforms.k.value).toBe('$var:f/kit/key') + expect(v.flow_env.CFG).toBe('$jsonvar:f/kit/cfg') + // Inline code literal is untouched. + expect(v.modules[0].value.content).toBe('return "$var:f/source_folder/key"') + }) + + it('dedupes a path missing as both a script and a flow', async () => { + // A missing script + flow sharing a path each push the bare path once; the + // list must stay unique so a keyed UI render of it can't collide. + const root: FetchedItem = { + kind: 'flow', + path: 'u/admin/root', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'u/admin/dup' } }, + { id: 'b', value: { type: 'flow', path: 'u/admin/dup' } } + ] + } + } + const d = { + fetchItem: async (ref: ItemRef) => (ref.path === 'u/admin/root' ? root : undefined), + resolveResourceType: async () => undefined + } + const bundle = await buildProjectBundle([{ kind: 'flow', path: 'u/admin/root' }], 'proj', d) + expect(bundle.unresolved).toEqual(['u/admin/dup']) + }) +}) + +describe('extractVarRefsFromValue', () => { + it('collects whole-value `$var:`/`$jsonvar:` refs, deduped, walking nested JSON', () => { + const value = { + flow_env: { API: '$var:u/admin/key' }, + modules: [ + { value: { input_transforms: { a: { type: 'static', value: '$var:f/proj/token' } } } }, + { value: { input_transforms: { b: { type: 'static', value: '$jsonvar:u/admin/cfg' } } } }, + { value: { input_transforms: { c: { type: 'static', value: '$var:u/admin/key' } } } } + ] + } + expect(extractVarRefsFromValue(value).sort()).toEqual([ + 'f/proj/token', + 'u/admin/cfg', + 'u/admin/key' + ]) + }) + it('ignores a `$var:` token embedded in inline code (not a whole value)', () => { + // The worker only substitutes a value that *is* the reference, so an inline + // script literal must not be treated as a variable arg. + const value = { + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/example/template"' } }] + } + expect(extractVarRefsFromValue(value)).toEqual([]) + }) +}) + +describe('retargetProjectExport', () => { + const baseExport = (): ProjectExport => ({ + project: { slug: 'proj', name: 'Proj', summary: '', readme: null }, + scripts: [ + { + path: 'f/proj/hello', + content: 'const r = "$res:f/proj/db"', + summary: 'hello' + } + ], + flows: [ + { + path: 'f/proj/main_flow', + value: { + modules: [ + { id: 'a', value: { type: 'script', path: 'f/proj/hello', input_transforms: {} } } + ] + } + } + ], + apps: [ + { + path: 'f/proj/dashboard', + value: { grid: [{ data: { componentInput: { runnable: {} } } }] } + }, + { + path: 'f/proj/rawapp', + app_type: 'raw', + value: { raw: JSON.stringify({ files: {}, runnables: {} }) } + } + ], + resources: [{ path: 'f/proj/db', resource_type: 'postgresql' }], + triggers: [ + { + path: 'f/proj/every_day', + kind: 'schedule', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { schedule: '0 0 12 * * *' } + }, + { + path: 'f/proj/kafka_in', + kind: 'kafka', + runnable_path: 'f/proj/hello', + runnable_kind: 'script', + config: { kafka_resource_path: 'f/proj/db' } + } + ] + }) + + it('returns the bundle unchanged when the folder matches the slug', () => { + const bundle = baseExport() + expect(retargetProjectExport(bundle, 'proj', 'proj')).toBe(bundle) + }) + + it('relocates every item path and internal reference into the target folder', () => { + const out = retargetProjectExport(baseExport(), 'proj', 'dest') + expect(out.scripts[0].path).toBe('f/dest/hello') + expect(out.scripts[0].content).toContain('$res:f/dest/db') + expect(out.flows[0].path).toBe('f/dest/main_flow') + expect(out.flows[0].value.modules[0].value.path).toBe('f/dest/hello') + expect(out.apps.map((a) => a.path)).toEqual(['f/dest/dashboard', 'f/dest/rawapp']) + expect(out.resources[0].path).toBe('f/dest/db') + expect(out.triggers[0].path).toBe('f/dest/every_day') + expect(out.triggers[0].runnable_path).toBe('f/dest/hello') + // Plain-string resource path in a trigger config is remapped too. + expect(out.triggers[1].config.kafka_resource_path).toBe('f/dest/db') + }) + + it('leaves external and hub paths untouched', () => { + const bundle = baseExport() + bundle.scripts[0].content = 'const a = "$res:u/admin/db"; const b = "$res:hub/1/x"' + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.scripts[0].content).toContain('$res:u/admin/db') + expect(out.scripts[0].content).toContain('$res:hub/1/x') + }) + + it('retargets internal $var:/$jsonvar: refs but leaves external ones', () => { + const bundle = baseExport() + bundle.flows[0].value.modules[0].value.input_transforms = { + key: { type: 'static', value: '$var:f/proj/api_key' }, + ext: { type: 'static', value: '$var:u/admin/personal' } + } + bundle.flows[0].value.flow_env = { CFG: '$jsonvar:f/proj/cfg' } + bundle.triggers[1].config.queue_url = '$var:f/proj/sqs' + const out = retargetProjectExport(bundle, 'proj', 'dest') + const it = out.flows[0].value.modules[0].value.input_transforms + expect(it.key.value).toBe('$var:f/dest/api_key') + expect(it.ext.value).toBe('$var:u/admin/personal') + expect(out.flows[0].value.flow_env.CFG).toBe('$jsonvar:f/dest/cfg') + expect(out.triggers[1].config.queue_url).toBe('$var:f/dest/sqs') + }) + + it('leaves an inert $var: literal embedded in inline code unchanged', () => { + const bundle = baseExport() + // Same path as a real runtime ref, but here it is a literal inside code: it + // must not be rewritten even once the path enters the retarget map. + bundle.flows[0].value.modules[0].value = { + type: 'rawscript', + content: 'return "$var:f/proj/api_key"', + input_transforms: { real: { type: 'static', value: '$var:f/proj/api_key' } } + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + const mod = out.flows[0].value.modules[0].value + expect(mod.content).toBe('return "$var:f/proj/api_key"') + expect(mod.input_transforms.real.value).toBe('$var:f/dest/api_key') + }) +}) + +describe('collectExportVarPaths', () => { + it('gathers variable refs from flows, apps, and triggers (deduped)', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [], + flows: [{ path: 'f/proj/f', value: { flow_env: { A: '$var:f/proj/a' }, modules: [] } }], + apps: [ + { + path: 'f/proj/raw', + app_type: 'raw', + value: { raw: JSON.stringify({ runnables: { r: { fields: { x: '$var:u/admin/b' } } } }) } + } + ], + triggers: [{ path: 'f/proj/t', kind: 'sqs', config: { queue_url: '$jsonvar:f/proj/a' } }], + resources: [] + } + expect(collectExportVarPaths(bundle).sort()).toEqual(['f/proj/a', 'u/admin/b']) + }) +}) + +describe('trigger handler relocation', () => { + it('rewriteTriggerConfig remaps script/- and flow/-prefixed handler refs', () => { + const map = new Map([ + ['u/admin/handler', 'f/proj/handler'], + ['u/admin/recovery_flow', 'f/proj/recovery_flow'] + ]) + const out = rewriteTriggerConfig( + { + error_handler_path: 'u/admin/handler', + on_failure: 'script/u/admin/handler', + on_recovery: 'flow/u/admin/recovery_flow', + on_success: 'script/u/admin/unmapped' + }, + map + ) + expect(out.error_handler_path).toBe('f/proj/handler') + expect(out.on_failure).toBe('script/f/proj/handler') + expect(out.on_recovery).toBe('flow/f/proj/recovery_flow') + expect(out.on_success).toBe('script/u/admin/unmapped') + }) + + it('remaps $script:/$flow: only in the url field, never in literal payloads', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + initial_messages: [{ raw_message: '$script:u/admin/builder' }] + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.initial_messages[0].raw_message).toBe('$script:u/admin/builder') + }) + + it('leaves literal handler-shaped strings in args untouched', () => { + const map = new Map([['f/proj/handler', 'f/dest/handler']]) + const out = rewriteTriggerConfig( + { + on_failure: 'script/f/proj/handler', + args: { note: 'script/f/proj/handler' } + }, + map + ) + expect(out.on_failure).toBe('script/f/dest/handler') + expect(out.args.note).toBe('script/f/proj/handler') + }) + + it('leaves nested url keys untouched, rewriting only the top-level websocket url', () => { + const map = new Map([['u/admin/builder', 'f/proj/builder']]) + const out = rewriteTriggerConfig( + { + url: '$script:u/admin/builder', + args: { url: '$script:u/admin/builder' } + }, + map + ) + expect(out.url).toBe('$script:f/proj/builder') + expect(out.args.url).toBe('$script:u/admin/builder') + }) + + it('extracts and relocates $res refs nested in static input transform JSON', () => { + const value = { + modules: [ + { + id: 'a', + value: { + type: 'script', + path: 'f/proj/step', + input_transforms: { + provider: { type: 'static', value: { resource: '$res:u/admin/openai' } }, + note: { type: 'static', value: 'plain text' } + } + } + } + ] + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/openai' }) + const out = rewriteFlowValue(value, new Map([['u/admin/openai', 'f/proj/openai']])) + const it0 = out.modules[0].value.input_transforms + expect(it0.provider.value).toEqual({ resource: '$res:f/proj/openai' }) + expect(typeof it0.note.value).toBe('string') + }) + + it('retargetProjectExport remaps trigger error handlers with the bundle', () => { + const bundle: ProjectExport = { + project: { slug: 'proj', name: 'P', summary: '', readme: null }, + scripts: [{ path: 'f/proj/handler', content: '' }], + flows: [], + apps: [], + resources: [], + triggers: [ + { + path: 'f/proj/sched', + kind: 'schedule', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { schedule: '0 0 * * * *', on_failure: 'script/f/proj/handler' } + }, + { + path: 'f/proj/mq', + kind: 'mqtt', + runnable_path: 'f/proj/handler', + runnable_kind: 'script', + config: { error_handler_path: 'f/proj/handler' } + } + ] + } + const out = retargetProjectExport(bundle, 'proj', 'dest') + expect(out.triggers[0].config.on_failure).toBe('script/f/dest/handler') + expect(out.triggers[1].config.error_handler_path).toBe('f/dest/handler') + }) +}) + +describe('extractTriggerConfigResourceRefs', () => { + it('collects $res: tokens nested anywhere in a trigger config', () => { + expect( + extractTriggerConfigResourceRefs({ + schedule: '0 0 * * * *', + args: { channel: '$res:u/admin/slack' }, + on_failure_extra_args: { db: 'res://f/other/pg' }, + error_handler_args: { nested: { deep: '$res:u/admin/slack' } } + }) + ).toEqual(['u/admin/slack', 'f/other/pg']) + }) +}) + +describe('flow_env and preprocessor_module', () => { + const flowValue = { + modules: [], + preprocessor_module: { + id: 'pre', + value: { type: 'script', path: 'u/admin/preproc', input_transforms: {} } + }, + flow_env: { SLACK: '$res:u/admin/slack', PLAIN: 'not-a-ref' } + } + + it('walks nested children of the failure module', () => { + const refs = extractFlowRefs({ + modules: [], + failure_module: { + id: 'failure', + value: { + type: 'forloopflow', + modules: [ + { id: 'f-a', value: { type: 'script', path: 'u/admin/cleanup', input_transforms: {} } } + ] + } + } + }) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/cleanup' }) + }) + + it('extractFlowRefs sees preprocessor scripts and flow_env resources', () => { + const refs = extractFlowRefs(flowValue) + expect(refs).toContainEqual({ kind: 'script', path: 'u/admin/preproc' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/slack' }) + }) + + it('sees and relocates $res refs nested inside JSON flow_env values', () => { + const value = { + modules: [], + flow_env: { CFG: { db: '$res:u/admin/pg', opts: ['res://u/admin/s3'] } } + } + const refs = extractFlowRefs(value) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/pg' }) + expect(refs).toContainEqual({ kind: 'resource', path: 'u/admin/s3' }) + const map = new Map([ + ['u/admin/pg', 'f/proj/pg'], + ['u/admin/s3', 'f/proj/s3'] + ]) + const out = rewriteFlowValue(value, map) + expect(out.flow_env.CFG.db).toBe('$res:f/proj/pg') + expect(out.flow_env.CFG.opts[0]).toBe('$res:f/proj/s3') + }) + + it('rewriteFlowValue relocates both', () => { + const map = new Map([ + ['u/admin/preproc', 'f/proj/preproc'], + ['u/admin/slack', 'f/proj/slack'] + ]) + const out = rewriteFlowValue(flowValue, map) + expect(out.preprocessor_module.value.path).toBe('f/proj/preproc') + expect(out.flow_env.SLACK).toBe('$res:f/proj/slack') + expect(out.flow_env.PLAIN).toBe('not-a-ref') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectBundle.ts b/frontend/src/lib/components/workspaceSettings/projectBundle.ts new file mode 100644 index 0000000000..1fd62498d5 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectBundle.ts @@ -0,0 +1,652 @@ +// Pure logic for the "project = folder" Hub bundle. A project is one folder +// `f//...`. Bundling: collect the transitive closure, relocate external +// refs (`u//`, `f//` -> `f//`, `_2`/`_3`… +// on collision) and rewrite them. Hub refs stay external; runtime string-concat +// paths are out of scope. No API/Svelte deps so it's unit-testable. + +import { getAllModules } from '$lib/components/flows/flowExplorer' +import { isRunnableByPath } from '$lib/components/apps/inputType' + +export type RefKind = 'resource' | 'script' | 'flow' + +export interface Ref { + kind: RefKind + /** Bare path, without the `$res:` / `res://` prefix for resources. */ + path: string +} + +export type PathClass = 'internal' | 'hub' | 'external' + +/** A single `$res:PATH` / `res://PATH` token (path captured in group 1). */ +const RES_TOKEN_RE = /(?:\$res:|res:\/\/)([\w\-./]+)/g + +// A whole-string `$var:PATH` / `$jsonvar:PATH` value. The worker substitutes these +// only when an argument value *is* the reference (walking nested JSON), never a +// token embedded in inline code, so the whole value must match. `_KIND` captures +// the prefix (group 1) and path (group 2) so a rewrite can preserve `var`/`jsonvar`. +const VAR_VALUE_RE = /^\$(?:json)?var:([\w\-./]+)$/ +const VAR_VALUE_RE_KIND = /^\$(var|jsonvar):([\w\-./]+)$/ + +// Variable paths a value will resolve at runtime (flow static inputs, flow_env, +// app runnable inputs, trigger config fields). Walk the parsed structure and match +// whole string values so inline code carrying a literal `$var:` string is ignored. +export function extractVarRefsFromValue(value: any): string[] { + const out = new Set() + const walk = (v: any) => { + if (typeof v === 'string') { + const m = VAR_VALUE_RE.exec(v) + if (m) out.add(m[1]) + } else if (Array.isArray(v)) { + for (const x of v) walk(x) + } else if (v && typeof v === 'object') { + for (const k of Object.keys(v)) walk(v[k]) + } + } + walk(value) + return [...out] +} + +export function classifyPath(path: string, slug: string): PathClass { + if (path.startsWith(`f/${slug}/`) || path === `f/${slug}`) return 'internal' + if (path.startsWith('hub/')) return 'hub' + return 'external' +} + +export function extractScriptRefs(content: string): Ref[] { + const out: Ref[] = [] + const seen = new Set() + let m: RegExpExecArray | null + RES_TOKEN_RE.lastIndex = 0 + while ((m = RES_TOKEN_RE.exec(content)) !== null) { + if (!seen.has(m[1])) { + seen.add(m[1]) + out.push({ kind: 'resource', path: m[1] }) + } + } + return out +} + +/** + * References inside a flow value: + * - inline rawscript code with `$res:` (resource) + * - static step inputs whose value is a `$res:` literal (resource) + * - `type: script` steps that reference a script by path (script) + * - `type: flow` steps that reference a sub-flow by path (flow) + */ +export function extractFlowRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + // getAllModules flattens the whole tree (loops, branches, aiagent tools, + // failure module) so each module only needs local inspection; the + // preprocessor module sits outside `modules` and is walked the same way. + for (const mod of allFlowModules(value)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if (v.type === 'script' && typeof v.path === 'string') add('script', v.path) + if (v.type === 'flow' && typeof v.path === 'string') add('flow', v.path) + if (typeof v.content === 'string') { + for (const r of extractScriptRefs(v.content)) add('resource', r.path) + } + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Static values can be a bare `$res:` string or arbitrary JSON with + // refs nested anywhere — the worker resolves both, so scan the full + // serialization. + if (t?.type === 'static' && t.value !== undefined) { + const text = typeof t.value === 'string' ? t.value : JSON.stringify(t.value) + for (const r of extractScriptRefs(text)) add('resource', r.path) + } + } + } + } + // flow_env values support `$res:path` references — as whole string values or + // nested inside JSON values (the worker resolves both), so scan the full + // serialization. + if (value?.flow_env && typeof value.flow_env === 'object') { + for (const r of extractScriptRefs(JSON.stringify(value.flow_env))) add('resource', r.path) + } + return out +} + +// Every module of a flow value: the tree under `modules`, the failure module, +// and the preprocessor module (which lives outside `modules`). Any walk over a +// flow's modules must go through this — a walk that misses a module class +// silently drops its dependencies from bundles or migrations. All three go in +// the root list (not getAllModules' failure_module parameter, which appends +// the module without expanding its descendants) so nested children of a +// failure or preprocessor module are walked too. +export function allFlowModules(value: any) { + return getAllModules([ + ...(value?.modules ?? []), + ...(value?.preprocessor_module ? [value.preprocessor_module] : []), + ...(value?.failure_module ? [value.failure_module] : []) + ]) +} + +// Visit every object node in an app value tree (JSON-safe, no cycles). +function walkAppNodes(value: any, visit: (node: Record) => void): void { + if (value == null || typeof value !== 'object') return + if (Array.isArray(value)) { + for (const v of value) walkAppNodes(v, visit) + return + } + visit(value) + for (const k of Object.keys(value)) walkAppNodes(value[k], visit) +} + +// `runnableByPath`/`path` nodes reference a workspace runnable by path. +function runnableRef(node: Record): Ref | undefined { + if (!isRunnableByPath(node as any) || typeof node.path !== 'string') return undefined + if (node.runType === 'flow') return { kind: 'flow', path: node.path } + if (node.runType === 'script') return { kind: 'script', path: node.path } + return undefined // hubscript -> external hub, ignored +} + +// App refs: `$res:` resources anywhere in the value, plus script/flow runnables +// referenced by path in components. +export function extractAppRefs(value: any): Ref[] { + const out: Ref[] = [] + const seen = new Set() + const add = (kind: RefKind, path: string) => { + const key = `${kind}:${path}` + if (!seen.has(key)) { + seen.add(key) + out.push({ kind, path }) + } + } + walkAppNodes(value, (node) => { + const r = runnableRef(node) + if (r) add(r.kind, r.path) + }) + for (const r of extractScriptRefs(JSON.stringify(value ?? {}))) add('resource', r.path) + return out +} + +/** + * Build the relocation map. Internal paths (`f//...`) map to themselves + * and are reserved first; external paths relocate to `f//` (`_2`/`_3`… + * on collision). Input is sorted so suffix assignment is deterministic. + */ +export function buildPathMap(paths: Iterable, slug: string): Map { + const map = new Map() + const used = new Set() + const sorted = [...new Set(paths)].sort() + for (const p of sorted) { + if (classifyPath(p, slug) === 'internal') { + map.set(p, p) + used.add(p) + } + } + for (const old of sorted) { + if (map.has(old)) continue + const name = old.split('/').filter(Boolean).pop() ?? old + let candidate = `f/${slug}/${name}` + let n = 2 + while (used.has(candidate)) candidate = `f/${slug}/${name}_${n++}` + used.add(candidate) + map.set(old, candidate) + } + return map +} + +// Both ref forms normalize to `$res:` on rewrite. +export function rewriteContent(content: string, map: Map): string { + return content.replace(RES_TOKEN_RE, (whole, path) => { + const next = map.get(path) + return next ? `$res:${next}` : whole + }) +} + +// Structurally relocate whole-string `$var:`/`$jsonvar:` values — the only form the +// worker resolves. Walks the parsed value so an inert token embedded in inline code +// or arbitrary text is left untouched, unlike token replacement over serialized +// strings. Only paths present in the map move (the retarget map carries variables). +export function rewriteVarRefsInValue(value: any, map: Map): any { + if (typeof value === 'string') { + const m = VAR_VALUE_RE_KIND.exec(value) + if (m) { + const next = map.get(m[2]) + if (next) return `$${m[1]}:${next}` + } + return value + } + if (Array.isArray(value)) return value.map((v) => rewriteVarRefsInValue(v, map)) + if (value && typeof value === 'object') { + const out: Record = {} + for (const k of Object.keys(value)) out[k] = rewriteVarRefsInValue(value[k], map) + return out + } + return value +} + +/** + * `$res:`/`res://` tokens anywhere in a trigger config — schedule args, + * on_*_extra_args, error_handler_args, … (e.g. the built-in Slack handler + * stores its channel resource this way). These must enter the bundle path map + * so `rewriteTriggerConfig` relocates them and a stub is exported. + */ +export function extractTriggerConfigResourceRefs(config: any): string[] { + return extractScriptRefs(JSON.stringify(config ?? {})).map((r) => r.path) +} + +/** + * Trigger configs reference resources as plain path strings (e.g. + * `kafka_resource_path: "f/slug/db"`), not `$res:` tokens, so token rewriting + * misses them. Deep-walk the config and remap any string that exact-matches a + * map key (map keys are full bundle paths, so an exact match is a reference), + * or a `script/`/`flow/` handler reference (schedules' on_failure + * et al.), falling back to `$res:` token rewriting for embedded refs. + */ +// Top-level config fields whose string values are prefixed runnable refs. +// Prefixed forms are remapped ONLY in these known positions: deciding meaning +// from string shape alone rewrote literal payloads that merely looked like +// refs. Bare-path exact matches and $res: tokens stay position-independent. +const HANDLER_REF_FIELDS = new Set(['on_failure', 'on_recovery', 'on_success']) + +export function rewriteTriggerConfig(config: any, map: Map, depth = 0): any { + if (typeof config === 'string') { + const direct = map.get(config) + if (direct) return direct + return rewriteContent(config, map) + } + if (Array.isArray(config)) return config.map((v) => rewriteTriggerConfig(v, map, depth + 1)) + if (config && typeof config === 'object') { + return Object.fromEntries( + Object.entries(config).map(([k, v]) => { + if (depth === 0 && typeof v === 'string') { + // Websocket url: $script: / $flow:. + if (k === 'url') { + const m = /^\$(script|flow):(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `$${m[1]}:${map.get(m[2])}`] + } + // Schedule handlers: script/ / flow/. + if (HANDLER_REF_FIELDS.has(k)) { + const m = /^(script|flow)\/(.+)$/.exec(v) + if (m && map.has(m[2])) return [k, `${m[1]}/${map.get(m[2])}`] + } + } + return [k, rewriteTriggerConfig(v, map, depth + 1)] + }) + ) + } + return config +} + +export function rewriteFlowValue(value: any, map: Map): any { + const cloned = JSON.parse(JSON.stringify(value ?? {})) + for (const mod of allFlowModules(cloned)) { + const v: any = (mod as any)?.value + if (!v || typeof v !== 'object') continue + if ( + (v.type === 'script' || v.type === 'flow') && + typeof v.path === 'string' && + map.has(v.path) + ) { + v.path = map.get(v.path) + } + if (typeof v.content === 'string') v.content = rewriteContent(v.content, map) + const it = v.input_transforms + if (it && typeof it === 'object') { + for (const key of Object.keys(it)) { + const t = it[key] + // Mirror extraction: rewrite refs wherever they sit, preserving the + // value's type (a string stays a string, JSON round-trips). + if (t?.type === 'static' && t.value !== undefined) { + if (typeof t.value === 'string') { + t.value = rewriteContent(t.value, map) + } else { + t.value = JSON.parse(rewriteContent(JSON.stringify(t.value), map)) + } + } + } + } + } + if (cloned?.flow_env && typeof cloned.flow_env === 'object') { + // Tokens can sit inside nested JSON values, not just string values; the + // serialize→rewrite→parse round-trip reaches all of them (paths contain + // no characters that would break JSON string literals). + cloned.flow_env = JSON.parse(rewriteContent(JSON.stringify(cloned.flow_env), map)) + } + return cloned +} + +// Relocate `$res:` tokens (one round-trip, also produces a fresh clone) then +// runnable-by-path refs structurally. Incidental `f//` strings stay intact. +export function rewriteAppValue(value: any, map: Map): any { + if (value == null) return value + const cloned = JSON.parse(rewriteContent(JSON.stringify(value), map)) + walkAppNodes(cloned, (node) => { + if (runnableRef(node) && map.has(node.path)) node.path = map.get(node.path) + }) + return cloned +} + +// Raw/compiled apps store their structure as a JSON string (`{ runnables, files }`). +// Parse it so runnable-by-path refs in the runnables map are seen, reusing the +// same walk; fall back to plain `$res:` scanning if it isn't valid JSON. +export function extractRawAppRefs(content: string): Ref[] { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return extractScriptRefs(content) + } + return extractAppRefs(parsed) +} + +export function rewriteRawAppContent(content: string, map: Map): string { + let parsed: any + try { + parsed = JSON.parse(content) + } catch { + return rewriteContent(content, map) + } + return JSON.stringify(rewriteAppValue(parsed, map)) +} + +// --------------------------------------------------------------------------- +// Hub project export format (what /projects/{slug}/export returns) and its +// retargeting into a destination folder. Kept here, next to the rewriters, +// so the bundle format is defined in one module for both publish and install. +// --------------------------------------------------------------------------- + +export type ExportItem = Record +export interface ProjectMigration { + datatable_name: string + sql: string + sql_down?: string + enabled: boolean +} +export interface ProjectExport { + project: { slug: string; name: string; summary: string; readme: string | null } + scripts: ExportItem[] + flows: ExportItem[] + apps: ExportItem[] + resources: ExportItem[] + triggers: ExportItem[] + migrations?: ProjectMigration[] +} + +// Map bundled paths `f//...` -> `f//...`. Only enumerated +// paths go in, so rewriters touch real refs, never incidental text. +export function buildRetargetMap( + bundle: ProjectExport, + fromSlug: string, + folder: string +): Map { + const map = new Map() + const prefix = `f/${fromSlug}/` + const add = (p: unknown) => { + if (typeof p === 'string' && p.startsWith(prefix)) { + map.set(p, `f/${folder}/${p.slice(prefix.length)}`) + } + } + for (const s of bundle.scripts) add(s.path) + for (const f of bundle.flows) add(f.path) + for (const a of bundle.apps) add(a.path) + for (const r of bundle.resources) add(r.path) + for (const t of bundle.triggers) { + add(t.path) + add(t.runnable_path) + } + // Variables aren't enumerated in the export; their `$var:`/`$jsonvar:` refs live + // inside item values. Relocate the internal ones so a renamed-folder import + // rewrites them into the target folder instead of retaining the old prefix. + for (const p of collectExportVarPaths(bundle)) add(p) + return map +} + +// Internal-or-external variable paths referenced by the export's flows, apps and +// triggers. Scripts carry no variable args. Raw apps hold their structure in the +// `value.raw` JSON string. +export function collectExportVarPaths(bundle: ProjectExport): string[] { + const out = new Set() + const collect = (value: any) => { + for (const p of extractVarRefsFromValue(value)) out.add(p) + } + for (const f of bundle.flows) collect(f.value) + for (const a of bundle.apps) collect(a.app_type === 'raw' ? safeParseRaw(a.value?.raw) : a.value) + for (const t of bundle.triggers) collect(t.config) + return [...out] +} + +function safeParseRaw(raw: unknown): any { + if (typeof raw !== 'string') return undefined + try { + return JSON.parse(raw) + } catch { + return undefined + } +} + +// Structural retarget: rewrite each item's path and its internal refs, +// leaving Hub refs and arbitrary content untouched. +export function retargetProjectExport( + bundle: ProjectExport, + fromSlug: string, + folder: string +): ProjectExport { + if (folder === fromSlug) return bundle + const map = buildRetargetMap(bundle, fromSlug, folder) + const remap = (p: unknown) => (typeof p === 'string' ? (map.get(p) ?? p) : p) + return { + ...bundle, + scripts: bundle.scripts.map((s) => ({ + ...s, + path: remap(s.path), + content: rewriteContent(s.content ?? '', map) + })), + flows: bundle.flows.map((f) => ({ + ...f, + path: remap(f.path), + value: rewriteVarRefsInValue(rewriteFlowValue(f.value, map), map) + })), + apps: bundle.apps.map((a) => ({ + ...a, + path: remap(a.path), + // Raw apps keep their structure in the `value.raw` JSON string. + value: + a.app_type === 'raw' + ? { + ...a.value, + raw: rewriteRawVarRefs(rewriteRawAppContent(a.value?.raw ?? '', map), map) + } + : rewriteVarRefsInValue(rewriteAppValue(a.value, map), map) + })), + resources: bundle.resources.map((r) => ({ ...r, path: remap(r.path) })), + triggers: bundle.triggers.map((t) => ({ + ...t, + path: remap(t.path), + runnable_path: remap(t.runnable_path), + // Configs hold `$res:` tokens, plain resource paths (kafka_resource_path + // etc.) and whole-string `$var:` values — rewrite all three. + config: t.config ? rewriteVarRefsInValue(rewriteTriggerConfig(t.config, map), map) : t.config + })) + } +} + +// Var relocation for a raw app's `value.raw` JSON string: parse, structurally +// rewrite whole-string var values, re-serialize; leave invalid JSON untouched. +function rewriteRawVarRefs(raw: string, map: Map): string { + const parsed = safeParseRaw(raw) + if (parsed === undefined) return raw + return JSON.stringify(rewriteVarRefsInValue(parsed, map)) +} + +export type ItemKind = 'script' | 'flow' | 'app' | 'raw_app' + +export interface ItemRef { + kind: ItemKind + path: string +} + +export interface FetchedItem { + kind: ItemKind + path: string + summary?: string + description?: string + /** scripts + raw_apps */ + content?: string + /** flows + apps */ + value?: any + /** scripts */ + language?: string + schema?: any + lock?: string + scriptKind?: string +} + +export interface BundleDeps { + /** Fetch a workspace item by ref, or undefined if it doesn't exist. */ + fetchItem: (ref: ItemRef) => Promise + /** Resolve a resource path to its type, or undefined if missing. */ + resolveResourceType: (path: string) => Promise +} + +export interface BundledItem extends FetchedItem { + /** Path the item takes inside the project folder. */ + newPath: string +} + +export interface ResourceStub { + originalPath: string + newPath: string + resource_type: string +} + +export interface ProjectBundle { + items: BundledItem[] + resourceStubs: ResourceStub[] + /** Original -> relocated path for every item and resource (incl. unresolved). */ + pathMap: Map + /** External paths we couldn't fetch/resolve (missing items or untyped resources). */ + unresolved: string[] +} + +function refsForFetched(item: FetchedItem): Ref[] { + if (item.kind === 'script') return extractScriptRefs(item.content ?? '') + if (item.kind === 'flow') return extractFlowRefs(item.value) + if (item.kind === 'app') return extractAppRefs(item.value) + if (item.kind === 'raw_app') return extractRawAppRefs(item.content ?? '') + return [] +} + +// Whole-string `$var:`/`$jsonvar:` paths an item resolves at runtime. Scripts carry +// no variable args; raw apps hold their structure in the `content` JSON string. +function varRefsForFetched(item: FetchedItem): string[] { + if (item.kind === 'flow' || item.kind === 'app') return extractVarRefsFromValue(item.value) + if (item.kind === 'raw_app') return extractVarRefsFromValue(safeParseRaw(item.content)) + return [] +} + +// Walks the transitive closure: scripts referenced by path are pulled in +// recursively, resources become empty stubs, hub refs stay external. +export async function buildProjectBundle( + seed: ItemRef[], + slug: string, + deps: BundleDeps, + extraResourcePaths: string[] = [], + extraVarPaths: string[] = [] +): Promise { + const fetched = new Map() + const queued = new Set() + const resourcePaths = new Set() + const varPaths = new Set() + const unresolved: string[] = [] + + // Resources and variables referenced by triggers (by config value, not `$res:` + // in code) — relocated through the same map so the export stays slug-relative. + for (const p of extraResourcePaths) { + if (classifyPath(p, slug) !== 'hub') resourcePaths.add(p) + } + for (const p of extraVarPaths) varPaths.add(p) + + // Key by `${kind}:${path}`, not bare path: a script and flow can share a path, + // and keying by path alone would silently drop one. + const refKey = (kind: string, path: string) => `${kind}:${path}` + + // Refs at the same BFS depth are independent: fetch each level concurrently. + let level: ItemRef[] = [] + for (const s of seed) { + const key = refKey(s.kind, s.path) + if (!queued.has(key)) { + queued.add(key) + level.push(s) + } + } + while (level.length > 0) { + const results = await Promise.all( + level.map(async (ref) => ({ ref, item: await deps.fetchItem(ref) })) + ) + const next: ItemRef[] = [] + for (const { ref, item } of results) { + if (!item) { + unresolved.push(ref.path) + continue + } + fetched.set(refKey(ref.kind, ref.path), item) + for (const r of refsForFetched(item)) { + if (classifyPath(r.path, slug) === 'hub') continue + if (r.kind === 'resource') { + resourcePaths.add(r.path) + } else if (r.kind === 'script' || r.kind === 'flow') { + const key = refKey(r.kind, r.path) + if (!queued.has(key)) { + queued.add(key) + next.push({ kind: r.kind, path: r.path }) + } + } + } + // Relocate the item's runtime variable refs into the project folder too, so + // the export is slug-relative regardless of the source folder (import then + // materializes them as placeholders). Variables are never hub-hosted. + for (const p of varRefsForFetched(item)) varPaths.add(p) + } + level = next + } + + const fetchedItems = [...fetched.values()] + const itemPaths = fetchedItems.map((it) => it.path) + const map = buildPathMap([...itemPaths, ...resourcePaths, ...varPaths], slug) + + const items: BundledItem[] = fetchedItems.map((it) => { + const rewritten: BundledItem = { ...it, newPath: map.get(it.path) ?? it.path } + if (it.kind === 'script') { + rewritten.content = rewriteContent(it.content ?? '', map) + } else if (it.kind === 'raw_app') { + rewritten.content = rewriteRawVarRefs(rewriteRawAppContent(it.content ?? '', map), map) + } else if (it.kind === 'flow') { + rewritten.value = rewriteVarRefsInValue(rewriteFlowValue(it.value, map), map) + } else if (it.kind === 'app') { + rewritten.value = rewriteVarRefsInValue(rewriteAppValue(it.value, map), map) + } + return rewritten + }) + + const resourceStubs: ResourceStub[] = [] + const resolved = await Promise.all( + [...resourcePaths].map(async (path) => ({ path, type: await deps.resolveResourceType(path) })) + ) + for (const { path, type } of resolved) { + if (!type) { + unresolved.push(path) + continue + } + resourceStubs.push({ originalPath: path, newPath: map.get(path) ?? path, resource_type: type }) + } + + // `unresolved` keys missing items by kind:path but stores the bare path, so a + // missing script and flow (or a runnable and resource) sharing a path can push + // the same string twice. Dedupe: callers use it as a display/blocker list where + // duplicate keys would break keyed rendering. + return { items, resourceStubs, pathMap: map, unresolved: [...new Set(unresolved)] } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts new file mode 100644 index 0000000000..a02d4e264d --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect } from 'vitest' +import { refContainmentViolation, varContainmentViolation } from './projectInstall' +import type { Ref } from './projectBundle' + +describe('refContainmentViolation', () => { + const folder = 'proj' + const violation = (r: Ref) => refContainmentViolation([r], folder) + + it('allows references relocated into the target folder', () => { + expect(violation({ kind: 'resource', path: 'f/proj/db' })).toBeUndefined() + expect(violation({ kind: 'script', path: 'f/proj/helper' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'f/proj/sub' })).toBeUndefined() + }) + + it('allows hub script/flow references but never hub resources', () => { + expect(violation({ kind: 'script', path: 'hub/1/x/y' })).toBeUndefined() + expect(violation({ kind: 'flow', path: 'hub/1/a/b' })).toBeUndefined() + // Resources are not hub-hosted, so a hub/ resource path is still an escape. + expect(violation({ kind: 'resource', path: 'hub/1/x/y' })).toBeDefined() + }) + + it('rejects references bound to another namespace', () => { + // The crux: an in-folder runnable pointing its resource at an existing asset. + expect(violation({ kind: 'resource', path: 'u/admin/db' })).toContain('escapes') + expect(violation({ kind: 'script', path: 'f/other/helper' })).toContain('escapes') + expect(violation({ kind: 'flow', path: 'u/admin/sub' })).toContain('escapes') + }) + + it('does not treat a prefix-only folder match as internal', () => { + expect(violation({ kind: 'script', path: 'f/proj2/helper' })).toContain('escapes') + }) + + it('reports the first offending reference and passes a fully-contained set', () => { + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'script', path: 'hub/1/x/y' } + ], + folder + ) + ).toBeUndefined() + expect( + refContainmentViolation( + [ + { kind: 'resource', path: 'f/proj/db' }, + { kind: 'resource', path: 'u/admin/secret' } + ], + folder + ) + ).toContain('u/admin/secret') + }) +}) + +describe('varContainmentViolation', () => { + const folder = 'proj' + + it('allows in-folder variable references', () => { + expect(varContainmentViolation({ token: '$var:f/proj/token' }, folder)).toBeUndefined() + expect(varContainmentViolation({ x: 'no refs here' }, folder)).toBeUndefined() + }) + + it('rejects a `$var:` or `$jsonvar:` bound to another namespace', () => { + // The crux: a variable arg the ref extractors miss, resolved under the perms. + expect(varContainmentViolation({ queue_url: '$var:u/admin/token' }, folder)).toContain( + 'u/admin/token' + ) + expect(varContainmentViolation({ cfg: '$jsonvar:f/other/secret' }, folder)).toContain('escapes') + }) + + it('ignores a `$var:` literal embedded in inline code', () => { + const flowValue = { + flow_env: { API: '$var:f/proj/api_key' }, + modules: [{ value: { type: 'rawscript', content: 'return "$var:u/admin/should_not_flag"' } }] + } + expect(varContainmentViolation(flowValue, folder)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectInstall.ts b/frontend/src/lib/components/workspaceSettings/projectInstall.ts new file mode 100644 index 0000000000..1827e00283 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectInstall.ts @@ -0,0 +1,405 @@ +// Imports a Hub project export into a workspace: one importer per item kind, +// each item reported individually so one bad item never aborts the rest. +// UI-free — the install page owns folder choice and migration review. + +import { + AppService, + FlowService, + FolderService, + ResourceService, + ScriptService, + VariableService, + WorkspaceService +} from '$lib/gen' +import { + TRIGGER_KINDS, + createWorkspaceTriggerDisabled, + triggerHandlerRefs, + type WorkspaceTrigger, + type WorkspaceTriggerKind +} from '../triggers/workspaceTriggersList' +import { updatePolicy } from '$lib/components/apps/editor/appPolicy' +import { updateRawAppPolicy } from '$lib/sharedUtils' +import type { App } from '$lib/components/apps/types' +import { runScriptAndPollResult } from '$lib/components/jobs/utils' +import { + classifyPath, + collectExportVarPaths, + extractAppRefs, + extractFlowRefs, + extractRawAppRefs, + extractScriptRefs, + extractTriggerConfigResourceRefs, + extractVarRefsFromValue, + retargetProjectExport, + type ExportItem, + type ProjectExport, + type ProjectMigration, + type Ref +} from './projectBundle' + +export interface InstallResult { + path: string + ok: boolean + error?: string +} + +// Guarding an item's own path is not enough: the `$res:`/script/flow refs baked +// into its content are live bindings the backend acts on. A well-formed export +// relocates them all into f// (hub/ script refs stay external); anything +// else points a runnable at an existing asset in another namespace, so refuse the +// item rather than bind it there. Resources are never hub-hosted, so a hub/ path +// there is not a valid escape hatch. Mirrors the trigger-config containment. +export function refContainmentViolation(refs: Ref[], folder: string): string | undefined { + for (const r of refs) { + const cls = classifyPath(r.path, folder) + if (cls === 'internal') continue + if (cls === 'hub' && r.kind !== 'resource') continue + return `reference '${r.path}' escapes the target folder f/${folder}/ — skipped` + } + return undefined +} + +// `$var:`/`$jsonvar:` references (in flow static inputs, flow_env, app runnable +// inputs, trigger config) are resolved at runtime under the imported runnable's +// permissions and are never hub-hosted. Retargeting relocates a project's own refs +// into the target folder; anything still outside it points at another namespace, so +// reject those. Takes the parsed value so inline code carrying a literal is ignored. +export function varContainmentViolation(value: any, folder: string): string | undefined { + for (const p of extractVarRefsFromValue(value)) { + if (classifyPath(p, folder) !== 'internal') { + return `variable '${p}' escapes the target folder f/${folder}/ — skipped` + } + } + return undefined +} + +// Surface the backend's explanation: API errors carry the real message in +// `.body` (plain text for Windmill 4xx), while `.message` is the generic +// status text ("Bad Request"). Prefer the body so e.g. a path/route_path +// collision reads as the actual reason, not just "Bad Request". +function errorMessage(e: any): string { + const body = e?.body + if (typeof body === 'string' && body.trim() !== '') return body + if (body && typeof body === 'object') + return body.error?.message ?? body.message ?? JSON.stringify(body) + return e?.message ?? String(e) +} + +// Recompute an app's execution policy from its (retargeted) value, mirroring +// what the editor does on deploy. `triggerables_v2` is keyed by +// `:rawscript/`; retargeting rewrites that +// content, so a copied or empty policy would leave every inline runnable +// "forbidden by policy" at runtime. Default to publisher (auth required). +async function computeAppPolicy(value: any): Promise { + const policy = (await updatePolicy(value as App, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} +async function computeRawAppPolicy(runnables: Record): Promise { + const policy = (await updateRawAppPolicy(runnables, undefined)) as any + if (!policy.execution_mode) policy.execution_mode = 'publisher' + return policy +} + +function importScript(workspace: string, s: ExportItem): Promise { + return ScriptService.createScript({ + workspace, + requestBody: { + path: s.path, + summary: s.summary ?? '', + description: s.description ?? '', + content: s.content ?? '', + language: s.language, + schema: s.schema ?? undefined, + kind: s.kind ?? 'script', + lock: s.lockfile ?? undefined + } + }) +} + +function importFlow(workspace: string, f: ExportItem): Promise { + return FlowService.createFlow({ + workspace, + requestBody: { + path: f.path, + summary: f.summary ?? '', + description: f.description ?? '', + value: f.value, + schema: f.schema ?? undefined + } + }) +} + +// Stubs only: never overwrite an existing resource's value (updateIfExists +// stays false so a path collision is reported as a failed item instead). +function importResourceStub(workspace: string, r: ExportItem): Promise { + return ResourceService.createResource({ + workspace, + updateIfExists: false, + requestBody: { + path: r.path, + resource_type: r.resource_type, + value: {}, + description: 'Imported stub — fill in the value.' + } + }) +} + +// Variables hold secrets/config, so their values are never shipped. Create an empty +// secret placeholder for a project variable the importer must fill, mirroring the +// resource stubs. Conflict-safe: an already-present variable (the importer filled it, +// or a re-import) is left untouched rather than clobbered. +async function importVariablePlaceholder(workspace: string, path: string): Promise { + if (await VariableService.existsVariable({ workspace, path })) return + await VariableService.createVariable({ + workspace, + requestBody: { + path, + value: '', + is_secret: true, + description: 'Imported placeholder — fill in the value.' + } + }) +} + +async function importApp(workspace: string, a: ExportItem): Promise { + if (a.app_type === 'raw') { + let parsed: any + try { + parsed = JSON.parse(a.value?.raw ?? '{}') + } catch (e: any) { + throw new Error(`invalid raw app bundle: ${e?.message ?? String(e)}`) + } + const files = { ...(parsed.files ?? {}) } + const js = files['/bundle.js'] ?? '' + const css = files['/bundle.css'] ?? '' + delete files['/bundle.js'] + delete files['/bundle.css'] + const runnables = parsed.runnables ?? {} + return AppService.createAppRaw({ + workspace, + formData: { + app: { + path: a.path, + summary: a.summary ?? '', + value: { + files, + runnables, + // Keep the full-code app's explicit data table declaration. + ...(parsed.data !== undefined ? { data: parsed.data } : {}), + ...(parsed.datatables !== undefined ? { datatables: parsed.datatables } : {}) + }, + policy: await computeRawAppPolicy(runnables) + }, + js, + css + } + }) + } + return AppService.createApp({ + workspace, + requestBody: { + path: a.path, + summary: a.summary ?? '', + value: a.value, + policy: await computeAppPolicy(a.value) + } + }) +} + +// Apply one migration to the target data table. If the data table opted into +// migrations, record it (datatable_migrations + _wm_migrations, run only this +// version); otherwise run the SQL once as a preview job (unrecorded). +async function applyOneMigration( + workspace: string, + projectSlug: string, + m: ProjectMigration +): Promise { + let recorded = false + try { + const status = await WorkspaceService.getDatatableMigrationsStatus({ + workspace, + datatableName: m.datatable_name + }) + recorded = !!status.enabled + } catch {} + + if (recorded) { + // Record the shipped down migration (DROP the created tables) so it can be + // rolled back. + const codeDown = (m.sql_down ?? '').trim() + const created = await WorkspaceService.createDatatableMigration({ + workspace, + datatableName: m.datatable_name, + requestBody: { + name: `hub_import_${projectSlug}`, + code_up: m.sql, + code_down: codeDown || undefined + } + }) + await WorkspaceService.runDatatableMigrations({ + workspace, + datatableName: m.datatable_name, + only: created.timestamp + }) + } else { + await runScriptAndPollResult({ + workspace, + requestBody: { + language: 'postgresql', + content: m.sql, + args: { database: `datatable://${m.datatable_name}` } + } + }) + } +} + +/** + * Install a project export into `workspace` under `f//`: create the + * folder, retarget every item, import kind by kind, then apply the (already + * reviewed) migrations. Each item's outcome is reported through `onResult`; + * failures never abort the remaining items. + */ +export async function installProject(args: { + workspace: string + exportData: ProjectExport + folder: string + migrations: ProjectMigration[] + hasEeLicense: boolean + onResult: (r: InstallResult) => void +}): Promise { + const { workspace, exportData, folder, migrations, hasEeLicense, onResult } = args + + const record = (path: string, p: Promise): Promise => + p.then( + () => onResult({ path, ok: true }), + (e: any) => onResult({ path, ok: false, error: errorMessage(e) }) + ) + + try { + await FolderService.createFolder({ workspace, requestBody: { name: folder } }) + } catch {} + + const proj = retargetProjectExport(exportData, exportData.project.slug, folder) + + // The export is remote input: every path it wants to write must stay inside + // the folder the user chose. Anything else (crafted export, or an export + // whose items weren't relocated into f// at publish) is refused + // per-item instead of being created in another namespace. + const prefix = `f/${folder}/` + const guard = (path: unknown, ...also: unknown[]): string | undefined => { + for (const p of [path, ...also]) { + if (typeof p !== 'string' || !p.startsWith(prefix)) { + return `path '${String(p)}' escapes the target folder ${prefix} — skipped` + } + } + return undefined + } + const checked = (path: unknown, run: () => Promise, ...also: unknown[]) => { + const violation = guard(path, ...also) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + // `refs` catches structured runnable/`$res:` refs; `varValue` is the parsed item + // walked for `$var:`/`$jsonvar:` argument refs (which the ref extractors miss). + const checkedItem = (path: unknown, refs: Ref[], varValue: any, run: () => Promise) => { + const violation = + guard(path) ?? + refContainmentViolation(refs, folder) ?? + varContainmentViolation(varValue, folder) + return violation + ? record(String(path), Promise.reject(new Error(violation))) + : record(String(path), run()) + } + + for (const s of proj.scripts) { + // `$var:` is resolved in job args (flow inputs, schedule args, trigger config), + // not in script source, so there is no variable arg to contain here. + await checkedItem(s.path, extractScriptRefs(s.content ?? ''), undefined, () => + importScript(workspace, s) + ) + } + for (const f of proj.flows) { + await checkedItem(f.path, extractFlowRefs(f.value), f.value, () => importFlow(workspace, f)) + } + for (const r of proj.resources) { + await checked(r.path, () => importResourceStub(workspace, r)) + } + // Placeholders for the project's internal `$var:`/`$jsonvar:` refs (retargeted + // into this folder). External refs are rejected per-item, so only stub in-folder + // ones; guard again in case an out-of-folder ref slipped through retargeting. + for (const p of collectExportVarPaths(proj)) { + if (!p.startsWith(prefix)) continue + await record(`variable: ${p}`, importVariablePlaceholder(workspace, p)) + } + for (const a of proj.apps) { + const isRaw = a.app_type === 'raw' + const refs = isRaw ? extractRawAppRefs(a.value?.raw ?? '') : extractAppRefs(a.value) + // Raw apps hold their runnables in the `value.raw` JSON string; parse it so the + // walk sees the same structure the backend resolves. Malformed raw fails at import. + let varValue: any = a.value + if (isRaw) { + try { + varValue = JSON.parse(a.value?.raw ?? '{}') + } catch { + varValue = undefined + } + } + await checkedItem(a.path, refs, varValue, () => importApp(workspace, a)) + } + // A trigger's config is a live binding, not inert content: resource fields, + // handler runnables and $res: refs it names are acted on by the backend, so + // every one must stay inside the chosen folder (handlers may also point at + // hub/ scripts). Otherwise a crafted export could bind the trigger to + // existing assets in another namespace. + const triggerConfigViolation = (t: ExportItem): string | undefined => { + const cfg = (t.config ?? {}) as Record + for (const r of triggerHandlerRefs({ kind: t.kind, config: cfg } as WorkspaceTrigger)) { + if (!r.path.startsWith(prefix) && !r.path.startsWith('hub/')) { + return `handler '${r.path}' escapes the target folder ${prefix} — skipped` + } + } + const resourceRefs = new Set(extractTriggerConfigResourceRefs(cfg)) + const field = TRIGGER_KINDS[t.kind as WorkspaceTriggerKind]?.resourceField + const fieldValue = field ? cfg[field] : undefined + if (typeof fieldValue === 'string' && fieldValue !== '') resourceRefs.add(fieldValue) + for (const p of resourceRefs) { + if (!p.startsWith(prefix)) { + return `resource '${p}' escapes the target folder ${prefix} — skipped` + } + } + // Config fields (e.g. SQS queue_url) can carry `$var:`/`$jsonvar:` refs too. + return varContainmentViolation(cfg, folder) + } + for (const t of proj.triggers) { + const violation = guard(t.path, t.runnable_path) ?? triggerConfigViolation(t) + await record( + String(t.path), + violation + ? Promise.reject(new Error(violation)) + : createWorkspaceTriggerDisabled( + workspace, + { + kind: t.kind, + path: t.path, + script_path: t.runnable_path, + is_flow: t.runnable_kind === 'flow', + summary: t.summary ?? null, + config: t.config ?? null + }, + { hasEeLicense } + ) + ) + } + + // Apply the reviewed data table migrations after items exist. + for (const m of migrations) { + await record( + `data table: ${m.datatable_name}`, + applyOneMigration(workspace, exportData.project.slug, m) + ) + } +} diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts new file mode 100644 index 0000000000..7d35c51650 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.test.ts @@ -0,0 +1,372 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +// inferAssets loads WASM; stub it so script detection is deterministic and no +// wasm init runs in the test. +const inferAssetsMock = vi.fn() +vi.mock('$lib/infer', () => ({ inferAssets: (...a: any[]) => inferAssetsMock(...a) })) + +// Only getDatatableFullSchema is used by the generator; stub the whole service. +const getDatatableFullSchemaMock = vi.fn() +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + getDatatableFullSchema: (...a: any[]) => getDatatableFullSchemaMock(...a) + } +})) + +import { detectDatatableTables, generateDatatableMigrations } from './projectMigrations' +import type { FetchedItem } from './projectBundle' + +describe('detectDatatableTables', () => { + beforeEach(() => inferAssetsMock.mockReset()) + + it('collects datatable/table refs from scripts (re-parsed), flows and raw apps', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [ + { kind: 'datatable', path: 'main/customers' }, + { kind: 'resource', path: 'u/admin/pg' } // ignored + ] + }) + const items: FetchedItem[] = [ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'select 1' }, + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/orders' }] + } + } + ] + } + }, + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: { + r1: { inlineScript: { assets: [{ kind: 'datatable', path: 'analytics/events' }] } } + } + }) + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])].sort()).toEqual(['customers', 'orders']) + expect([...(usage.get('analytics') ?? [])]).toEqual(['events']) + }) + + it('collects datatable refs from the preprocessor module', async () => { + inferAssetsMock.mockResolvedValue({ status: 'ok', assets: [] }) + const items: FetchedItem[] = [ + { + kind: 'flow', + path: 'f/p/fl', + value: { + modules: [], + preprocessor_module: { + id: 'pre', + value: { + type: 'rawscript', + language: 'duckdb', + content: '', + assets: [{ kind: 'datatable', path: 'main/inbox' }] + } + } + } + } + ] + const usage = await detectDatatableTables(items) + expect([...(usage.get('main') ?? [])]).toEqual(['inbox']) + }) + + it('records a datatable used with no specific table', async () => { + inferAssetsMock.mockResolvedValue({ + status: 'ok', + assets: [{ kind: 'datatable', path: 'main' }] + }) + const usage = await detectDatatableTables([ + { kind: 'script', path: 'f/p/s', language: 'duckdb', content: 'x' } + ]) + expect(usage.has('main')).toBe(true) + expect(usage.get('main')?.size).toBe(0) + }) + + it('reads a full-code app’s explicit data.tables declaration', async () => { + const items: FetchedItem[] = [ + { + kind: 'raw_app', + path: 'f/p/app', + content: JSON.stringify({ + runnables: {}, + data: { + datatable: 'main', + schema: 'app1', + tables: ['main/customers', 'main/app1:orders'] + } + }) + } + ] + const usage = await detectDatatableTables(items) + // public-schema ref keeps the bare name; non-public keeps schema.table. + expect([...(usage.get('main') ?? [])].sort()).toEqual(['app1.orders', 'customers']) + }) +}) + +describe('generateDatatableMigrations', () => { + beforeEach(() => getDatatableFullSchemaMock.mockReset()) + + const schema = { + public: { + customers: { + name: 'customers', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'email', datatype: 'text', nullable: true } + ], + foreign_keys: [] + }, + orders: { + name: 'orders', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'customer_id', datatype: 'integer', nullable: false } + ], + foreign_keys: [ + { + target_table: 'public.customers', + columns: [{ source_column: 'customer_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + + it('creates referenced tables in FK-dependency order in one transaction, enabled', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders', 'customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + const m = migrations[0] + expect(m.datatable_name).toBe('main') + expect(m.enabled).toBe(true) + expect(m.sql.startsWith('BEGIN;')).toBe(true) + expect(m.sql.trimEnd().endsWith('COMMIT;')).toBe(true) + // customers (FK target) must be created before orders (FK source). + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + // A single wrapping transaction, not one per table. + expect(m.sql.match(/BEGIN;/g)?.length).toBe(1) + // Idempotent: won't abort if a pulled-in parent already exists in the target. + expect(m.sql).toContain('CREATE TABLE IF NOT EXISTS "public"."customers"') + // Down migration lists drops commented out (nothing dropped by default), + // in reverse order: orders (child) before customers (parent). + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."orders";') + expect(m.sql_down).toContain('-- DROP TABLE IF EXISTS "public"."customers";') + // No uncommented DROP TABLE anywhere. + expect(/^\s*DROP TABLE/m.test(m.sql_down)).toBe(false) + expect(m.sql_down.indexOf('"public"."orders"')).toBeLessThan( + m.sql_down.indexOf('"public"."customers"') + ) + }) + + it('accepts schema-qualified table refs', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['public.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + }) + + it('leaves a qualified ref unresolved when its schema misses, never another schema\'s table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['sales.orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"sales.orders" is referenced but was not found') + expect(migrations[0].sql).not.toContain('CREATE TABLE "') + }) + + it('emits all CREATE TABLEs before any FK constraint so circular FKs work', async () => { + const cyclicSchema = { + public: { + a: { + name: 'a', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'b_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.b', + columns: [{ source_column: 'b_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + }, + b: { + name: 'b', + columns: [ + { name: 'id', datatype: 'integer', primary_key: true, nullable: false }, + { name: 'a_id', datatype: 'integer', nullable: true } + ], + foreign_keys: [ + { + target_table: 'public.a', + columns: [{ source_column: 'a_id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(cyclicSchema) + const usage = new Map([['main', new Set(['a', 'b'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('"public"."a"') + expect(sql).toContain('"public"."b"') + // Both FK constraints present, and every CREATE TABLE precedes the first one. + expect(sql.match(/ADD CONSTRAINT/g)?.length).toBe(2) + const lastCreate = sql.lastIndexOf('CREATE TABLE IF NOT EXISTS') + const firstConstraint = sql.indexOf('DO $$') + expect(lastCreate).toBeGreaterThan(-1) + expect(firstConstraint).toBeGreaterThan(lastCreate) + }) + + it('guards FK creation so re-running on an existing table does not abort', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + // The ADD CONSTRAINT must be wrapped in a pg_constraint existence check. + expect(sql).toContain('DO $$') + expect(sql).toContain('SELECT 1 FROM pg_constraint') + expect(sql).toContain(`conrelid = '"public"."orders"'::regclass`) + // No unguarded ALTER TABLE ... ADD at the start of a line. + expect(/^ALTER TABLE .* ADD CONSTRAINT/m.test(sql)).toBe(false) + }) + + it('creates non-public schemas before their tables', async () => { + const appSchema = { + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(appSchema) + const usage = new Map([['main', new Set(['app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const sql = migrations[0].sql + expect(sql).toContain('CREATE SCHEMA IF NOT EXISTS "app";') + expect(sql.indexOf('CREATE SCHEMA IF NOT EXISTS "app";')).toBeLessThan( + sql.indexOf('CREATE TABLE IF NOT EXISTS "app"."customers"') + ) + expect(sql).not.toContain('CREATE SCHEMA IF NOT EXISTS "public"') + }) + + it('keeps same-named tables from different schemas both created', async () => { + const twoSchemas = { + public: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + }, + app: { + customers: { + name: 'customers', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(twoSchemas) + const usage = new Map([['main', new Set(['public.customers', 'app.customers'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('"app"."customers"') + }) + + it('transitively pulls in FK-referenced tables not directly used', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + // Only `orders` is referenced; `customers` (its FK target) must still be + // created, and before `orders`. + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + const m = migrations[0] + expect(m.enabled).toBe(true) + expect(m.sql).toContain('"public"."customers"') + expect(m.sql).toContain('"public"."orders"') + expect(m.sql.indexOf('"public"."customers"')).toBeLessThan(m.sql.indexOf('"public"."orders"')) + }) + + it('drops a foreign key whose target is not in the schema', async () => { + // `orders` references a `warehouses` table that no longer exists in the + // schema: the FK must be pruned so the migration still runs. + const schemaWithDanglingFk = { + public: { + orders: { + name: 'orders', + columns: [{ name: 'id', datatype: 'integer', primary_key: true, nullable: false }], + foreign_keys: [ + { + target_table: 'public.warehouses', + columns: [{ source_column: 'id', target_column: 'id' }], + on_delete: 'NO ACTION', + on_update: 'NO ACTION' + } + ] + } + } + } + getDatatableFullSchemaMock.mockResolvedValue(schemaWithDanglingFk) + const usage = new Map([['main', new Set(['orders'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."orders"') + expect(migrations[0].sql).not.toContain('warehouses') + }) + + it('emits a disabled comment entry when a referenced table is not found', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['nonexistent'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations).toHaveLength(1) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('-- Table "nonexistent" is referenced but was not found') + expect(migrations[0].sql).not.toContain('BEGIN;') + }) + + it('keeps found tables and comments the missing ones in one migration', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set(['customers', 'ghost'])]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(true) + expect(migrations[0].sql).toContain('"public"."customers"') + expect(migrations[0].sql).toContain('-- Table "ghost" is referenced but was not found') + // Comments precede the runnable transaction. + expect(migrations[0].sql.indexOf('-- Table "ghost"')).toBeLessThan( + migrations[0].sql.indexOf('BEGIN;') + ) + }) + + it('comments a data table used with no specific table', async () => { + getDatatableFullSchemaMock.mockResolvedValue(schema) + const usage = new Map([['main', new Set()]]) + const migrations = await generateDatatableMigrations('ws', usage) + expect(migrations[0].enabled).toBe(false) + expect(migrations[0].sql).toContain('no specific table was referenced') + }) +}) diff --git a/frontend/src/lib/components/workspaceSettings/projectMigrations.ts b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts new file mode 100644 index 0000000000..b4e72389c0 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/projectMigrations.ts @@ -0,0 +1,345 @@ +// Best-effort data table migration generation for the "project = folder" Hub +// bundle. Detects which data tables (and tables within them) a project's +// scripts/flows/raw apps reference via `datatable` assets, then generates a +// `CREATE TABLE` bundle per data table from the source workspace's live schema, +// so importing the project into another workspace can recreate those tables. +// +// Best-effort by design: the generated SQL is shown to the publisher and is +// fully editable before publishing. Low-code (non-raw) apps have no persisted +// asset list and are not scanned. + +import { inferAssets } from '$lib/infer' +import type { SupportedLanguage } from '$lib/common' +import { allFlowModules } from './projectBundle' +import { getFlowModuleAssets } from '$lib/components/assets/lib' +import { extractDataConfig, parseDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' +import { + apiSchemaToEditorSchema, + generateAddedTableSql, + type DatabaseSchema +} from '$lib/components/datatableSchemaSql' +import { WorkspaceService } from '$lib/gen' +import type { FetchedItem } from './projectBundle' + +export interface GeneratedMigration { + datatable_name: string + /** Up migration: creates the tables. */ + sql: string + /** Down migration: drops the created tables. Best-effort, generated once and + * editable by the publisher (not re-derived from `sql`). */ + sql_down: string + enabled: boolean +} + +// A datatable asset path is `datatable`, `datatable/table`, or +// `datatable/schema.table` (see the SQL asset parser). The first segment is the +// data table name; the remainder identifies a specific table (absent = whole +// data table, no table to create). +function parseDatatableAssetPath(path: string): { datatable: string; table?: string } { + const slash = path.indexOf('/') + if (slash === -1) return { datatable: path } + const datatable = path.slice(0, slash) + const table = path.slice(slash + 1).trim() + return { datatable, table: table || undefined } +} + +function addDatatableTable( + map: Map>, + datatable: string, + table: string | undefined +): void { + if (!datatable) return + const set = map.get(datatable) ?? new Set() + if (table) set.add(table) + map.set(datatable, set) +} + +function addUsage(map: Map>, path: string): void { + const { datatable, table } = parseDatatableAssetPath(path) + addDatatableTable(map, datatable, table) +} + +/** + * Scan a project's fetched items for data table usage and return + * `datatable -> set of table refs` (a table ref is `table` or `schema.table`). + * - scripts: re-parse the code with the asset parser (`inferAssets`) + * - flows: read each module's stored `assets` + * - full-code (raw) apps: read the explicit `data.tables` declaration; fall back + * to `runnables[key].inlineScript.assets` for older apps + */ +export async function detectDatatableTables( + items: FetchedItem[] +): Promise>> { + const map = new Map>() + + for (const item of items) { + if (item.kind === 'script') { + const res = await inferAssets( + item.language as SupportedLanguage | undefined, + item.content ?? '' + ) + if (res.status === 'ok') { + for (const a of res.assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'flow') { + for (const mod of allFlowModules(item.value)) { + const assets = getFlowModuleAssets(mod) + if (assets) for (const a of assets) if (a.kind === 'datatable') addUsage(map, a.path) + } + } else if (item.kind === 'raw_app') { + let parsed: any + try { + parsed = JSON.parse(item.content ?? '{}') + } catch { + continue + } + // Full-code apps explicitly declare the data tables/tables they use + // (`data.tables`, refs like `main/customers` or `main/schema:table`), so + // read that rather than parsing assets. + const config = extractDataConfig(parsed) + if (config) { + for (const ref of config.tables) { + const r = parseDataTableRef(ref) + const table = r.table + ? r.schema && r.schema !== 'public' + ? `${r.schema}.${r.table}` + : r.table + : undefined + addDatatableTable(map, r.datatable, table) + } + } + // Older raw apps instead carry datatable usage as inline-script assets. + const runnables = parsed?.runnables ?? {} + for (const key of Object.keys(runnables)) { + const assets = runnables[key]?.inlineScript?.assets + if (Array.isArray(assets)) + for (const a of assets) + if (a?.kind === 'datatable' && typeof a.path === 'string') addUsage(map, a.path) + } + } + } + return map +} + +// Resolve a table ref (`table` or `schema.table`) to a concrete +// `{ schemaName, tableName }` present in the live schema, or undefined if the +// table can't be found (dropped since, typo, …). A schema-qualified ref that +// misses stays unresolved: falling back to a same-named table in another +// schema would generate a migration for an unrelated table while the code +// still references the missing one. +function resolveTable( + schema: DatabaseSchema, + tableRef: string +): { schemaName: string; tableName: string } | undefined { + const dot = tableRef.indexOf('.') + if (dot !== -1) { + const schemaName = tableRef.slice(0, dot) + const tableName = tableRef.slice(dot + 1) + return schema[schemaName]?.[tableName] ? { schemaName, tableName } : undefined + } + // Bare name: find it across every schema, first match wins. + for (const schemaName of Object.keys(schema)) { + if (schema[schemaName][tableRef]) return { schemaName, tableName: tableRef } + } + return undefined +} + +type ResolvedTable = { schemaName: string; tableName: string } + +const tableKey = (t: ResolvedTable) => `${t.schemaName}.${t.tableName}` + +// Grow the set of tables to create so it's closed under foreign keys: a used +// table's FK targets (and their FK targets, transitively) are pulled in, so the +// generated CREATE TABLEs never reference a table that isn't also created. FK +// targets that don't resolve in this schema are left out (their FK is pruned by +// pruneSchemaForTables). +function expandFkClosure(schema: DatabaseSchema, seed: ResolvedTable[]): ResolvedTable[] { + const inSet = new Map(seed.map((t) => [tableKey(t), t])) + const queue = [...seed] + while (queue.length > 0) { + const t = queue.shift()! + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && !inSet.has(tableKey(target))) { + inSet.set(tableKey(target), target) + queue.push(target) + } + } + } + return [...inSet.values()] +} + +// A copy of the schema restricted to `tables`, with each table's foreign keys +// filtered to targets that are also in `tables`. generateAddedTableSql emits every +// FK it finds on a table, so pruning here keeps a stray FK (to a table outside the +// migration) from making the generated SQL fail. +function pruneSchemaForTables(schema: DatabaseSchema, tables: ResolvedTable[]): DatabaseSchema { + const inSet = new Set(tables.map(tableKey)) + const pruned: DatabaseSchema = {} + for (const t of tables) { + const orig = schema[t.schemaName]?.[t.tableName] + if (!orig) continue + ;(pruned[t.schemaName] ??= {})[t.tableName] = { + ...orig, + foreignKeys: (orig.foreignKeys ?? []).filter((fk) => { + const target = resolveTable(schema, fk.targetTable ?? '') + return target != null && inSet.has(tableKey(target)) + }) + } + } + return pruned +} + +// Order tables so a table is created after the in-set tables it references via a +// foreign key. Keyed by schema-qualified name (like the rest of the pipeline) so +// two same-named tables in different schemas aren't collapsed. Falls back to input +// order on a cycle so generation never hangs. +function orderByFkDependency(schema: DatabaseSchema, tables: ResolvedTable[]): ResolvedTable[] { + const inSet = new Set(tables.map(tableKey)) + const deps = new Map>() + for (const t of tables) { + const fks = schema[t.schemaName]?.[t.tableName]?.foreignKeys ?? [] + const targets = new Set() + for (const fk of fks) { + const target = resolveTable(schema, fk.targetTable ?? '') + if (target && tableKey(target) !== tableKey(t) && inSet.has(tableKey(target))) { + targets.add(tableKey(target)) + } + } + deps.set(tableKey(t), targets) + } + const ordered: ResolvedTable[] = [] + const done = new Set() + const visiting = new Set() + const byKey = new Map(tables.map((t) => [tableKey(t), t])) + const visit = (key: string) => { + if (done.has(key) || visiting.has(key)) return + visiting.add(key) + for (const dep of deps.get(key) ?? []) visit(dep) + visiting.delete(key) + done.add(key) + const t = byKey.get(key) + if (t) ordered.push(t) + } + for (const t of tables) visit(tableKey(t)) + return ordered +} + +// Pull a readable one-line message out of an API error for embedding in a SQL +// comment (collapse whitespace so it can't break out of the `--` line). +function errorText(e: any): string { + const body = e?.body + const raw = + typeof body === 'string' && body.trim() + ? body + : body && typeof body === 'object' + ? (body.error?.message ?? body.message ?? JSON.stringify(body)) + : (e?.message ?? String(e)) + return String(raw).replace(/\s+/g, ' ').trim() +} + +/** + * Generate one best-effort migration per used data table. Resolved tables (plus + * the tables they depend on via foreign key, in FK-dependency order) become a + * single CREATE TABLE transaction, enabled by default. Anything that couldn't be + * auto-generated — a table not found in the schema, a data table referenced as a + * whole, or a schema that couldn't be loaded — is written as a `--` SQL comment + * describing the problem, so the publisher sees what's missing instead of a blank + * entry. A migration with no runnable statements (only comments) is left disabled. + */ +export async function generateDatatableMigrations( + workspace: string, + usage: Map> +): Promise { + const out: GeneratedMigration[] = [] + for (const [datatable, tableRefs] of usage) { + let schema: DatabaseSchema + try { + const api = await WorkspaceService.getDatatableFullSchema({ + workspace, + requestBody: { source: `datatable://${datatable}` } + }) + schema = apiSchemaToEditorSchema(api) + } catch (e) { + // Couldn't reach the schema at all: leave a commented stub explaining why, + // so the publisher can fill it in rather than seeing a silent blank. + out.push({ + datatable_name: datatable, + sql: + `-- Could not load the schema of data table "${datatable}": ${errorText(e)}\n` + + `-- Add the CREATE TABLE statement(s) for the tables this project uses.`, + sql_down: '', + enabled: false + }) + continue + } + // Resolve the referenced tables; record a comment for each one we can't find + // so a partial migration still explains what's missing. + const resolved: ResolvedTable[] = [] + const comments: string[] = [] + for (const ref of tableRefs) { + const t = resolveTable(schema, ref) + if (t) resolved.push(t) + else + comments.push( + `-- Table "${ref}" is referenced but was not found in data table "${datatable}"; add its CREATE TABLE manually.` + ) + } + if (tableRefs.size === 0) { + comments.push( + `-- Data table "${datatable}" is used but no specific table was referenced; nothing to generate automatically.` + ) + } + // Pull in the tables the referenced ones depend on via FK, then generate + // against a schema whose FKs are restricted to this set, so the migration + // creates everything it references and never emits a dangling FK. + const closure = expandFkClosure(schema, resolved) + const ordered = orderByFkDependency(schema, closure) + const prunedSchema = pruneSchemaForTables(schema, ordered) + // Every CREATE TABLE is emitted before any FK constraint: circular FKs have + // no valid creation order, so constraints can only run once all tables exist. + const creates: string[] = [] + const constraints: string[] = [] + for (const t of ordered) { + // IF NOT EXISTS: FK closure pulls in shared parent tables (e.g. a + // referenced `orders` drags in `customers`) that often already exist in + // the target, so a plain CREATE would abort the whole transaction. The + // caveat — an existing differently-shaped table is silently left as-is — + // is acceptable for a best-effort, editable migration. + const gen = generateAddedTableSql( + { schemaName: t.schemaName, tableName: t.tableName, kind: 'added' }, + prunedSchema, + { ifNotExists: true } + ) + if (!gen) continue + creates.push(gen.create) + constraints.push(...gen.constraints) + } + const statements = [...creates, ...constraints] + // Comments (the errors) go on top; the CREATE TABLE transaction, if any, + // follows. Enabled only when there's something to run. + const parts: string[] = [] + if (comments.length > 0) parts.push(comments.join('\n')) + if (statements.length > 0) parts.push(`BEGIN;\n${statements.join('\n\n')}\nCOMMIT;`) + // Best-effort down migration: the DROP TABLE statements are commented out + // because the FK closure pulls in shared parent tables that may have + // pre-existed in the target (dropping them would lose data the project never + // created). The publisher uncomments the tables this migration should drop. + const drops = [...ordered] + .reverse() + .map((t) => `-- DROP TABLE IF EXISTS "${t.schemaName}"."${t.tableName}";`) + const sqlDown = + drops.length > 0 + ? `-- Rollback: uncomment the tables this migration should drop (leave shared\n` + + `-- tables that already existed in the workspace commented out).\nBEGIN;\n${drops.join('\n')}\nCOMMIT;` + : '' + out.push({ + datatable_name: datatable, + sql: parts.join('\n\n'), + sql_down: sqlDown, + enabled: statements.length > 0 + }) + } + return out.sort((a, b) => a.datatable_name.localeCompare(b.datatable_name)) +} diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index 42de9d38a5..343e7a3e48 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -14,7 +14,8 @@ import { sendUserToast } from '$lib/utils' import DataTable from '$lib/components/table/DataTable.svelte' import Cell from '$lib/components/table/Cell.svelte' - import { Pen, Trash, Plus } from 'lucide-svelte' + import { Pen, Trash, Plus, UploadCloud } from 'lucide-svelte' + import DeployToHub from '$lib/components/workspaceSettings/DeployToHub.svelte' import Head from '$lib/components/table/Head.svelte' import Row from '$lib/components/table/Row.svelte' import Badge from '$lib/components/common/badge/Badge.svelte' @@ -30,6 +31,8 @@ let newFolderName: string = $state('') let folders: FolderW[] | undefined = $state(undefined) let folderDrawer: Drawer | undefined = $state() + let hubDrawer: Drawer | undefined = $state() + let publishFolderName: string = $state('') async function loadFolders(): Promise { folders = (await FolderService.listFolders({ workspace: $workspaceStore! })).map((x) => { @@ -88,6 +91,22 @@ + + { + hubDrawer?.closeDrawer() + publishFolderName = '' + }} + > + {#if publishFolderName} + {#key publishFolderName} + + {/key} + {/if} + + + {#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.folders}