diff --git a/backend/migrations/20250625100238_assets.down.sql b/backend/migrations/20250625100238_assets.down.sql index 8bb3a65690..3d904ff2d6 100644 --- a/backend/migrations/20250625100238_assets.down.sql +++ b/backend/migrations/20250625100238_assets.down.sql @@ -1,3 +1,3 @@ -DROP TABLE assets; +DROP TABLE asset; DROP TYPE ASSET_USAGE_KIND; DROP TYPE ASSET_KIND; \ No newline at end of file diff --git a/backend/migrations/20250625100238_assets.up.sql b/backend/migrations/20250625100238_assets.up.sql index 1062b67c3d..ca29e724ed 100644 --- a/backend/migrations/20250625100238_assets.up.sql +++ b/backend/migrations/20250625100238_assets.up.sql @@ -1,7 +1,8 @@ -CREATE TYPE ASSET_USAGE_KIND AS ENUM ('script', 'flow', 'flow_step'); -CREATE TYPE ASSET_KIND AS ENUM ('s3_object', 'resource'); +CREATE TYPE ASSET_USAGE_KIND AS ENUM ('script', 'flow'); +CREATE TYPE ASSET_KIND AS ENUM ('s3object', 'resource'); -CREATE TABLE assets ( +CREATE TABLE asset ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, path VARCHAR(255) NOT NULL, kind ASSET_KIND NOT NULL, usage_path VARCHAR(255) NOT NULL, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2f19159c0e..da6df5d1e5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12942,6 +12942,71 @@ paths: schema: type: string + /w/{workspace}/assets/link: + post: + summary: Deletes all current assets of the corresponding entity and updates them to the new ones + operationId: link + tags: + - asset + parameters: + - $ref: '#/components/parameters/WorkspaceId' + requestBody: + description: link assets + required: true + content: + application/json: + schema: + type: object + properties: + assets: + type: array + items: + $ref: '#/components/schemas/Asset' + usage_path: + type: string + usage_kind: + $ref: '#/components/schemas/AssetUsageKind' + required: [assets, usage_path, usage_kind] + responses: + '201': + description: assets linked + + /w/{workspace}/assets/list: + get: + summary: List all assets in the workspace + operationId: list + tags: + - asset + parameters: + - $ref: '#/components/parameters/WorkspaceId' + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + responses: + '200': + description: assets linked + content: + application/json: + schema: + type: array + items: + type: object + required: [path, kind, usages] + properties: + path: + type: string + kind: + $ref: '#/components/schemas/AssetKind' + usages: + type: array + items: + type: object + required: [usage_path, usage_kind] + properties: + usage_path: + type: string + usage_kind: + $ref: '#/components/schemas/AssetUsageKind' + components: securitySchemes: bearerAuth: @@ -17164,3 +17229,21 @@ components: type: string description: Microsoft Teams channel name minLength: 1 + AssetUsageKind: + type: string + enum: + - script + - flow + AssetKind: + type: string + enum: + - s3object + - resource + Asset: + type: object + properties: + path: + type: string + kind: + $ref: '#/components/schemas/AssetKind' + required: [path, kind] diff --git a/backend/windmill-api/src/assets.rs b/backend/windmill-api/src/assets.rs new file mode 100644 index 0000000000..0071b89aae --- /dev/null +++ b/backend/windmill-api/src/assets.rs @@ -0,0 +1,114 @@ +use axum::{extract::{Path, Query}, routing::{post, get}, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sqlx::{Postgres, Transaction}; +use windmill_common::{db::UserDB, error::{JsonResult, Result}, utils::Pagination}; + +use crate::db::ApiAuthed; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/link", post(link_assets)) + .route("/list", get(list_assets)) +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetKind { + S3Object, + Resource, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type)] +#[sqlx(type_name = "ASSET_USAGE_KIND", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum AssetUsageKind { + Script, + Flow, +} + +#[derive(Deserialize)] +pub struct Asset { + pub path: String, + pub kind: AssetKind, +} + +#[derive(Deserialize)] +pub struct LinkAssetsBody { + pub assets: Vec, + pub usage_path: String, + pub usage_kind: AssetUsageKind, +} + +async fn link_assets( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Json(body): Json, +) -> JsonResult<()> { + let mut tx = user_db.begin(&authed).await?; + link_assets_internal(&mut tx, w_id, body).await?; + tx.commit().await?; + Ok(Json(())) +} + +async fn link_assets_internal( + tx: &mut Transaction<'_, Postgres>, + w_id: String, + body: LinkAssetsBody, +) -> Result<()> { + sqlx::query!( + r#"DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3;"#, + w_id, + body.usage_path, + body.usage_kind as AssetUsageKind + ) + .execute(&mut **tx) + .await?; + + for asset in body.assets { + sqlx::query!( + r#"INSERT INTO asset (workspace_id, path, kind, usage_path, usage_kind) VALUES ($1, $2, $3, $4, $5);"#, + w_id, + asset.path, + asset.kind as AssetKind, + body.usage_path, + body.usage_kind as AssetUsageKind + ) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + +async fn list_assets( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(pagination): Query +) -> JsonResult> { + let limit = pagination.per_page.unwrap_or(50).min(100); + let assets = sqlx::query_scalar!( + r#"SELECT + jsonb_build_object( + 'path', path, + 'kind', kind, + 'usages', ARRAY_AGG(jsonb_build_object( + 'usage_path', usage_path, + 'usage_kind', usage_kind + )) + ) as "list!: _" + FROM asset + WHERE workspace_id = $1 + GROUP BY path, kind + LIMIT $2 OFFSET $3"#, + w_id, + limit as i64, + (pagination.page.unwrap_or(1).saturating_sub(1) * limit) as i64 + ) + .fetch_all(&mut *user_db.begin(&authed).await?) + .await?; + + Ok(Json(assets)) +} \ No newline at end of file diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a415803339..9d75d6e6e3 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -70,6 +70,7 @@ mod agent_workers_oss; mod ai; mod apps; pub mod args; +mod assets; mod audit; pub mod auth; mod capture; @@ -560,6 +561,7 @@ pub async fn run_server( // Reordered alphabetically .nest("/acls", granular_acls::workspaced_service()) .nest("/apps", apps::workspaced_service()) + .nest("/assets", assets::workspaced_service()) .nest("/audit", audit::workspaced_service()) .nest("/capture", capture::workspaced_service()) .nest( diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index f3320126dd..6aae29beb2 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -11,9 +11,10 @@ type TriggersCount, PostgresTriggerService, CaptureService, - type ScriptLang + type ScriptLang, + AssetService } from '$lib/gen' - import { inferArgs } from '$lib/infer' + import { inferArgs, inferAssets } from '$lib/infer' import { initialCode } from '$lib/script_helpers' import AIFormSettings from './copilot/AIFormSettings.svelte' import { @@ -97,6 +98,7 @@ } from './triggers/utils' import DraftTriggersConfirmationModal from './common/confirmationModal/DraftTriggersConfirmationModal.svelte' import { Triggers } from './triggers/triggers.svelte' + import { parseAsset } from './assets/lib' interface Props { script: NewScript & { draft_triggers?: Trigger[] } @@ -562,6 +564,18 @@ ) } + const assets = (await inferAssets(script.language, script.content)) + .map(parseAsset) + .filter((a) => !!a) + await AssetService.link({ + workspace: $workspaceStore!, + requestBody: { + assets, + usage_kind: 'script', + usage_path: script.path + } + }) + const { draft_triggers: _, ...newScript } = structuredClone($state.snapshot(script)) savedScript = structuredClone($state.snapshot(newScript)) as NewScriptWithDraft setDraftTriggers([]) diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts new file mode 100644 index 0000000000..77a0b9dd40 --- /dev/null +++ b/frontend/src/lib/components/assets/lib.ts @@ -0,0 +1,20 @@ +import type { AssetKind } from '$lib/gen' + +export type Asset = { + path: string + kind: AssetKind +} + +export function parseAsset(asset: string): Asset | undefined { + if (asset.startsWith('$res:')) return { path: asset.substring(5), kind: 'resource' } + if (asset.startsWith('s3://')) return { path: asset.substring(5), kind: 's3object' } +} + +export function formatAsset(asset: Asset): string { + switch (asset.kind) { + case 'resource': + return `$res:${asset.path}` + case 's3object': + return `s3://${asset.path}` + } +} diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index 5399a3490e..6f565d1a69 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -33,7 +33,8 @@ Plus, Unplug, AlertCircle, - Database + Database, + Pyramid } from 'lucide-svelte' import UserMenu from './UserMenu.svelte' import DiscordIcon from '../icons/brands/Discord.svelte' @@ -178,6 +179,14 @@ disabled: $userStore?.operator, aiId: 'sidebar-menu-link-resources', aiDescription: 'Button to navigate to resources' + }, + { + label: 'Assets', + href: `${base}/assets`, + icon: Pyramid, + disabled: $userStore?.operator, + aiId: 'sidebar-menu-link-assets', + aiDescription: 'Button to navigate to assets' } ]) let defaultExtraTriggerLinks = $derived([ diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index 8435512789..9a486e66a1 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -1,5 +1,6 @@ // https://github.com/sveltejs/svelte/issues/14600 +import { untrack } from 'svelte' import type { StateStore } from './utils' export function withProps(component: Component, props: Props) { @@ -60,3 +61,34 @@ export function usePromise( return ret } + +export type UsePaginatedResult = { + items: T[] + status: 'loading' | 'error' | 'ok' + currentPage: number + loadMore: () => void +} + +export function usePaginated(query: (page: number) => Promise<{ items: T[] }>) { + let s: UsePaginatedResult = $state({ + items: [], + status: 'loading', + currentPage: 1, + loadMore: () => { + s.currentPage++ + s.status = 'loading' + promise.refresh() + } + }) + + const promise = usePromise(() => query(s.currentPage)) + $effect(() => { + if (promise.status === 'ok') { + untrack(() => { + s.status = promise.status + s.items = [...s.items, ...promise.value.items] + }) + } + }) + return s +} diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.js b/frontend/src/routes/(root)/(logged)/assets/+page.js new file mode 100644 index 0000000000..4b3138368d --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/assets/+page.js @@ -0,0 +1,5 @@ +export function load() { + return { + stuff: { title: 'Assets' } + } +} diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/+page.svelte new file mode 100644 index 0000000000..e37420a8a6 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/assets/+page.svelte @@ -0,0 +1,90 @@ + + +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.resources} + +{:else} + + + + + + Asset name + + + + + {#each assets.items as item} + {@const assetUri = formatAsset(item)} + + {assetUri} + + (viewOccurences = item)}> + {pluralize(item.usages.length, 'occurrence')} + + + + {/each} + + + + + + + + +{/if} + + (viewOccurences = undefined)} +> + (viewOccurences = undefined)}> + {#each viewOccurences?.usages ?? [] as u} +
+

{u.usage_kind}

+

{u.usage_path}

+
+ {/each} + +