From 6206aa1fff47ddc4b1835272c231d3ec4a49b47b Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Tue, 29 Sep 2026 17:25:46 +0200 Subject: [PATCH] feat: filter resource types by hub category and tag the most picked Co-Authored-By: Claude Opus 5.5 (1M context) --- backend/windmill-api/openapi.yaml | 26 +++ backend/windmill-store/src/resources.rs | 125 +++++++++++ .../src/lib/components/AppConnectInner.svelte | 196 +++++++++--------- .../src/lib/components/pickerPopularity.ts | 24 +++ .../lib/components/resourceTypeCategories.ts | 121 ----------- 5 files changed, 270 insertions(+), 222 deletions(-) delete mode 100644 frontend/src/lib/components/resourceTypeCategories.ts diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 85ae9388c8..1eec8dcf2b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9491,6 +9491,32 @@ paths: - app - picks + /w/{workspace}/resources/type/hub/categories: + get: + summary: list the hub category of each resource type + operationId: listHubResourceTypeCategories + tags: + - resource + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: each hub resource type whose integration has a category on the hub, empty if the hub cannot be reached + content: + application/json: + schema: + type: array + items: + type: object + properties: + name: + type: string + category: + type: string + required: + - name + - category + /w/{workspace}/resources/type/hub/pick/{name}: post: summary: record a hub resource type pick diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 5bb15b456e..0047e2a0b8 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -92,6 +92,10 @@ pub fn workspaced_service() -> Router { .route("/type/listnames", get(list_resource_types_names)) .route("/type/resource_counts", get(list_resource_counts_by_type)) .route("/type/hub/info", get(list_hub_resource_type_info)) + .route( + "/type/hub/categories", + get(list_hub_resource_type_categories), + ) .route("/type/hub/pick/{name}", post(pick_hub_resource_type)) .route("/type/get/{name}", get(get_resource_type)) .route("/type/exists/{name}", get(exists_resource_type)) @@ -3112,6 +3116,127 @@ async fn list_hub_resource_type_info( Ok(Json(info)) } +/// Categories are editorial and change rarely, and reading them fans out one hub request per +/// integration — so they are kept for a day. A read that found nothing is retried sooner, +/// without costing every drawer open a fan-out against a hub that has no categories. +const HUB_APP_CATEGORIES_TTL: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); +const HUB_APP_CATEGORIES_EMPTY_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const HUB_APP_CATEGORIES_CONCURRENCY: usize = 16; + +static HUB_APP_CATEGORIES: LazyLock>>>> = + LazyLock::new(|| std::sync::RwLock::new(None)); +/// Drawers opened while the fan-out runs wait for it rather than each starting their own. +static HUB_APP_CATEGORIES_REFRESH: LazyLock> = + LazyLock::new(|| tokio::sync::Mutex::new(())); + +#[derive(Deserialize)] +struct HubIntegrationContent { + content: HubIntegrationContentBody, +} + +#[derive(Deserialize)] +struct HubIntegrationContentBody { + #[serde(default)] + category: Option, +} + +fn hub_app_categories_cached(hub_base_url: &str) -> Option> { + let guard = HUB_APP_CATEGORIES.read().ok()?; + let entry = guard.as_ref()?; + let ttl = if entry.value.is_empty() { + HUB_APP_CATEGORIES_EMPTY_TTL + } else { + HUB_APP_CATEGORIES_TTL + }; + (entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl) + .then(|| entry.value.clone()) +} + +/// The hub's category for each integration that has one, keyed by the integration name as +/// the resource type index spells it. The hub only serves categories one integration at a +/// time (`/integrations//content`, 404 when it has none). +async fn hub_app_categories( + db: &DB, + hub_base_url: &str, + apps: HashSet, +) -> HashMap { + if let Some(cached) = hub_app_categories_cached(hub_base_url) { + return cached; + } + let _refresh = HUB_APP_CATEGORIES_REFRESH.lock().await; + if let Some(cached) = hub_app_categories_cached(hub_base_url) { + return cached; + } + + use futures::StreamExt; + // The name goes into a URL path; a hub slug never needs escaping, so anything that + // would is skipped rather than requested. + let slug = |app: &str| { + app.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + .then(|| app.to_lowercase()) + }; + let categories: HashMap = futures::stream::iter(apps) + .filter_map(|app| async move { slug(&app).map(|slug| (app, slug)) }) + .map(|(app, slug)| async move { + let response = windmill_common::utils::http_get_from_hub( + &windmill_common::utils::HTTP_CLIENT, + &format!("{hub_base_url}/integrations/{slug}/content"), + false, + None, + Some(db), + ) + .await + .ok()?; + if !response.status().is_success() { + return None; + } + let category = response + .json::() + .await + .ok()? + .content + .category?; + Some((app, category)) + }) + .buffer_unordered(HUB_APP_CATEGORIES_CONCURRENCY) + .filter_map(|entry| async move { entry }) + .collect() + .await; + + hub_cache_put(&HUB_APP_CATEGORIES, hub_base_url, categories.clone()); + categories +} + +#[derive(Serialize)] +struct HubResourceTypeCategory { + name: String, + category: String, +} + +/// The hub's category for each resource type, inherited from its integration. Types whose +/// integration has no category are left out; an unreachable hub yields an empty list. +async fn list_hub_resource_type_categories( + Extension(db): Extension, +) -> JsonResult> { + let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone(); + let index = hub_resource_types(&db, &hub_base_url) + .await + .unwrap_or_default(); + let apps = index.values().map(|rt| rt.app.clone()).collect(); + let categories = hub_app_categories(&db, &hub_base_url, apps).await; + + Ok(Json( + index + .into_iter() + .filter_map(|(name, rt)| { + let category = categories.get(&rt.app)?.clone(); + Some(HubResourceTypeCategory { name, category }) + }) + .collect(), + )) +} + async fn get_resource_type( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index ae100dd1bc..91f7fd8ed6 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -23,7 +23,6 @@ } from '$lib/gen' import { emptyString, truncateRev, urlize } from '$lib/utils' import { registryEntryFor, registryCcCapableFor, stripSandboxSuffix } from './oauthRegistry' - import { RESOURCE_TYPE_CATEGORIES, resourceTypeCategory } from './resourceTypeCategories' import { createEventDispatcher, onDestroy, tick } from 'svelte' import Path from './Path.svelte' import { Badge, Button, ListRow, RadioCard, Skeleton } from './common' @@ -47,6 +46,7 @@ import { alphabetical, byPopularity, + hubResourceTypeCategories, hubResourceTypePicks, localResourceTypeCounts, recordHubResourceTypePick @@ -107,7 +107,7 @@ /** An instance entry with shared credentials (admin id+secret): connect with * no input. Shown under "Instance-configured"; bring-your-own-only providers - * (no shared creds) are listed with the other resource types instead. */ + * (no shared creds) are shown under "Others" instead. */ function isSharedConnect(key: string): boolean { return connectsInfo[key]?.has_shared_credentials ?? false } @@ -257,9 +257,8 @@ * doesn't enter their own — the exchange runs server-side with those creds */ let ccInstanceConfigured = $state(false) - /** The user wants their own credentials (picked the provider from the resource - * type list rather than the instance section) — overrides the shared instance - * credentials for this connection */ + /** The user wants their own credentials (picked the provider from the "Others" + * section) — overrides the shared instance credentials for this connection */ let ccBringYourOwn = $state(false) /** Connect with the shared instance credentials (no form) rather than the @@ -325,7 +324,7 @@ return registryCcCapableFor(key) } - /** Step-1 resource type selection: CC-capable resource types open the client- + /** Step-1 "Others" selection: CC-capable resource types open the client- * credentials form with the user's own credentials — even when the instance * has shared ones (the "Instance-configured OAuth APIs" section is the entry * point for those). Every other type opens the raw manual form. */ @@ -413,14 +412,31 @@ * alphabetically and re-sort when this lands. */ let popularity: (a: string, b: string) => number = $state(alphabetical) + /** The hub's most picked types, tagged "Popular" wherever they land in the list. */ + let hubPopular: Set = $state(new Set()) + let hubPicks: Record = $state({}) + /** Hub category per type; fetched apart from the picks since the first read is slow. */ + let categories: Record = $state({}) + let selectedCategory: string | undefined = $state(undefined) + + const POPULAR_TAG_COUNT = 10 async function loadPopularity() { if (!effectiveWorkspace) return + hubResourceTypeCategories(effectiveWorkspace).then((c) => (categories = c)) const [hub, local] = await Promise.all([ hubResourceTypePicks(effectiveWorkspace), localResourceTypeCounts(effectiveWorkspace) ]) popularity = byPopularity(hub, local) + hubPicks = hub + hubPopular = new Set( + Object.entries(hub) + .filter(([, picks]) => picks > 0) + .sort(([a, pa], [b, pb]) => pb - pa || a.localeCompare(b)) + .slice(0, POPULAR_TAG_COUNT) + .map(([name]) => name) + ) } async function loadConnects() { @@ -508,7 +524,7 @@ }) .catch(() => {}) - // The resource type list holds every type — including instance-configured OAuth + // "Others" lists every resource type — including instance-configured OAuth // providers — so any of them can also be connected with the user's own // credentials or manually, not only via the shared instance setup (same as // the authorization-code behavior). @@ -1032,6 +1048,8 @@ let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) let searching = $derived(filter.trim() !== '') + /** A narrowed list: sections that end up empty are dropped rather than explained. */ + let filtering = $derived(searching || selectedCategory !== undefined) // Searching, the query owns the order: uFuzzy scores the name and the description as one // string, so "google" ranks every type whose description mentions Google alongside the @@ -1060,71 +1078,45 @@ rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined ) - let manualOrderedKeys = $derived((rankedConnectsManual ?? []).map((x) => x.key)) + const categoryOf = (key: string): string | undefined => categories[stripSandboxSuffix(key)] + const inSelectedCategory = (key: string) => + selectedCategory === undefined || categoryOf(key) === selectedCategory + + // The filters on offer: every category some listed type falls in, the most picked first + // so the ones people reach for lead the row. + let categoryFilters = $derived.by(() => { + const picks = new Map() + for (const { key } of connectsManual ?? []) { + const category = categoryOf(key) + if (category) picks.set(category, (picks.get(category) ?? 0) + (hubPicks[key] ?? 0)) + } + return [...picks.entries()] + .sort(([a, pa], [b, pb]) => pb - pa || a.localeCompare(b)) + .map(([category]) => category) + }) + + let manualOrderedKeys = $derived( + (rankedConnectsManual ?? []).map((x) => x.key).filter(inSelectedCategory) + ) + let filteredRankedConnects = $derived(rankedConnects?.filter((x) => inSelectedCategory(x.key))) let customKeys = $derived(manualOrderedKeys.filter((key) => customResourceTypes.has(key))) let otherKeys = $derived(manualOrderedKeys.filter((key) => !customResourceTypes.has(key))) - // Browsing, the common types are grouped into categories ahead of "Others". Searching, - // that grouping would outrank the search itself — a weak Databases match sorting above - // the exact hit in Others — so the ranked order stands on its own. - let manualSections = $derived.by(() => { - type Section = { title: string; popular: boolean; keys: string[]; seeMore?: string } - const others: Section = { - title: 'Others', - popular: false, - keys: searching ? otherKeys : otherKeys.filter((key) => !resourceTypeCategory(key)) - } - if (searching) return [others] - const keysIn = (title: string, popular: boolean) => - otherKeys.filter((key) => { - const category = resourceTypeCategory(key) - return category?.title === title && category.popular === popular - }) - const popular: Section[] = [] - const rest: Section[] = [] - for (const category of RESOURCE_TYPE_CATEGORIES) { - const restKeys = keysIn(category.title, false) - const popularKeys = keysIn(category.title, true) - if (restKeys.length > 0) rest.push({ title: category.title, popular: false, keys: restKeys }) - if (popularKeys.length > 0) - popular.push({ - title: category.title, - popular: true, - keys: popularKeys, - seeMore: restKeys.length > 0 ? `See ${category.others}` : undefined - }) - } - // An empty Others still renders while loading (its skeletons) and on an unsynced - // list (the sync hint). - const keepOthers = - others.keys.length > 0 || !rankedConnectsManual || (connectsManual?.length ?? 0) < 10 - return keepOthers ? [...popular, ...rest, others] : [...popular, ...rest] - }) - // Every row in the order it is rendered, so arrow keys walk the sections as one list. // A provider appears in more than one, so rows are addressed by index, not by name. let navItems = $derived([ ...customKeys.map((key) => ({ key, oauth: false })), - ...(rankedConnects ?? []).map((x) => ({ key: x.key, oauth: true })), - ...manualSections.flatMap((section) => section.keys.map((key) => ({ key, oauth: false }))) + ...(filteredRankedConnects ?? []).map((x) => ({ key: x.key, oauth: true })), + ...otherKeys.map((key) => ({ key, oauth: false })) ]) // Both lists start undefined and render skeletons; "nothing found" only means something // once they have landed. let listsLoaded = $derived(rankedConnectsManual !== undefined && rankedConnects !== undefined) const rowDomId = (index: number) => `resource-type-row-${index}` - const categorySectionId = (title: string) => - `resource-type-category-${title.toLowerCase().replace(/[^a-z]+/g, '-')}` const oauthRowOffset = $derived(customKeys.length) - const manualRowOffsets = $derived.by(() => { - let offset = customKeys.length + (rankedConnects?.length ?? 0) - return manualSections.map((section) => { - const start = offset - offset += section.keys.length - return start - }) - }) + const otherRowOffset = $derived(customKeys.length + (filteredRankedConnects?.length ?? 0)) // Sections are rendered in a fixed order, so the best match is not necessarily the first // row: rank the rows against the query to find it. @@ -1201,12 +1193,26 @@ class="pl-7 text-xs w-full" /> + {#if categoryFilters.length > 0} +
+ {#each [undefined, ...categoryFilters] as category (category ?? '')} + + {/each} +
+ {/if} - {#snippet sectionHeading(title: string, count: number, popular: boolean = false)} -

- {title}{#if popular}Popular{/if}{#if searching}{count}{/if} + {#snippet sectionHeading(title: string, count: number)} +

+ {title}{#if filtering}{count}{/if}

{/snippet} @@ -1217,6 +1223,9 @@ {#snippet title()} {resourceTypeDisplayName(key)} {key} + {#if hubPopular.has(stripSandboxSuffix(key))} + Popular + {/if} {/snippet} {#snippet subtitle()} {plainDescription(resourceTypeDescriptions[key])} @@ -1249,7 +1258,7 @@ {:else} -
+
{#if customKeys.length > 0}
{@render sectionHeading('Custom resource types', customKeys.length)} @@ -1261,15 +1270,15 @@
{/if} - {#if !searching || (rankedConnects?.length ?? 0) > 0} + {#if !filtering || (filteredRankedConnects?.length ?? 0) > 0}
{@render sectionHeading( 'Instance-configured OAuth APIs', - rankedConnects?.length ?? 0 + filteredRankedConnects?.length ?? 0 )}
- {#if rankedConnects} - {#each rankedConnects as { key }, i} + {#if filteredRankedConnects} + {#each filteredRankedConnects as { key }, i} {@render resourceButton(key, oauthRowOffset + i, true)} {/each} {:else} @@ -1289,44 +1298,29 @@
{/if} - {#each manualSections as section, s (`${section.title}-${section.popular}`)} - {#if !searching || section.keys.length > 0} -
- {@render sectionHeading(section.title, section.keys.length, section.popular)} + {#if !filtering || otherKeys.length > 0} +
+ {@render sectionHeading(selectedCategory ?? 'Others', otherKeys.length)} - {#if section.title === 'Others' && !searching && connectsManual && connectsManual?.length < 10} -
- Resource types have not been synced with the hub -
- {/if} - -
- {#if rankedConnectsManual} - {#each section.keys as key, i} - {@render resourceButton(key, manualRowOffsets[s] + i, false)} - {/each} - {:else} - {#each new Array(9) as _} - - {/each} - {/if} + {#if !searching && connectsManual && connectsManual?.length < 10} +
+ Resource types have not been synced with the hub
- {#if section.seeMore} - + {/if} + +
+ {#if rankedConnectsManual} + {#each otherKeys as key, i} + {@render resourceButton(key, otherRowOffset + i, false)} + {/each} + {:else} + {#each new Array(9) as _} + + {/each} {/if} -
- {/if} - {/each} +
+ + {/if}
{/if} diff --git a/frontend/src/lib/components/pickerPopularity.ts b/frontend/src/lib/components/pickerPopularity.ts index 2d585ff858..04032d3a85 100644 --- a/frontend/src/lib/components/pickerPopularity.ts +++ b/frontend/src/lib/components/pickerPopularity.ts @@ -53,6 +53,30 @@ export async function hubResourceTypePicks(workspace: string): Promise [rt.name, rt.picks])) } +const hubCategoriesCached = createCache( + async ({ workspace }: { workspace: string }): Promise> => { + try { + const categories = await ResourceService.listHubResourceTypeCategories({ workspace }) + return Object.fromEntries(categories.map((c) => [c.name, c.category])) + } catch { + return {} + } + }, + { invalidateMs: CACHE_MS } +) + +/** + * The hub's category for each resource type ("Databases", "AI", …), inherited from its + * integration. Types whose integration has none are absent, as is everything when the hub is + * switched off. + */ +export async function hubResourceTypeCategories( + workspace: string +): Promise> { + if (get(disableHubStore)) return {} + return hubCategoriesCached({ workspace }) +} + /** * How many resources of each type this workspace holds — the only evidence about this * particular team. Keyed by resource type, which is what the add-resource drawer lists. diff --git a/frontend/src/lib/components/resourceTypeCategories.ts b/frontend/src/lib/components/resourceTypeCategories.ts deleted file mode 100644 index 246e27cd24..0000000000 --- a/frontend/src/lib/components/resourceTypeCategories.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { stripSandboxSuffix } from './oauthRegistry' - -/** Hand-picked sections for the resource types people reach for most, in the order the add - * resource drawer renders them. `popular` holds the handful picked most of the time and is - * listed ahead of every full section; `keys` holds the rest of the category. Everything not - * listed falls under "Others". Keys are hub resource type names; a key the instance has not - * synced simply renders nothing. */ -export const RESOURCE_TYPE_CATEGORIES: { - title: string - /** Names the rest of the category on the link from its popular section. */ - others: string - popular: string[] - keys: string[] -}[] = [ - { - title: 'Databases', - others: 'other databases', - popular: [ - 'postgresql', - 'mysql', - 'ms_sql_server', - 'bigquery', - 'snowflake', - 'mongodb', - 'supabase', - 'neondb' - ], - keys: [ - 'snowflake_oauth', - 'oracledb', - 'redshift', - 'clickhouse', - 'databricks', - 'cockroachdb', - 'planetscale', - 'turso', - 'firebase', - 'surrealdb' - ] - }, - { - title: 'AI', - others: 'other AI providers', - popular: ['openai', 'anthropic', 'googleai', 'mistral', 'deepseek', 'groq'], - keys: [ - 'azure_openai', - 'aws_bedrock', - 'azure_foundry', - 'openrouter', - 'togetherai', - 'cohere', - 'deep_infra', - 'customai', - 'pinecone', - 'chromadb' - ] - }, - { - title: 'Object storage', - others: 'other object storage', - popular: ['s3', 'azure_blob', 'gcloud_storage'], - keys: ['s3_aws_oidc'] - }, - { - title: 'Cloud providers', - others: 'other cloud providers', - popular: ['aws', 'gcloud', 'gcp_service_account', 'azure'], - keys: ['aws_oidc', 'azure_oauth', 'azure_workload_identity', 'cloudflare', 'digitalocean'] - }, - { - title: 'Messaging & email', - others: 'other messaging & email', - popular: ['slack', 'ms_teams_webhook', 'smtp', 'gmail', 'discord_webhook', 'telegram'], - keys: [ - 'discord_bot_configuration', - 'whatsapp_business', - 'twilio', - 'sendgrid', - 'resend', - 'mailgun' - ] - }, - { - title: 'Event streaming', - others: 'other event streaming', - popular: ['kafka', 'nats', 'mqtt'], - keys: ['amqp'] - }, - { - title: 'Productivity', - others: 'other productivity tools', - popular: ['gsheets', 'gdrive', 'notion', 'airtable', 'jira'], - keys: ['gcal', 'gdocs', 'confluence', 'linear', 'asana', 'trello', 'clickup'] - }, - { - title: 'Developer tools', - others: 'other developer tools', - popular: ['github', 'gitlab'], - keys: ['github_app', 'bitbucket', 'git_repository', 'sentry', 'datadog'] - }, - { - title: 'CRM & payments', - others: 'other CRM & payments', - popular: ['hubspot', 'salesforce', 'stripe'], - keys: ['shopify', 'zendesk'] - } -] - -type ResourceTypeCategory = { title: string; popular: boolean } - -const categoryByKey = new Map( - RESOURCE_TYPE_CATEGORIES.flatMap(({ title, popular, keys }) => [ - ...popular.map((key): [string, ResourceTypeCategory] => [key, { title, popular: true }]), - ...keys.map((key): [string, ResourceTypeCategory] => [key, { title, popular: false }]) - ]) -) - -/** A sandbox client (`salesforce_sandbox`) files under its parent's section. */ -export function resourceTypeCategory(key: string): ResourceTypeCategory | undefined { - return categoryByKey.get(stripSandboxSuffix(key)) -}