feat: filter resource types by hub category and tag the most picked

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-09-29 17:25:46 +02:00
co-authored by Claude Opus 5.5
parent a29ce39dc2
commit 6206aa1fff
5 changed files with 270 additions and 222 deletions
+26
View File
@@ -9491,6 +9491,32 @@ paths:
- app
- picks
/w/{workspace}/resources/type/hub/categories:
get:
summary: list the hub category of each resource type
operationId: listHubResourceTypeCategories
tags:
- resource
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: each hub resource type whose integration has a category on the hub, empty if the hub cannot be reached
content:
application/json:
schema:
type: array
items:
type: object
properties:
name:
type: string
category:
type: string
required:
- name
- category
/w/{workspace}/resources/type/hub/pick/{name}:
post:
summary: record a hub resource type pick
+125
View File
@@ -92,6 +92,10 @@ pub fn workspaced_service() -> Router {
.route("/type/listnames", get(list_resource_types_names))
.route("/type/resource_counts", get(list_resource_counts_by_type))
.route("/type/hub/info", get(list_hub_resource_type_info))
.route(
"/type/hub/categories",
get(list_hub_resource_type_categories),
)
.route("/type/hub/pick/{name}", post(pick_hub_resource_type))
.route("/type/get/{name}", get(get_resource_type))
.route("/type/exists/{name}", get(exists_resource_type))
@@ -3112,6 +3116,127 @@ async fn list_hub_resource_type_info(
Ok(Json(info))
}
/// Categories are editorial and change rarely, and reading them fans out one hub request per
/// integration — so they are kept for a day. A read that found nothing is retried sooner,
/// without costing every drawer open a fan-out against a hub that has no categories.
const HUB_APP_CATEGORIES_TTL: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
const HUB_APP_CATEGORIES_EMPTY_TTL: std::time::Duration = std::time::Duration::from_secs(60 * 60);
const HUB_APP_CATEGORIES_CONCURRENCY: usize = 16;
static HUB_APP_CATEGORIES: LazyLock<std::sync::RwLock<Option<HubCached<HashMap<String, String>>>>> =
LazyLock::new(|| std::sync::RwLock::new(None));
/// Drawers opened while the fan-out runs wait for it rather than each starting their own.
static HUB_APP_CATEGORIES_REFRESH: LazyLock<tokio::sync::Mutex<()>> =
LazyLock::new(|| tokio::sync::Mutex::new(()));
#[derive(Deserialize)]
struct HubIntegrationContent {
content: HubIntegrationContentBody,
}
#[derive(Deserialize)]
struct HubIntegrationContentBody {
#[serde(default)]
category: Option<String>,
}
fn hub_app_categories_cached(hub_base_url: &str) -> Option<HashMap<String, String>> {
let guard = HUB_APP_CATEGORIES.read().ok()?;
let entry = guard.as_ref()?;
let ttl = if entry.value.is_empty() {
HUB_APP_CATEGORIES_EMPTY_TTL
} else {
HUB_APP_CATEGORIES_TTL
};
(entry.hub_base_url == hub_base_url && entry.fetched_at.elapsed() < ttl)
.then(|| entry.value.clone())
}
/// The hub's category for each integration that has one, keyed by the integration name as
/// the resource type index spells it. The hub only serves categories one integration at a
/// time (`/integrations/<app>/content`, 404 when it has none).
async fn hub_app_categories(
db: &DB,
hub_base_url: &str,
apps: HashSet<String>,
) -> HashMap<String, String> {
if let Some(cached) = hub_app_categories_cached(hub_base_url) {
return cached;
}
let _refresh = HUB_APP_CATEGORIES_REFRESH.lock().await;
if let Some(cached) = hub_app_categories_cached(hub_base_url) {
return cached;
}
use futures::StreamExt;
// The name goes into a URL path; a hub slug never needs escaping, so anything that
// would is skipped rather than requested.
let slug = |app: &str| {
app.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
.then(|| app.to_lowercase())
};
let categories: HashMap<String, String> = futures::stream::iter(apps)
.filter_map(|app| async move { slug(&app).map(|slug| (app, slug)) })
.map(|(app, slug)| async move {
let response = windmill_common::utils::http_get_from_hub(
&windmill_common::utils::HTTP_CLIENT,
&format!("{hub_base_url}/integrations/{slug}/content"),
false,
None,
Some(db),
)
.await
.ok()?;
if !response.status().is_success() {
return None;
}
let category = response
.json::<HubIntegrationContent>()
.await
.ok()?
.content
.category?;
Some((app, category))
})
.buffer_unordered(HUB_APP_CATEGORIES_CONCURRENCY)
.filter_map(|entry| async move { entry })
.collect()
.await;
hub_cache_put(&HUB_APP_CATEGORIES, hub_base_url, categories.clone());
categories
}
#[derive(Serialize)]
struct HubResourceTypeCategory {
name: String,
category: String,
}
/// The hub's category for each resource type, inherited from its integration. Types whose
/// integration has no category are left out; an unreachable hub yields an empty list.
async fn list_hub_resource_type_categories(
Extension(db): Extension<DB>,
) -> JsonResult<Vec<HubResourceTypeCategory>> {
let hub_base_url = (**windmill_common::HUB_BASE_URL.load()).clone();
let index = hub_resource_types(&db, &hub_base_url)
.await
.unwrap_or_default();
let apps = index.values().map(|rt| rt.app.clone()).collect();
let categories = hub_app_categories(&db, &hub_base_url, apps).await;
Ok(Json(
index
.into_iter()
.filter_map(|(name, rt)| {
let category = categories.get(&rt.app)?.clone();
Some(HubResourceTypeCategory { name, category })
})
.collect(),
))
}
async fn get_resource_type(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -23,7 +23,6 @@
} from '$lib/gen'
import { emptyString, truncateRev, urlize } from '$lib/utils'
import { registryEntryFor, registryCcCapableFor, stripSandboxSuffix } from './oauthRegistry'
import { RESOURCE_TYPE_CATEGORIES, resourceTypeCategory } from './resourceTypeCategories'
import { createEventDispatcher, onDestroy, tick } from 'svelte'
import Path from './Path.svelte'
import { Badge, Button, ListRow, RadioCard, Skeleton } from './common'
@@ -47,6 +46,7 @@
import {
alphabetical,
byPopularity,
hubResourceTypeCategories,
hubResourceTypePicks,
localResourceTypeCounts,
recordHubResourceTypePick
@@ -107,7 +107,7 @@
/** An instance entry with shared credentials (admin id+secret): connect with
* no input. Shown under "Instance-configured"; bring-your-own-only providers
* (no shared creds) are listed with the other resource types instead. */
* (no shared creds) are shown under "Others" instead. */
function isSharedConnect(key: string): boolean {
return connectsInfo[key]?.has_shared_credentials ?? false
}
@@ -257,9 +257,8 @@
* doesn't enter their own — the exchange runs server-side with those creds */
let ccInstanceConfigured = $state(false)
/** The user wants their own credentials (picked the provider from the resource
* type list rather than the instance section) — overrides the shared instance
* credentials for this connection */
/** The user wants their own credentials (picked the provider from the "Others"
* section) — overrides the shared instance credentials for this connection */
let ccBringYourOwn = $state(false)
/** Connect with the shared instance credentials (no form) rather than the
@@ -325,7 +324,7 @@
return registryCcCapableFor(key)
}
/** Step-1 resource type selection: CC-capable resource types open the client-
/** Step-1 "Others" selection: CC-capable resource types open the client-
* credentials form with the user's own credentials — even when the instance
* has shared ones (the "Instance-configured OAuth APIs" section is the entry
* point for those). Every other type opens the raw manual form. */
@@ -413,14 +412,31 @@
* alphabetically and re-sort when this lands.
*/
let popularity: (a: string, b: string) => number = $state(alphabetical)
/** The hub's most picked types, tagged "Popular" wherever they land in the list. */
let hubPopular: Set<string> = $state(new Set())
let hubPicks: Record<string, number> = $state({})
/** Hub category per type; fetched apart from the picks since the first read is slow. */
let categories: Record<string, string> = $state({})
let selectedCategory: string | undefined = $state(undefined)
const POPULAR_TAG_COUNT = 10
async function loadPopularity() {
if (!effectiveWorkspace) return
hubResourceTypeCategories(effectiveWorkspace).then((c) => (categories = c))
const [hub, local] = await Promise.all([
hubResourceTypePicks(effectiveWorkspace),
localResourceTypeCounts(effectiveWorkspace)
])
popularity = byPopularity(hub, local)
hubPicks = hub
hubPopular = new Set(
Object.entries(hub)
.filter(([, picks]) => picks > 0)
.sort(([a, pa], [b, pb]) => pb - pa || a.localeCompare(b))
.slice(0, POPULAR_TAG_COUNT)
.map(([name]) => name)
)
}
async function loadConnects() {
@@ -508,7 +524,7 @@
})
.catch(() => {})
// The resource type list holds every type — including instance-configured OAuth
// "Others" lists every resource type — including instance-configured OAuth
// providers — so any of them can also be connected with the user's own
// credentials or manually, not only via the shared instance setup (same as
// the authorization-code behavior).
@@ -1032,6 +1048,8 @@
let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([])
let searching = $derived(filter.trim() !== '')
/** A narrowed list: sections that end up empty are dropped rather than explained. */
let filtering = $derived(searching || selectedCategory !== undefined)
// Searching, the query owns the order: uFuzzy scores the name and the description as one
// string, so "google" ranks every type whose description mentions Google alongside the
@@ -1060,71 +1078,45 @@
rank(filteredConnectsManual) as typeof filteredConnectsManual | undefined
)
let manualOrderedKeys = $derived((rankedConnectsManual ?? []).map((x) => x.key))
const categoryOf = (key: string): string | undefined => categories[stripSandboxSuffix(key)]
const inSelectedCategory = (key: string) =>
selectedCategory === undefined || categoryOf(key) === selectedCategory
// The filters on offer: every category some listed type falls in, the most picked first
// so the ones people reach for lead the row.
let categoryFilters = $derived.by(() => {
const picks = new Map<string, number>()
for (const { key } of connectsManual ?? []) {
const category = categoryOf(key)
if (category) picks.set(category, (picks.get(category) ?? 0) + (hubPicks[key] ?? 0))
}
return [...picks.entries()]
.sort(([a, pa], [b, pb]) => pb - pa || a.localeCompare(b))
.map(([category]) => category)
})
let manualOrderedKeys = $derived(
(rankedConnectsManual ?? []).map((x) => x.key).filter(inSelectedCategory)
)
let filteredRankedConnects = $derived(rankedConnects?.filter((x) => inSelectedCategory(x.key)))
let customKeys = $derived(manualOrderedKeys.filter((key) => customResourceTypes.has(key)))
let otherKeys = $derived(manualOrderedKeys.filter((key) => !customResourceTypes.has(key)))
// Browsing, the common types are grouped into categories ahead of "Others". Searching,
// that grouping would outrank the search itself — a weak Databases match sorting above
// the exact hit in Others — so the ranked order stands on its own.
let manualSections = $derived.by(() => {
type Section = { title: string; popular: boolean; keys: string[]; seeMore?: string }
const others: Section = {
title: 'Others',
popular: false,
keys: searching ? otherKeys : otherKeys.filter((key) => !resourceTypeCategory(key))
}
if (searching) return [others]
const keysIn = (title: string, popular: boolean) =>
otherKeys.filter((key) => {
const category = resourceTypeCategory(key)
return category?.title === title && category.popular === popular
})
const popular: Section[] = []
const rest: Section[] = []
for (const category of RESOURCE_TYPE_CATEGORIES) {
const restKeys = keysIn(category.title, false)
const popularKeys = keysIn(category.title, true)
if (restKeys.length > 0) rest.push({ title: category.title, popular: false, keys: restKeys })
if (popularKeys.length > 0)
popular.push({
title: category.title,
popular: true,
keys: popularKeys,
seeMore: restKeys.length > 0 ? `See ${category.others}` : undefined
})
}
// An empty Others still renders while loading (its skeletons) and on an unsynced
// list (the sync hint).
const keepOthers =
others.keys.length > 0 || !rankedConnectsManual || (connectsManual?.length ?? 0) < 10
return keepOthers ? [...popular, ...rest, others] : [...popular, ...rest]
})
// Every row in the order it is rendered, so arrow keys walk the sections as one list.
// A provider appears in more than one, so rows are addressed by index, not by name.
let navItems = $derived([
...customKeys.map((key) => ({ key, oauth: false })),
...(rankedConnects ?? []).map((x) => ({ key: x.key, oauth: true })),
...manualSections.flatMap((section) => section.keys.map((key) => ({ key, oauth: false })))
...(filteredRankedConnects ?? []).map((x) => ({ key: x.key, oauth: true })),
...otherKeys.map((key) => ({ key, oauth: false }))
])
// Both lists start undefined and render skeletons; "nothing found" only means something
// once they have landed.
let listsLoaded = $derived(rankedConnectsManual !== undefined && rankedConnects !== undefined)
const rowDomId = (index: number) => `resource-type-row-${index}`
const categorySectionId = (title: string) =>
`resource-type-category-${title.toLowerCase().replace(/[^a-z]+/g, '-')}`
const oauthRowOffset = $derived(customKeys.length)
const manualRowOffsets = $derived.by(() => {
let offset = customKeys.length + (rankedConnects?.length ?? 0)
return manualSections.map((section) => {
const start = offset
offset += section.keys.length
return start
})
})
const otherRowOffset = $derived(customKeys.length + (filteredRankedConnects?.length ?? 0))
// Sections are rendered in a fixed order, so the best match is not necessarily the first
// row: rank the rows against the query to find it.
@@ -1201,12 +1193,26 @@
class="pl-7 text-xs w-full"
/>
</div>
{#if categoryFilters.length > 0}
<div class="flex flex-wrap gap-1 mt-2" role="group" aria-label="Filter by category">
{#each [undefined, ...categoryFilters] as category (category ?? '')}
<Button
variant="subtle"
unifiedSize="xs"
selected={selectedCategory === category}
aria-pressed={selectedCategory === category}
onClick={() => (selectedCategory = category)}
>
{category ?? 'All'}
</Button>
{/each}
</div>
{/if}
</div>
{#snippet sectionHeading(title: string, count: number, popular: boolean = false)}
<h2 class="mb-3 flex items-center gap-2 text-2xs font-normal uppercase text-secondary">
{title}{#if popular}<Badge color="blue" small class="normal-case">Popular</Badge
>{/if}{#if searching}<span class="text-hint">{count}</span>{/if}
{#snippet sectionHeading(title: string, count: number)}
<h2 class="mb-3 text-2xs font-normal uppercase text-secondary">
{title}{#if filtering}<span class="ml-2 text-hint">{count}</span>{/if}
</h2>
{/snippet}
@@ -1217,6 +1223,9 @@
{#snippet title()}
<span class="truncate leading-5">{resourceTypeDisplayName(key)}</span>
<span class="shrink-0 font-mono text-2xs font-normal text-hint">{key}</span>
{#if hubPopular.has(stripSandboxSuffix(key))}
<Badge color="blue" small class="self-center">Popular</Badge>
{/if}
{/snippet}
{#snippet subtitle()}
{plainDescription(resourceTypeDescriptions[key])}
@@ -1249,7 +1258,7 @@
{:else}
<!-- One gap between sections, owned by the column: a section that a search empties
out then takes its spacing with it. -->
<div class="flex flex-col gap-8">
<div class="flex flex-col gap-10">
{#if customKeys.length > 0}
<section>
{@render sectionHeading('Custom resource types', customKeys.length)}
@@ -1261,15 +1270,15 @@
</section>
{/if}
{#if !searching || (rankedConnects?.length ?? 0) > 0}
{#if !filtering || (filteredRankedConnects?.length ?? 0) > 0}
<section>
{@render sectionHeading(
'Instance-configured OAuth APIs',
rankedConnects?.length ?? 0
filteredRankedConnects?.length ?? 0
)}
<div class="flex flex-col gap-0.5">
{#if rankedConnects}
{#each rankedConnects as { key }, i}
{#if filteredRankedConnects}
{#each filteredRankedConnects as { key }, i}
{@render resourceButton(key, oauthRowOffset + i, true)}
{/each}
{:else}
@@ -1289,44 +1298,29 @@
</section>
{/if}
{#each manualSections as section, s (`${section.title}-${section.popular}`)}
{#if !searching || section.keys.length > 0}
<section id={section.popular ? undefined : categorySectionId(section.title)}>
{@render sectionHeading(section.title, section.keys.length, section.popular)}
{#if !filtering || otherKeys.length > 0}
<section>
{@render sectionHeading(selectedCategory ?? 'Others', otherKeys.length)}
{#if section.title === 'Others' && !searching && connectsManual && connectsManual?.length < 10}
<div class="text-secondary text-xs p-2">
Resource types have not been synced with the hub
</div>
{/if}
<div class="flex flex-col gap-0.5">
{#if rankedConnectsManual}
{#each section.keys as key, i}
{@render resourceButton(key, manualRowOffsets[s] + i, false)}
{/each}
{:else}
{#each new Array(9) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
{#if !searching && connectsManual && connectsManual?.length < 10}
<div class="text-secondary text-xs p-2">
Resource types have not been synced with the hub
</div>
{#if section.seeMore}
<Button
variant="subtle"
unifiedSize="sm"
wrapperClasses="mt-1 w-fit"
onClick={() =>
document
.getElementById(categorySectionId(section.title))
?.scrollIntoView({ behavior: 'smooth', block: 'start' })}
>
{section.seeMore}
</Button>
{/if}
<div class="flex flex-col gap-0.5">
{#if rankedConnectsManual}
{#each otherKeys as key, i}
{@render resourceButton(key, otherRowOffset + i, false)}
{/each}
{:else}
{#each new Array(9) as _}
<Skeleton layout={[[2]]} />
{/each}
{/if}
</section>
{/if}
{/each}
</div>
</section>
{/if}
</div>
{/if}
</div>
@@ -53,6 +53,30 @@ export async function hubResourceTypePicks(workspace: string): Promise<Popularit
return Object.fromEntries(info.map((rt) => [rt.name, rt.picks]))
}
const hubCategoriesCached = createCache(
async ({ workspace }: { workspace: string }): Promise<Record<string, string>> => {
try {
const categories = await ResourceService.listHubResourceTypeCategories({ workspace })
return Object.fromEntries(categories.map((c) => [c.name, c.category]))
} catch {
return {}
}
},
{ invalidateMs: CACHE_MS }
)
/**
* The hub's category for each resource type ("Databases", "AI", …), inherited from its
* integration. Types whose integration has none are absent, as is everything when the hub is
* switched off.
*/
export async function hubResourceTypeCategories(
workspace: string
): Promise<Record<string, string>> {
if (get(disableHubStore)) return {}
return hubCategoriesCached({ workspace })
}
/**
* How many resources of each type this workspace holds — the only evidence about this
* particular team. Keyed by resource type, which is what the add-resource drawer lists.
@@ -1,121 +0,0 @@
import { stripSandboxSuffix } from './oauthRegistry'
/** Hand-picked sections for the resource types people reach for most, in the order the add
* resource drawer renders them. `popular` holds the handful picked most of the time and is
* listed ahead of every full section; `keys` holds the rest of the category. Everything not
* listed falls under "Others". Keys are hub resource type names; a key the instance has not
* synced simply renders nothing. */
export const RESOURCE_TYPE_CATEGORIES: {
title: string
/** Names the rest of the category on the link from its popular section. */
others: string
popular: string[]
keys: string[]
}[] = [
{
title: 'Databases',
others: 'other databases',
popular: [
'postgresql',
'mysql',
'ms_sql_server',
'bigquery',
'snowflake',
'mongodb',
'supabase',
'neondb'
],
keys: [
'snowflake_oauth',
'oracledb',
'redshift',
'clickhouse',
'databricks',
'cockroachdb',
'planetscale',
'turso',
'firebase',
'surrealdb'
]
},
{
title: 'AI',
others: 'other AI providers',
popular: ['openai', 'anthropic', 'googleai', 'mistral', 'deepseek', 'groq'],
keys: [
'azure_openai',
'aws_bedrock',
'azure_foundry',
'openrouter',
'togetherai',
'cohere',
'deep_infra',
'customai',
'pinecone',
'chromadb'
]
},
{
title: 'Object storage',
others: 'other object storage',
popular: ['s3', 'azure_blob', 'gcloud_storage'],
keys: ['s3_aws_oidc']
},
{
title: 'Cloud providers',
others: 'other cloud providers',
popular: ['aws', 'gcloud', 'gcp_service_account', 'azure'],
keys: ['aws_oidc', 'azure_oauth', 'azure_workload_identity', 'cloudflare', 'digitalocean']
},
{
title: 'Messaging & email',
others: 'other messaging & email',
popular: ['slack', 'ms_teams_webhook', 'smtp', 'gmail', 'discord_webhook', 'telegram'],
keys: [
'discord_bot_configuration',
'whatsapp_business',
'twilio',
'sendgrid',
'resend',
'mailgun'
]
},
{
title: 'Event streaming',
others: 'other event streaming',
popular: ['kafka', 'nats', 'mqtt'],
keys: ['amqp']
},
{
title: 'Productivity',
others: 'other productivity tools',
popular: ['gsheets', 'gdrive', 'notion', 'airtable', 'jira'],
keys: ['gcal', 'gdocs', 'confluence', 'linear', 'asana', 'trello', 'clickup']
},
{
title: 'Developer tools',
others: 'other developer tools',
popular: ['github', 'gitlab'],
keys: ['github_app', 'bitbucket', 'git_repository', 'sentry', 'datadog']
},
{
title: 'CRM & payments',
others: 'other CRM & payments',
popular: ['hubspot', 'salesforce', 'stripe'],
keys: ['shopify', 'zendesk']
}
]
type ResourceTypeCategory = { title: string; popular: boolean }
const categoryByKey = new Map<string, ResourceTypeCategory>(
RESOURCE_TYPE_CATEGORIES.flatMap(({ title, popular, keys }) => [
...popular.map((key): [string, ResourceTypeCategory] => [key, { title, popular: true }]),
...keys.map((key): [string, ResourceTypeCategory] => [key, { title, popular: false }])
])
)
/** A sandbox client (`salesforce_sandbox`) files under its parent's section. */
export function resourceTypeCategory(key: string): ResourceTypeCategory | undefined {
return categoryByKey.get(stripSandboxSuffix(key))
}