feat: local hub embeddings search (#2463)

* feat: local hub embeddings search * fix: cargo lock * fix: sqlx prepare * fix: cargo tinyvector * feat: add DISABLE_EMBEDDING env var * fix: cargo lock
2026-09-12 00:06:14 +00:00 · 2023-10-19 17:27:42 +02:00
parent 94a52f1d2d
commit ef3e4b2623
19 changed files with 1527 additions and 315 deletions
@@ -343,6 +343,7 @@ it being synced automatically everyday.
 | SAML_METADATA                                 | None                                       | SAML Metadata URL to enable SAML SSO (EE only)                                                                                                                                                     | Server                |
 | SECRET_SALT                                   | None                                       | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server    | Server + Worker       |
 | OPENAI_AZURE_BASE_PATH                        | None                                       | Azure OpenAI API base path (no trailing slash)                                                                                                                                                     | Server                |
+| DISABLE_EMBEDDING                             | false                                      | Disable local embedding search of hub scripts                                                                                                                                                      | Server                |
 | DISABLE_NSJAIL                                | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |

@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT * from resource_type ORDER BY name",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "workspace_id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "schema",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 3,
+        "name": "description",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12"
+}
@@ -193,4 +193,10 @@ gcp_auth = "0.9.0"
 rust_decimal = { version = "1.31.0", features = ["db-postgres"]}
 jsonwebtoken = "8.3.0"
 pem = "3.0.1"
-nix = { version = "0.27.1", features = ["process", "signal"] }
+nix = { version = "0.27.1", features = ["process", "signal"] }
+tinyvector = { git = "https://github.com/windmill-labs/tinyvector" }
+hf-hub = "0.3.2"
+tokenizers = "0.14.1"
+candle-core = "0.3.0"
+candle-transformers = "0.3.0"
+candle-nn = "0.3.0"
@@ -75,4 +75,10 @@ mail-send.workspace = true
 samael = { workspace = true, optional = true }
 async-recursion.workspace = true
 rsa.workspace = true
-uuid.workspace = true
+uuid.workspace = true
+tinyvector.workspace = true
+hf-hub.workspace = true
+tokenizers.workspace = true
+candle-core.workspace = true
+candle-transformers.workspace = true
+candle-nn.workspace = true
@@ -2363,35 +2363,10 @@ paths:
                items:
                  type: string

-  /resources/type/hub/list:
+  /embeddings/query_resource_types:
    get:
-      summary: list hub resource types
-      operationId: listHubResourceTypes
-      tags:
-        - resource
-      responses:
-        "200":
-          description: resource type details
-          content:
-            application/json:
-              schema:
-                type: array
-                items:
-                  type: object
-                  properties:
-                    id:
-                      type: string
-                    name:
-                      type: string
-                    schema: {}
-                  required:
-                    - id
-                    - name
-
-  /resources/type/hub/query:
-    get:
-      summary: query hub resource types by similarity
-      operationId: queryHubResourceTypes
+      summary: query resource types by similarity
+      operationId: queryResourceTypes
      tags:
        - resource
      parameters:
@@ -2417,56 +2392,42 @@ paths:
                items:
                  type: object
                  properties:
-                    id:
+                    name:
                      type: string
+                    score:
+                      type: number
+                    schema: {}
                  required:
-                    - id
+                    - name
+                    - score

-  /scripts/hub/list:
+  /integrations/hub/list:
    get:
-      summary: list all available hub scripts
-      operationId: listHubScripts
+      summary: list hub integrations
+      operationId: listHubIntegrations
      tags:
-        - script
+        - integration
+      parameters:
+        - name: kind
+          description: query integrations kind
+          in: query
+          required: false
+          schema:
+            type: string
      responses:
        "200":
-          description: hub scripts list
+          description: integrations details
          content:
            application/json:
              schema:
-                type: object
-                properties:
-                  asks:
-                    type: array
-                    items:
-                      type: object
-                      properties:
-                        id:
-                          type: number
-                        ask_id:
-                          type: number
-                        summary:
-                          type: string
-                        app:
-                          type: string
-                        approved:
-                          type: boolean
-                        kind:
-                          type: string
-                          enum: [script, failure, trigger, command, approval]
-                        votes:
-                          type: number
-                        views:
-                          type: number
-                      required:
-                        - id
-                        - ask_id
-                        - summary
-                        - app
-                        - approved
-                        - kind
-                        - views
-                        - votes
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                  required:
+                    - name

  /flows/hub/list:
    get:
@@ -2642,8 +2603,73 @@ paths:
                required:
                  - content
                  - language
+  
+  /scripts/hub/top:
+    get:
+      summary: get top hub scripts
+      operationId: getTopHubScripts
+      tags:
+        - script
+      parameters:
+        - name: limit
+          description: query limit
+          in: query
+          required: false
+          schema:
+            type: number
+        - name: app
+          description: query scripts app
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: kind
+          description: query scripts kind
+          in: query
+          required: false
+          schema:
+            type: string
+      responses:
+        "200":
+          description: hub scripts list
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  asks:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        id:
+                          type: number
+                        ask_id:
+                          type: number
+                        summary:
+                          type: string
+                        app:
+                          type: string
+                        approved:
+                          type: boolean
+                        kind:
+                          type: string
+                          enum: [script, failure, trigger, command, approval]
+                        votes:
+                          type: number
+                        views:
+                          type: number
+                      required:
+                        - id
+                        - ask_id
+                        - summary
+                        - app
+                        - approved
+                        - kind
+                        - views
+                        - votes

-  /scripts/hub/query:
+  /embeddings/query_hub_scripts:
    get:
      summary: query hub scripts by similarity
      operationId: queryHubScripts
@@ -2668,6 +2694,12 @@ paths:
          required: false
          schema:
            type: number
+        - name: app
+          description: query scripts app
+          in: query
+          required: false
+          schema:
+            type: string
      responses:
        "200":
          description: script details
@@ -2678,10 +2710,29 @@ paths:
                items:
                  type: object
                  properties:
+                    ask_id:
+                      type: number
                    id:
+                      type: number
+                    version_id:
+                      type: number
+                    summary:
                      type: string
+                    app:
+                      type: string
+                    kind:
+                      type: string
+                      enum: [script, failure, trigger, command, approval]
+                    score:
+                      type: number
                  required:
+                    - ask_id
                    - id
+                    - version_id
+                    - summary
+                    - app
+                    - kind
+                    - score

  /w/{workspace}/scripts/list_search:
    get:
@@ -0,0 +1,462 @@
+use std::{collections::HashMap, sync::Arc};
+
+use anyhow::{self, Error, Result};
+use axum::{extract::Query, routing::get, Extension, Json, Router};
+use candle_core::{Device, Tensor};
+use candle_nn::VarBuilder;
+use candle_transformers::models::bert::{BertModel, Config, DTYPE};
+use hf_hub::{api::sync::Api, Cache, Repo};
+use serde::{Deserialize, Serialize};
+use sqlx::{Pool, Postgres};
+use tinyvector::{
+    db::{Db, Embedding},
+    similarity::Distance,
+};
+use tokenizers::Tokenizer;
+use tokio::sync::RwLock;
+use windmill_common::{error::JsonResult, utils::http_get_from_hub};
+
+use crate::{resources::ResourceType, HTTP_CLIENT};
+
+#[derive(Deserialize)]
+struct HubScriptsQuery {
+    text: String,
+    limit: Option<i64>,
+    kind: Option<String>,
+    app: Option<String>,
+}
+
+#[derive(Serialize)]
+pub struct HubScriptResult {
+    ask_id: i64,
+    id: i64,
+    version_id: i64,
+    summary: String,
+    app: String,
+    kind: String,
+    score: f32,
+}
+
+async fn query_hub_scripts(
+    Query(query): Query<HubScriptsQuery>,
+    Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
+) -> JsonResult<Vec<HubScriptResult>> {
+    let embeddings_db = embeddings_db.read().await;
+
+    if let Some(embeddings_db) = embeddings_db.as_ref() {
+        let results = embeddings_db
+            .query_hub_scripts(&query.text, query.limit, query.kind, query.app)
+            .await?;
+
+        Ok(Json(results))
+    } else {
+        Err(windmill_common::error::Error::InternalErr(
+            "Embeddings db not initialized".to_string(),
+        ))
+    }
+}
+
+#[derive(Deserialize)]
+struct ResourceTypesQuery {
+    text: String,
+    limit: Option<i64>,
+}
+
+#[derive(Serialize)]
+pub struct ResourceTypeResult {
+    name: String,
+    score: f32,
+    schema: Option<serde_json::Value>,
+}
+async fn query_resource_types(
+    Query(query): Query<ResourceTypesQuery>,
+    Extension(embeddings_db): Extension<Arc<RwLock<Option<EmbeddingsDb>>>>,
+) -> JsonResult<Vec<ResourceTypeResult>> {
+    let embeddings_db = embeddings_db.read().await;
+
+    if let Some(embeddings_db) = embeddings_db.as_ref() {
+        let results = embeddings_db
+            .query_resource_types(&query.text, query.limit)
+            .await?;
+
+        Ok(Json(results))
+    } else {
+        Err(windmill_common::error::Error::InternalErr(
+            "Embeddings db not initialized".to_string(),
+        ))
+    }
+}
+
+#[derive(Deserialize, Debug, Clone)]
+struct HubScript {
+    ask_id: i64,
+    id: i64,
+    version_id: i64,
+    summary: String,
+    app: String,
+    kind: String,
+    embedding: Vec<f32>,
+}
+
+#[derive(Deserialize, Debug)]
+struct HubResourceType {
+    name: String,
+    embedding: Vec<f32>,
+}
+
+pub struct ModelInstance {
+    model: BertModel,
+    tokenizer: Tokenizer,
+}
+
+impl ModelInstance {
+    pub async fn new() -> Result<Self> {
+        let device = Device::Cpu;
+        let model_id = "thenlper/gte-small".to_string();
+
+        let repo = Repo::model(model_id);
+
+        let cache = Cache::default().repo(repo.clone());
+
+        let api = Api::new()?;
+        let api = api.repo(repo);
+
+        let (config_filename, tokenizer_filename, weights_filename) = (
+            cache
+                .get("config.json")
+                .or_else(|| api.get("config.json").ok())
+                .ok_or(Error::msg("could not get config.json"))?,
+            cache
+                .get("tokenizer.json")
+                .or_else(|| api.get("tokenizer.json").ok())
+                .ok_or(Error::msg("could not get tokenizer.json"))?,
+            cache
+                .get("model.safetensors")
+                .or_else(|| api.get("model.safetensors").ok())
+                .ok_or(Error::msg("could not get model.safetensors"))?,
+        );
+        let config = std::fs::read_to_string(config_filename)?;
+        let config: Config = serde_json::from_str(&config)?;
+        let tokenizer = Tokenizer::from(
+            Tokenizer::from_file(tokenizer_filename)
+                .map_err(Error::msg)?
+                .with_padding(None)
+                .with_truncation(None)
+                .map_err(Error::msg)?
+                .to_owned(),
+        );
+
+        let vb =
+            unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? };
+        let model = BertModel::load(vb, &config)?;
+        Ok(Self { model, tokenizer })
+    }
+
+    pub async fn create_embedding(self: Arc<Self>, sentence: &str) -> Result<Vec<f32>> {
+        let sentence = sentence.to_owned();
+        tokio::task::spawn_blocking(move || {
+            let tokens = self
+                .tokenizer
+                .encode(sentence, true)
+                .map_err(Error::msg)?
+                .get_ids()
+                .to_vec();
+
+            let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?;
+            let token_type_ids = token_ids.zeros_like()?;
+
+            let embedding = self.model.forward(&token_ids, &token_type_ids)?;
+            let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?;
+            let embedding = normalize_l2(&embedding)?;
+
+            let embedding = embedding.get(0)?.to_vec1()?;
+
+            Ok(embedding)
+        })
+        .await?
+    }
+}
+
+pub struct EmbeddingsDb {
+    db: Db,
+    model_instance: Arc<ModelInstance>,
+}
+
+impl EmbeddingsDb {
+    pub async fn new(pg_db: &Pool<Postgres>, model_instance: Arc<ModelInstance>) -> Result<Self> {
+        let db = Db::new();
+
+        let mut embeddings_db = Self { db, model_instance: model_instance.clone() };
+
+        embeddings_db.fill_db(pg_db).await?;
+
+        Ok(embeddings_db)
+    }
+
+    async fn fill_db(&mut self, pg_db: &Pool<Postgres>) -> Result<()> {
+        if self.db.get_collection("scripts").is_some() {
+            self.db.delete_collection("scripts")?;
+        }
+
+        self.db
+            .create_collection("scripts".to_string(), 384, Distance::Cosine)?;
+
+        if self.db.get_collection("resource_types").is_some() {
+            self.db.delete_collection("resource_types")?;
+        }
+
+        self.db
+            .create_collection("resource_types".to_string(), 384, Distance::Cosine)?;
+
+        let response = http_get_from_hub(
+            &HTTP_CLIENT,
+            "https://hub.windmill.dev/scripts/embeddings",
+            "todo@windmill.dev",
+            false,
+            None,
+        )
+        .await?;
+        let hub_scripts = response.json::<Vec<HubScript>>().await?;
+
+        for script in &hub_scripts {
+            let mut hm = HashMap::new();
+            hm.insert("ask_id".to_string(), script.ask_id.clone().to_string());
+            hm.insert("summary".to_string(), script.summary.clone());
+            hm.insert("app".to_string(), script.app.clone());
+            hm.insert("kind".to_string(), script.kind.clone());
+            hm.insert("id".to_string(), script.id.clone().to_string());
+            hm.insert(
+                "version_id".to_string(),
+                script.version_id.clone().to_string(),
+            );
+            let embedding = Embedding {
+                id: script.ask_id.clone().to_string(),
+                vector: script.embedding.clone(),
+                metadata: Some(hm),
+            };
+            self.db.insert_into_collection("scripts", embedding)?;
+        }
+
+        let response = http_get_from_hub(
+            &HTTP_CLIENT,
+            "https://hub.windmill.dev/resource_types/embeddings",
+            "todo@windmill.dev",
+            false,
+            None,
+        )
+        .await?;
+        let hub_resource_types = response.json::<Vec<HubResourceType>>().await?;
+
+        let resource_types: Vec<ResourceType> = sqlx::query_as!(
+            ResourceType,
+            "SELECT * from resource_type ORDER \
+             BY name",
+        )
+        .fetch_all(pg_db)
+        .await?;
+
+        for rt in resource_types {
+            let mut hm = HashMap::new();
+            hm.insert("name".to_string(), rt.name.clone());
+            if let Some(schema) = rt.schema.clone() {
+                hm.insert("schema".to_string(), serde_json::to_string(&schema)?);
+            }
+            let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name);
+
+            let vector = if let Some(hub_rt) = hub_rt {
+                hub_rt.embedding.clone()
+            } else {
+                self.model_instance
+                    .clone()
+                    .create_embedding(&format!(
+                        "{};{}",
+                        rt.name,
+                        rt.description.unwrap_or_default()
+                    ))
+                    .await?
+            };
+
+            let embedding = Embedding { id: rt.name.clone(), vector, metadata: Some(hm) };
+            self.db
+                .insert_into_collection("resource_types", embedding)?;
+        }
+
+        Ok(())
+    }
+
+    pub async fn query_hub_scripts(
+        &self,
+        query: &str,
+        limit: Option<i64>,
+        kind: Option<String>,
+        app: Option<String>,
+    ) -> Result<Vec<HubScriptResult>> {
+        let model_instance = self.model_instance.clone();
+        let query_embedding = model_instance.create_embedding(query).await?;
+
+        let collection = self.db.get_collection("scripts");
+
+        let collection = collection.ok_or(Error::msg("no collection found"))?;
+
+        let filter = |embedding: &Embedding| {
+            if let Some(metadata) = embedding.metadata.as_ref() {
+                match (
+                    metadata.get("kind"),
+                    kind.clone(),
+                    metadata.get("app"),
+                    app.clone(),
+                ) {
+                    (Some(script_kind), Some(kind), Some(script_app), Some(app)) => {
+                        &kind == script_kind && &app == script_app
+                    }
+                    (Some(script_kind), Some(kind), _, _) => &kind == script_kind,
+                    (_, _, Some(script_app), Some(app)) => &app == script_app,
+                    (_, None, _, None) => true,
+                    _ => false,
+                }
+            } else {
+                false
+            }
+        };
+
+        let results = collection.get_similarity(
+            &query_embedding,
+            limit.unwrap_or(10) as usize,
+            Some(&filter),
+            Some(0.75),
+        );
+
+        let results: Result<Vec<_>> = results
+            .iter()
+            .map(|r| {
+                let metadata = r
+                    .embedding
+                    .metadata
+                    .as_ref()
+                    .ok_or(Error::msg("no metadata"))?;
+
+                Ok(HubScriptResult {
+                    ask_id: metadata
+                        .get("ask_id")
+                        .ok_or(Error::msg("no ask_id"))?
+                        .parse::<i64>()?,
+                    summary: metadata
+                        .get("summary")
+                        .ok_or(Error::msg("no summary"))?
+                        .to_owned(),
+                    app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(),
+                    kind: metadata
+                        .get("kind")
+                        .ok_or(Error::msg("no kind"))?
+                        .to_owned(),
+                    id: metadata
+                        .get("id")
+                        .ok_or(Error::msg("no id"))?
+                        .parse::<i64>()?,
+                    version_id: metadata
+                        .get("version_id")
+                        .ok_or(Error::msg("no version_id"))?
+                        .parse::<i64>()?,
+                    score: r.score,
+                })
+            })
+            .collect();
+
+        results
+    }
+
+    pub async fn query_resource_types(
+        &self,
+        query: &str,
+        limit: Option<i64>,
+    ) -> Result<Vec<ResourceTypeResult>> {
+        let model_instance = self.model_instance.clone();
+        let query_embedding = model_instance.create_embedding(query).await?;
+
+        let collection = self.db.get_collection("resource_types");
+
+        if collection.is_none() {
+            return Ok(vec![]);
+        }
+
+        let collection = collection.ok_or(Error::msg("no collection found"))?;
+        let results = collection.get_similarity(
+            &query_embedding,
+            limit.unwrap_or(10) as usize,
+            None,
+            Some(0.75),
+        );
+
+        let results: Result<_> = results
+            .iter()
+            .map(|r| {
+                let metadata = r
+                    .embedding
+                    .metadata
+                    .as_ref()
+                    .ok_or(Error::msg("no metadata"))?;
+                Ok(ResourceTypeResult {
+                    name: metadata
+                        .get("name")
+                        .ok_or(Error::msg("no name"))?
+                        .to_owned(),
+                    schema: match metadata.get("schema") {
+                        Some(schema) => serde_json::from_str(schema)?,
+                        None => None,
+                    },
+                    score: r.score,
+                })
+            })
+            .collect();
+
+        results
+    }
+}
+
+fn normalize_l2(v: &Tensor) -> Result<Tensor> {
+    Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?)
+}
+
+pub fn global_service(db: &Pool<Postgres>) -> Router {
+    let embeddings_db: Arc<RwLock<Option<EmbeddingsDb>>> = Arc::new(RwLock::new(None));
+
+    let disable_embedding = std::env::var("DISABLE_EMBEDDING")
+        .ok()
+        .map(|x| x.parse::<bool>().unwrap_or(false))
+        .unwrap_or(false);
+
+    if !disable_embedding {
+        let db_clone = db.clone();
+        let embeddings_clone: Arc<RwLock<Option<EmbeddingsDb>>> = embeddings_db.clone();
+        tokio::spawn(async move {
+            let model_instance = ModelInstance::new().await;
+
+            if let Ok(model_instance) = model_instance {
+                let model_instance = Arc::new(model_instance);
+                loop {
+                    let new_embeddings_db =
+                        EmbeddingsDb::new(&db_clone, model_instance.clone()).await;
+                    if let Err(e) = new_embeddings_db.as_ref() {
+                        tracing::error!("Failed to create embeddings db: {}", e);
+                    } else {
+                        let mut embeddings_db = embeddings_clone.write().await;
+                        *embeddings_db = new_embeddings_db.ok();
+                        tracing::info!("Loaded embeddings DB");
+                    }
+
+                    tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await;
+                }
+            } else {
+                tracing::error!(
+                    "Failed to initialize model instance: {}",
+                    model_instance.err().unwrap()
+                );
+            }
+        });
+    }
+
+    Router::new()
+        .route("/query_hub_scripts", get(query_hub_scripts))
+        .route("/query_resource_types", get(query_resource_types))
+        .layer(Extension(embeddings_db))
+}
@@ -0,0 +1,35 @@
+use crate::{db::ApiAuthed, HTTP_CLIENT};
+use axum::{body::StreamBody, extract::Query, response::IntoResponse, routing::get, Router};
+use windmill_common::{error::Error, utils::query_elems_from_hub};
+
+pub fn global_service() -> Router {
+    Router::new().route("/hub/list", get(list_hub_integrations))
+}
+
+#[derive(serde::Deserialize)]
+struct ListHubIntegrationsQuery {
+    kind: Option<String>,
+}
+async fn list_hub_integrations(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Query(query): Query<ListHubIntegrationsQuery>,
+) -> impl IntoResponse {
+    let mut query_params = vec![];
+
+    if let Some(kind) = query.kind {
+        query_params.push(("kind", kind));
+    }
+
+    let (status_code, headers, response) = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/integrations/list",
+        &email,
+        Some(query_params),
+    )
+    .await?;
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
+}
@@ -48,12 +48,14 @@ mod configs;
 mod db;
 mod drafts;
 pub mod ee;
+mod embeddings;
 mod favorite;
 mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
 mod inputs;
+mod integration;
 pub mod jobs;
 pub mod oauth2;
 mod openai;
@@ -204,11 +206,12 @@ pub async fn run_server(
                .nest("/workers", workers::global_service())
                .nest("/configs", configs::global_service())
                .nest("/scripts", scripts::global_service())
-                .nest("/resources", resources::global_service())
+                .nest("/integrations", integration::global_service())
                .nest("/groups", groups::global_service())
                .nest("/flows", flows::global_service())
                .nest("/apps", apps::global_service().layer(cors.clone()))
                .nest("/schedules", schedule::global_service())
+                .nest("/embeddings", embeddings::global_service(&db))
                .route_layer(from_extractor::<ApiAuthed>())
                .route_layer(from_extractor::<users::Tokened>())
                .nest("/jobs", jobs::global_root_service())
@@ -10,12 +10,9 @@ use crate::{
    db::{ApiAuthed, DB},
    users::{maybe_refresh_folders, require_owner_of_path, Tokened},
    webhook_util::{WebhookMessage, WebhookShared},
-    HTTP_CLIENT,
 };
 use axum::{
-    body::StreamBody,
    extract::{Extension, Path, Query},
-    response::IntoResponse,
    routing::{delete, get, post},
    Json, Router,
 };
@@ -30,18 +27,10 @@ use windmill_common::{
    db::UserDB,
    error::{Error, JsonResult, Result},
    jobs::QueuedJob,
-    utils::{
-        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
-    },
+    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
    variables,
 };

-pub fn global_service() -> Router {
-    Router::new()
-        .route("/type/hub/list", get(list_hub_resource_types))
-        .route("/type/hub/query", get(query_hub_resource_types))
-}
-
 pub fn workspaced_service() -> Router {
    Router::new()
        .route("/list", get(list_resources))
@@ -912,46 +901,3 @@ async fn update_resource_type(

    Ok(format!("resource_type {} updated", name))
 }
-
-async fn list_hub_resource_types(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
-    let (status_code, headers, response) = query_elems_from_hub(
-        &HTTP_CLIENT,
-        "https://hub.windmill.dev/resource_types/list",
-        &email,
-        None,
-    )
-    .await?;
-    Ok::<_, Error>((
-        status_code,
-        headers,
-        StreamBody::new(response.bytes_stream()),
-    ))
-}
-
-#[derive(Deserialize)]
-struct HubResourceTypesQuery {
-    text: String,
-    limit: Option<i64>,
-}
-async fn query_hub_resource_types(
-    ApiAuthed { email, .. }: ApiAuthed,
-    Query(query): Query<HubResourceTypesQuery>,
-) -> impl IntoResponse {
-    let mut query_params = vec![("text", query.text)];
-    if let Some(query_limit) = query.limit {
-        query_params.push(("limit", query_limit.to_string().clone()));
-    }
-    let (status_code, headers, response) = query_elems_from_hub(
-        &HTTP_CLIENT,
-        "https://hub.windmill.dev/resource_types/query",
-        &email,
-        Some(query_params),
-    )
-    .await?;
-
-    Ok::<_, Error>((
-        status_code,
-        headers,
-        StreamBody::new(response.bytes_stream()),
-    ))
-}
@@ -82,10 +82,9 @@ pub struct ScriptWDraft {

 pub fn global_service() -> Router {
    Router::new()
-        .route("/hub/list", get(list_hub_scripts))
+        .route("/hub/top", get(get_top_hub_scripts))
        .route("/hub/get/*path", get(get_hub_script_by_path))
        .route("/hub/get_full/*path", get(get_full_hub_script_by_path))
-        .route("/hub/query", get(query_hub_scripts))
 }

 pub fn global_unauthed_service() -> Router {
@@ -249,41 +248,31 @@ async fn list_scripts(
    Ok(Json(rows))
 }

-async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
-    let (status_code, headers, response) = query_elems_from_hub(
-        &HTTP_CLIENT,
-        "https://hub.windmill.dev/searchData?approved=true",
-        &email,
-        None,
-    )
-    .await?;
-    Ok::<_, Error>((
-        status_code,
-        headers,
-        StreamBody::new(response.bytes_stream()),
-    ))
+#[derive(Deserialize)]
+struct TopHubScriptsQuery {
+    limit: Option<i64>,
+    app: Option<String>,
+    kind: Option<String>,
 }

-#[derive(Deserialize)]
-struct HubScriptsQuery {
-    text: String,
-    kind: Option<String>,
-    limit: Option<i64>,
-}
-async fn query_hub_scripts(
+async fn get_top_hub_scripts(
    ApiAuthed { email, .. }: ApiAuthed,
-    Query(query): Query<HubScriptsQuery>,
+    Query(query): Query<TopHubScriptsQuery>,
 ) -> impl IntoResponse {
-    let mut query_params = vec![("text", query.text)];
-    if let Some(query_kind) = query.kind {
-        query_params.push(("kind", query_kind.clone()));
-    }
+    let mut query_params = vec![];
    if let Some(query_limit) = query.limit {
        query_params.push(("limit", query_limit.to_string().clone()));
    }
+    if let Some(query_app) = query.app {
+        query_params.push(("app", query_app.to_string().clone()));
+    }
+    if let Some(query_kind) = query.kind {
+        query_params.push(("kind", query_kind.to_string().clone()));
+    }
+
    let (status_code, headers, response) = query_elems_from_hub(
        &HTTP_CLIENT,
-        "https://hub.windmill.dev/scripts/query",
+        "https://hub.windmill.dev/scripts/top",
        &email,
        Some(query_params),
    )
@@ -13,7 +13,6 @@
 	import {
 		copilotInfo,
 		enterpriseLicense,
-		hubScripts,
 		tutorialsToDo,
 		userStore,
 		workspaceStore
@@ -39,7 +38,6 @@
 	import type { FlowEditorContext } from './flows/types'
 	import { cleanInputs, emptyFlowModuleState } from './flows/utils'
 	import { Pen } from 'lucide-svelte'
-	import { loadHubScripts } from '$lib/scripts'
 	import { createEventDispatcher } from 'svelte'
 	import Awareness from './Awareness.svelte'
 	import { getAllModules } from './flows/flowExplorer'
@@ -303,8 +301,6 @@

 	$: initialPath && $workspaceStore && loadSchedule()

-	loadHubScripts()
-
 	function onKeyDown(event: KeyboardEvent) {
 		let classes = event.target?.['className']
 		if (
@@ -400,25 +396,23 @@
 		try {
 			// make sure we display the results of the last request last
 			const ts = Date.now()
-			const scriptIds = await ScriptService.queryHubScripts({
-				text: `${text}`,
-				limit: 3,
-				kind: type
-			})
+			const scripts = (
+				await ScriptService.queryHubScripts({
+					text: `${text}`,
+					limit: 3,
+					kind: type
+				})
+			).map((s) => ({
+				...s,
+				path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
+				summary: `${s.summary} (${s.app})`
+			}))
 			if (ts < doneTs) return
 			doneTs = ts

-			const scripts = scriptIds
-				.map((qs) => {
-					const s = $hubScripts?.find((hs) => hs.ask_id === Number(qs.id))
-					return s
-				})
-				.filter((s) => !!s)
-
 			$copilotModulesStore[idx].hubCompletions = scripts as {
 				path: string
 				summary: string
-				approved: boolean
 				kind: string
 				app: string
 				ask_id: number
@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { ScriptService, FlowService, Script } from '$lib/gen'

-	import { hubScripts, workspaceStore } from '$lib/stores'
+	import { workspaceStore } from '$lib/stores'
 	import { createEventDispatcher, onMount } from 'svelte'

 	import Select from './apps/svelte-select/lib/index'
@@ -13,7 +13,7 @@
 	import { SELECT_INPUT_DEFAULT_STYLE } from '../defaults'
 	import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
 	import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
-	import { Code2, Globe } from 'lucide-svelte'
+	import { Code2 } from 'lucide-svelte'
 	import type { SupportedLanguage } from '$lib/common'
 	import { faExternalLink, faRotateRight } from '@fortawesome/free-solid-svg-icons'
 	import Icon from 'svelte-awesome'
@@ -24,8 +24,7 @@
 	export let initialPath: string | undefined = undefined
 	export let scriptPath: string | undefined = undefined
 	export let allowFlow = false
-	export let allowHub = false
-	export let itemKind: 'hub' | 'script' | 'flow' = allowHub ? 'hub' : 'script'
+	export let itemKind: 'script' | 'flow' = 'script'
 	export let kinds: Script.kind[] = [Script.kind.SCRIPT]
 	export let disabled = false
 	export let allowRefresh = false
@@ -37,7 +36,6 @@
 	let lang: SupportedLanguage | undefined

 	let options: [[string, any, any, string | undefined]] = [['Script', 'script', Code2, undefined]]
-	allowHub && options.unshift(['Hub', 'hub', Globe, undefined])
 	allowFlow && options.push(['Flow', 'flow', FlowIcon, '#14b8a6'])
 	const dispatch = createEventDispatcher()

@@ -48,18 +46,12 @@
 				label: `${flow.path}${flow.summary ? ` | ${truncate(flow.summary, 20)}` : ''}`
 			}))
 		} else if (itemKind == 'script') {
-				items = (await ScriptService.listScripts({ workspace: $workspaceStore!, kinds: kinds.join(",")})).map(
-					(script) => ({
-						value: script.path,
-						label: `${script.path}${script.summary ? ` | ${truncate(script.summary, 20)}` : ''}`
-					})
-				)
-		} else {
-			items =
-				$hubScripts?.map((x) => ({
-					value: x.path,
-					label: `${x.path}${x.summary ? ` | ${x.summary}` : ''}`
-				})) ?? []
+			items = (
+				await ScriptService.listScripts({ workspace: $workspaceStore!, kinds: kinds.join(',') })
+			).map((script) => ({
+				value: script.path,
+				label: `${script.path}${script.summary ? ` | ${truncate(script.summary, 20)}` : ''}`
+			}))
 		}
 	}

@@ -105,7 +97,7 @@
 	{/if}

 	{#if disabled}
-		<input type="text" value={scriptPath ?? ""} disabled />
+		<input type="text" value={scriptPath ?? ''} disabled />
 	{:else}
 		<Select
 			value={items?.find((x) => x.value == initialPath)}
@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons'
 	import { Icon } from 'svelte-awesome'
-	import { copilotInfo, hubScripts } from '$lib/stores'
+	import { copilotInfo } from '$lib/stores'
 	import { getContext } from 'svelte'
 	import type { FlowEditorContext } from '../flows/types'
 	import type { FlowCopilotContext, FlowCopilotModule } from './flow'
@@ -30,21 +30,19 @@
 		try {
 			// make sure we display the results of the last request last
 			const ts = Date.now()
-			const scriptIds = await ScriptService.queryHubScripts({
-				text: `${text}`,
-				limit: 3,
-				kind: trigger ? 'trigger' : 'script'
-			})
+			const scripts = (
+				await ScriptService.queryHubScripts({
+					text: `${text}`,
+					limit: 3,
+					kind: trigger ? 'trigger' : 'script'
+				})
+			).map((s) => ({
+				...s,
+				path: `hub/${s.id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`
+			}))
 			if (ts < doneTs) return
 			doneTs = ts

-			const scripts = scriptIds
-				.map((qs) => {
-					const s = $hubScripts?.find((hs) => hs.ask_id === Number(qs.id))
-					return s
-				})
-				.filter((s) => !!s)
-
 			hubCompletions = scripts as FlowCopilotModule['hubCompletions']
 		} catch (err) {
 			if (err.name !== 'CancelError') throw err
@@ -15,7 +15,6 @@ export type FlowCopilotModule = {
 	hubCompletions: {
 		path: string
 		summary: string
-		approved: boolean
 		kind: string
 		app: string
 		ask_id: number
@@ -24,7 +23,6 @@ export type FlowCopilotModule = {
 		| {
 				path: string
 				summary: string
-				approved: boolean
 				kind: string
 				app: string
 				ask_id: number
@@ -218,7 +216,7 @@ export async function glueCopilot(
 		abortController
 	)

-	const matches = response.matchAll(/(.+?): (.+)/g)
+	const matches = response.matchAll(/([a-zA-Z_0-9]+): (.+)/g)

 	const result: Record<string, string> = {}
 	for (const match of matches) {
@@ -108,8 +108,6 @@ async function getResourceTypes(scriptOptions: CopilotOptions) {
 		throw new Error('Workspace not initialized')
 	}

-	const localResourceTypes = await ResourceService.listResourceType({ workspace })
-
 	const elems =
 		scriptOptions.type === 'gen' || scriptOptions.type === 'edit' ? [scriptOptions.description] : []

@@ -132,22 +130,9 @@ async function getResourceTypes(scriptOptions: CopilotOptions) {
 		}
 	}

-	const hubResourceTypes = await ResourceService.listHubResourceTypes()
-	const queriedIds = (
-		await ResourceService.queryHubResourceTypes({
-			text: elems.join(';')
-		})
-	).map((rt) => rt.id)
-	const customResourceTypes = localResourceTypes.filter((rt) => rt.name.startsWith('c_'))
-	const resourceTypes = [
-		...hubResourceTypes
-			.filter((rt) => queriedIds.includes(String(rt.id)))
-			.map((rt) => ({
-				...rt,
-				schema: JSON.parse(rt.schema)
-			})),
-		...customResourceTypes
-	]
+	const resourceTypes = await ResourceService.queryResourceTypes({
+		text: elems.join(';')
+	})

 	return resourceTypes
 }
@@ -1,14 +1,11 @@
 <script lang="ts">
-	import { hubScripts } from '$lib/stores'
-	import { createEventDispatcher, onMount } from 'svelte'
-	import type { HubItem } from './model'
+	import { createEventDispatcher } from 'svelte'
 	import { Badge, Skeleton } from '$lib/components/common'
-	import SearchItems from '$lib/components/SearchItems.svelte'
-	import { capitalize, classNames } from '$lib/utils'
+	import { capitalize, classNames, sendUserToast } from '$lib/utils'
 	import NoItemFound from '$lib/components/home/NoItemFound.svelte'
 	import { APP_TO_ICON_COMPONENT } from '$lib/components/icons'
 	import ListFilters from '$lib/components/home/ListFilters.svelte'
-	import { loadHubScripts } from '$lib/scripts'
+	import { IntegrationService, ScriptService } from '$lib/gen'

 	export let kind: 'script' | 'trigger' | 'approval' | 'failure' = 'script'
 	export let filter = ''
@@ -16,34 +13,87 @@

 	const dispatch = createEventDispatcher()

-	let filteredItems: (HubItem & { marked?: string })[] = []
 	let appFilter: string | undefined = undefined
+	let items: {
+		path: string
+		summary: string
+		id: number
+		ask_id: number
+		app: string
+		kind: 'script' | 'trigger' | 'approval' | 'failure' | 'command'
+	}[] = []

-	$: items = ($hubScripts ?? []).filter((i) => i.kind === kind)
-	$: prefilteredItems = appFilter ? (items ?? []).filter((i) => i.app == appFilter) : items ?? []
-	$: apps = Array.from(new Set(filteredItems?.map((x) => x.app) ?? [])).sort()
+	let allApps: string[] = []
+	let apps: string[] = []

-	onMount(() => {
-		if (!$hubScripts) {
-			loadHubScripts()
+	$: apps = filter.length > 0 ? Array.from(new Set(items?.map((x) => x.app) ?? [])).sort() : allApps
+
+	$: applyFilter(filter, kind, appFilter)
+	$: getAllApps(kind)
+
+	async function getAllApps(filterKind: typeof kind) {
+		try {
+			allApps = (
+				await IntegrationService.listHubIntegrations({
+					kind: filterKind
+				})
+			).map((x) => x.name)
+			console.log('allApps', allApps)
+		} catch (err) {
+			sendUserToast(err.message, true)
 		}
-	})
-	const maxItems = 40
+	}
+
+	let doneTs = 0
+	async function applyFilter(
+		filter: string,
+		filterKind: typeof kind,
+		appFilter: string | undefined
+	) {
+		try {
+			const ts = Date.now()
+			const scripts =
+				filter.length > 0
+					? await ScriptService.queryHubScripts({
+							text: `${filter}`,
+							limit: 40,
+							kind: filterKind,
+							app: appFilter
+					  })
+					: (
+							await ScriptService.getTopHubScripts({
+								limit: 40,
+								app: appFilter,
+								kind: filterKind
+							})
+					  ).asks ?? []
+			if (ts < doneTs) return
+			doneTs = ts
+			const processed = scripts.map((x) => ({
+				...x,
+				path: `hub/${x.version_id}/${x.app}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
+				summary: `${x.summary} (${x.app})`
+			}))
+			items = processed
+		} catch (err) {
+			sendUserToast(err.message, true)
+		}
+	}
 </script>

-<SearchItems {filter} items={prefilteredItems} bind:filteredItems f={(x) => x.summary} />
+<!-- <SearchItems {filter} items={prefilteredItems} bind:filteredItems f={(x) => x.summary} /> -->
 <div class="w-full flex mt-1 items-center gap-2">
 	<slot />
 	<input type="text" placeholder="Search Hub Scripts" bind:value={filter} class="text-2xl grow" />
 </div>
-<ListFilters {syncQuery} filters={apps} bind:selectedFilter={appFilter} resourceType />

-{#if $hubScripts}
-	{#if filteredItems.length == 0}
+{#if items.length > 0 && apps.length > 0}
+	<ListFilters {syncQuery} filters={apps} bind:selectedFilter={appFilter} resourceType />
+	{#if items.length == 0}
 		<NoItemFound />
 	{:else}
 		<ul class="divide-y border rounded-md">
-			{#each filteredItems.slice(0, maxItems) as item (item.path)}
+			{#each items as item (item.path)}
 				<li class="flex flex-row w-full">
 					<button
 						class="p-4 gap-4 flex flex-row grow hover:bg-surface-hover bg-surface transition-all items-center rounded-md"
@@ -65,11 +115,7 @@

 							<div class="w-full text-left font-normal">
 								<div class="text-primary flex-wrap text-md font-semibold mb-1">
-									{#if item.marked}
-										{@html item.marked ?? ''}
-									{:else}
-										{item.summary ?? ''}
-									{/if}
+									{item.summary ?? ''}
 								</div>
 								<div class="text-secondary text-xs">
 									{item.path}
@@ -84,9 +130,9 @@
 			{/each}
 		</ul>
 	{/if}
-	{#if filteredItems.length > maxItems}
+	{#if items.length == 40}
 		<div class="text-tertiary text-sm py-4">
-			There are more items ({filteredItems.length}) than being displayed. Refine your search.
+			There are more items than being displayed. Refine your search.
 		</div>
 	{/if}
 {:else}
@@ -1,7 +1,7 @@
 import { get } from 'svelte/store'
 import type { Schema, SupportedLanguage } from './common'
 import { FlowService, Script, ScriptService } from './gen'
-import { workspaceStore, hubScripts } from './stores'
+import { workspaceStore } from './stores'

 export function scriptLangToEditorLang(lang: Script.language) {
 	if (lang == 'deno') {
@@ -94,24 +94,3 @@ export async function getLatestHashForScript(path: string): Promise<string> {
 	})
 	return script.hash
 }
-
-export async function loadHubScripts() {
-	try {
-		const scripts = (await ScriptService.listHubScripts()).asks ?? []
-		const processed = scripts
-			.map((x) => ({
-				path: `hub/${x.id}/${x.app}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
-				summary: `${x.summary} (${x.app})`,
-				approved: x.approved,
-				kind: x.kind,
-				app: x.app,
-				views: x.views,
-				votes: x.votes,
-				ask_id: x.ask_id
-			}))
-			.sort((a, b) => b.views - a.views)
-		hubScripts.set(processed)
-	} catch {
-		console.error('Hub is not available')
-	}
-}
@@ -58,17 +58,6 @@ export const userWorkspaces: Readable<
 		return originalWorkspaces
 	}
 })
-export const hubScripts = writable<
-	| Array<{
-			path: string
-			summary: string
-			approved: boolean
-			kind: string
-			app: string
-			ask_id: number
-	  }>
-	| undefined
->(undefined)
 export const copilotInfo = writable<{
 	exists_openai_resource_path: boolean
 	code_completion_enabled: boolean