feat: filter resource types passed to gpt-4 (#2430)

* feat: code gen filter resource types

* fix: minor

* feat: using snake case for flow inputs

* fix: filter resource types for python

* fix: correct resrouce type name + wmill import

* fix: disable automerge ci

* fix: remove unnecessary json parsing

backend/windmill-api/src/apps.rs

@@ -15,7 +15,9 @@ use crate::{
     HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Json, Path, Query},
+    response::IntoResponse,
     routing::{delete, get, post},
     Router,
 };
@@ -35,7 +37,7 @@ use windmill_common::{
     jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
+        http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
     },
 };
 use windmill_queue::{push, PushIsolationLevel, QueueTransaction};
@@ -544,14 +546,19 @@ async fn create_app(
     Ok((StatusCode::CREATED, app.path))
 }
 
-async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchUiData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 pub async fn get_hub_app_by_id(

backend/windmill-api/src/flows.rs

@@ -14,6 +14,8 @@ use crate::{
     webhook_util::{WebhookMessage, WebhookShared},
     HTTP_CLIENT,
 };
+use axum::body::StreamBody;
+use axum::response::IntoResponse;
 use axum::{
     extract::{Extension, Path, Query},
     routing::{delete, get, post},
@@ -26,6 +28,7 @@ use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::utils::query_elems_from_hub;
 use windmill_common::{
     db::UserDB,
     error::{self, to_anyhow, Error, JsonResult, Result},
@@ -33,9 +36,7 @@ use windmill_common::{
     jobs::JobPayload,
     schedule::Schedule,
     scripts::Schema,
-    utils::{
-        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
-    },
+    utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, StripPath},
 };
 use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
 
@@ -156,14 +157,19 @@ async fn list_flows(
     Ok(Json(rows))
 }
 
-async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let flows = list_elems_from_hub(
+async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchFlowData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(flows))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 async fn list_paths(

backend/windmill-api/src/lib.rs

@@ -205,6 +205,7 @@ pub async fn run_server(
                 .nest("/workers", workers::global_service())
                 .nest("/configs", configs::global_service())
                 .nest("/scripts", scripts::global_service())
+                .nest("/resources", resources::global_service())
                 .nest("/groups", groups::global_service())
                 .nest("/flows", flows::global_service())
                 .nest("/apps", apps::global_service().layer(cors.clone()))

backend/windmill-api/src/openai.rs

@@ -7,7 +7,6 @@ use crate::{
 use axum::{
     body::{Bytes, StreamBody},
     extract::{Extension, Path},
-    http::HeaderMap,
     response::IntoResponse,
     routing::post,
     Router,
@@ -170,12 +169,8 @@ async fn proxy(
         ));
     }
 
-    let mut headers = HeaderMap::new();
-    for (k, v) in response.headers().iter() {
-        headers.insert(k, v.clone());
-    }
-
     let status_code = response.status();
+    let headers = response.headers().clone();
     let stream = response.bytes_stream();
 
     Ok((status_code, headers, StreamBody::new(stream)))

backend/windmill-api/src/resources.rs

@@ -10,9 +10,12 @@ use crate::{
     db::{ApiAuthed, DB},
     users::{maybe_refresh_folders, require_owner_of_path, Tokened},
     webhook_util::{WebhookMessage, WebhookShared},
+    HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Path, Query},
+    response::IntoResponse,
     routing::{delete, get, post},
     Json, Router,
 };
@@ -27,10 +30,18 @@ use windmill_common::{
     db::UserDB,
     error::{Error, JsonResult, Result},
     jobs::QueuedJob,
-    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
+    utils::{
+        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
+    },
     variables,
 };
 
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/type/hub/list", get(list_hub_resource_types))
+        .route("/type/hub/query", get(query_hub_resource_types))
+}
+
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_resources))
@@ -898,3 +909,46 @@ async fn update_resource_type(
 
     Ok(format!("resource_type {} updated", name))
 }
+
+async fn list_hub_resource_types(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/resource_types/list",
+        &email,
+        None,
+    )
+    .await?;
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
+}
+
+#[derive(Deserialize)]
+struct HubResourceTypesQuery {
+    text: String,
+    limit: Option<i64>,
+}
+async fn query_hub_resource_types(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Query(query): Query<HubResourceTypesQuery>,
+) -> impl IntoResponse {
+    let mut query_params = vec![("text", query.text)];
+    if let Some(query_limit) = query.limit {
+        query_params.push(("limit", query_limit.to_string().clone()));
+    }
+    let (status_code, headers, response) = query_elems_from_hub(
+        &HTTP_CLIENT,
+        "https://hub.windmill.dev/resource_types/query",
+        &email,
+        Some(query_params),
+    )
+    .await?;
+
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
+}

backend/windmill-api/src/scripts.rs

@@ -14,7 +14,9 @@ use crate::{
     HTTP_CLIENT,
 };
 use axum::{
+    body::StreamBody,
     extract::{Extension, Path, Query},
+    response::IntoResponse,
     routing::{get, post},
     Json, Router,
 };
@@ -41,8 +43,7 @@ use windmill_common::{
     },
     users::username_to_permissioned_as,
     utils::{
-        list_elems_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
-        Pagination, StripPath,
+        not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
 };
 use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};
@@ -233,14 +234,19 @@ async fn list_scripts(
     Ok(Json(rows))
 }
 
-async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
-    let asks = list_elems_from_hub(
+async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse {
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/searchData?approved=true",
         &email,
+        None,
     )
     .await?;
-    Ok(Json(asks))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 #[derive(Deserialize)]
@@ -252,17 +258,26 @@ struct HubScriptsQuery {
 async fn query_hub_scripts(
     ApiAuthed { email, .. }: ApiAuthed,
     Query(query): Query<HubScriptsQuery>,
-) -> JsonResult<serde_json::Value> {
-    let asks = query_elems_from_hub(
+) -> impl IntoResponse {
+    let mut query_params = vec![("text", query.text)];
+    if let Some(query_kind) = query.kind {
+        query_params.push(("kind", query_kind.clone()));
+    }
+    if let Some(query_limit) = query.limit {
+        query_params.push(("limit", query_limit.to_string().clone()));
+    }
+    let (status_code, headers, response) = query_elems_from_hub(
         &HTTP_CLIENT,
         "https://hub.windmill.dev/scripts/query",
         &email,
-        &query.text,
-        &query.kind,
-        &query.limit,
+        Some(query_params),
     )
     .await?;
-    Ok(Json(asks))
+    Ok::<_, Error>((
+        status_code,
+        headers,
+        StreamBody::new(response.bytes_stream()),
+    ))
 }
 
 fn hash_script(ns: &NewScript) -> i64 {