From ef3e4b2623d07e605d0507983f7976ec6656b4f6 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 19 Oct 2023 17:27:42 +0200 Subject: [PATCH] feat: local hub embeddings search (#2463) * feat: local hub embeddings search * fix: cargo lock * fix: sqlx prepare * fix: cargo tinyvector * feat: add DISABLE_EMBEDDING env var * fix: cargo lock --- README.md | 1 + ...d578c31cf151ab3ef813f7aada76533761e12.json | 38 + backend/Cargo.lock | 756 +++++++++++++++++- backend/Cargo.toml | 8 +- backend/windmill-api/Cargo.toml | 8 +- backend/windmill-api/openapi.yaml | 189 +++-- backend/windmill-api/src/embeddings.rs | 462 +++++++++++ backend/windmill-api/src/integration.rs | 35 + backend/windmill-api/src/lib.rs | 5 +- backend/windmill-api/src/resources.rs | 56 +- backend/windmill-api/src/scripts.rs | 45 +- .../src/lib/components/FlowBuilder.svelte | 28 +- .../src/lib/components/ScriptPicker.svelte | 28 +- .../src/lib/components/copilot/StepGen.svelte | 24 +- frontend/src/lib/components/copilot/flow.ts | 4 +- frontend/src/lib/components/copilot/lib.ts | 21 +- .../flows/pickers/PickHubScript.svelte | 100 ++- frontend/src/lib/scripts.ts | 23 +- frontend/src/lib/stores.ts | 11 - 19 files changed, 1527 insertions(+), 315 deletions(-) create mode 100644 backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json create mode 100644 backend/windmill-api/src/embeddings.rs create mode 100644 backend/windmill-api/src/integration.rs diff --git a/README.md b/README.md index 52cecf90f0..61b45d91f4 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,7 @@ it being synced automatically everyday. | SAML_METADATA | None | SAML Metadata URL to enable SAML SSO (EE only) | Server | | SECRET_SALT | None | Secret Salt used for encryption and decryption of secrets. If defined, the secrets will not be decryptable unless the right salt is passed in, which is the case for the workers and the server | Server + Worker | | OPENAI_AZURE_BASE_PATH | None | Azure OpenAI API base path (no trailing slash) | Server | +| DISABLE_EMBEDDING | false | Disable local embedding search of hub scripts | Server | | DISABLE_NSJAIL | true | Disable Nsjail Sandboxing | Worker | | DISABLE_SERVER | false | Disable the external API, operate as a worker only instance | Worker | diff --git a/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json b/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json new file mode 100644 index 0000000000..948e13e994 --- /dev/null +++ b/backend/.sqlx/query-eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * from resource_type ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "schema", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "description", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "eb1f7f01461f5a7540c273b37e5d578c31cf151ab3ef813f7aada76533761e12" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9c456a6112..97225bc914 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -142,7 +142,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -152,7 +152,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -853,6 +853,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + [[package]] name = "byteorder" version = "1.5.0" @@ -886,6 +892,184 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "candle-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d648a16c5e5a9ffa4a6630e13d32b6d7c106f6885fd6d8453e1d8b405c390322" +dependencies = [ + "byteorder", + "candle-gemm", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.8.5", + "rand_distr", + "rayon", + "safetensors", + "thiserror", + "yoke", + "zip", +] + +[[package]] +name = "candle-gemm" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9b07a4b0ba1a304b44432006580980ddff9748c201261c279437e7b11bba68" +dependencies = [ + "candle-gemm-c32", + "candle-gemm-c64", + "candle-gemm-common", + "candle-gemm-f16", + "candle-gemm-f32", + "candle-gemm-f64", + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-c32" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f595241dad99811de285e029889f57c29dd98e33de7a8a6b881867b1488d7d4a" +dependencies = [ + "candle-gemm-common", + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-c64" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "648f22fd8f5a4f330e29d791845b514966421308a6a2b5fedb949ee07e54c77f" +dependencies = [ + "candle-gemm-common", + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-common" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e03c01b4ca3b9d71e4eb89e42946a08f8b0d2f1b861f7fa2ea0966233f1e0b08" +dependencies = [ + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-f16" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97f8af2a482131713d28a337abff6debf26c529afa1837caf2ba190909b2107c" +dependencies = [ + "candle-gemm-common", + "candle-gemm-f32", + "dyn-stack", + "half", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-f32" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938927961e2f0c0a6064fcf3524ea3f7f455fe5708419532a6fea9aea1ab45ae" +dependencies = [ + "candle-gemm-common", + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-gemm-f64" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d192d7126e59b81ef4cf13cd9f194e6dbdc09171f65d0074d059dc009ac06775" +dependencies = [ + "candle-gemm-common", + "dyn-stack", + "lazy_static", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "candle-nn" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e438605fbbd1235dbfc5b10beda5ebe4bb0a19969b864a85b922c97ac9f686" +dependencies = [ + "candle-core", + "half", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror", +] + +[[package]] +name = "candle-transformers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196742c5676b9af54c782304609f3e480c871e5588451d9806b2df8fc11e324c" +dependencies = [ + "candle-core", + "candle-nn", + "num-traits", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "tracing", + "wav", +] + [[package]] name = "cargo-lock" version = "9.0.0" @@ -935,7 +1119,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -1058,6 +1242,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.45.0", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1575,7 +1772,7 @@ dependencies = [ "serde", "tokio", "uuid 1.5.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1701,6 +1898,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -1711,6 +1917,18 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -1762,6 +1980,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "131726693bce13b09331bee70734fe266666332b6ddfef23e9dca5b8bf6dea66" +[[package]] +name = "dyn-stack" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe7f8d7bcc523381d3c437b82cf74805de3931de0da69309ae0fe1bdf7a256e" +dependencies = [ + "bytemuck", + "reborrow", +] + [[package]] name = "either" version = "1.9.0" @@ -1780,6 +2008,12 @@ dependencies = [ "log", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -1802,7 +2036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", ] [[package]] @@ -1813,7 +2056,7 @@ checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ "cfg-if", "home", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1852,7 +2095,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall 0.3.5", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2157,7 +2400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" dependencies = [ "libc", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -2283,6 +2526,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "rand 0.8.5", + "rand_distr", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2365,6 +2621,23 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" +dependencies = [ + "dirs", + "indicatif", + "log", + "native-tls", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror", + "ureq", +] + [[package]] name = "hkdf" version = "0.12.3" @@ -2389,7 +2662,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2571,6 +2844,19 @@ dependencies = [ "hashbrown 0.14.1", ] +[[package]] +name = "indicatif" +version = "0.17.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + [[package]] name = "infer" version = "0.2.3" @@ -2594,7 +2880,7 @@ checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2624,7 +2910,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", "rustix 0.38.20", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2987,6 +3273,22 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + [[package]] name = "magic-crypt" version = "3.1.12" @@ -3077,6 +3379,16 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", + "stable_deref_trait", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -3126,7 +3438,28 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "monostate" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", ] [[package]] @@ -3354,6 +3687,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.1" @@ -3369,6 +3708,28 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "opaque-debug" version = "0.3.0" @@ -3430,6 +3791,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "outref" version = "0.5.1" @@ -3468,7 +3835,7 @@ dependencies = [ "libc", "redox_syscall 0.4.1", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -3805,9 +4172,15 @@ dependencies = [ "libc", "log", "pin-project-lite", - "windows-sys", + "windows-sys 0.48.0", ] +[[package]] +name = "portable-atomic" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" + [[package]] name = "postgres" version = "0.19.7" @@ -4201,6 +4574,16 @@ dependencies = [ "getrandom 0.2.10", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -4210,6 +4593,52 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redis" version = "0.23.3" @@ -4308,6 +4737,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "regex-syntax" version = "0.8.2" @@ -4380,6 +4815,12 @@ dependencies = [ "winreg", ] +[[package]] +name = "riff" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1" + [[package]] name = "ring" version = "0.16.20" @@ -4406,7 +4847,7 @@ dependencies = [ "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4600,7 +5041,7 @@ dependencies = [ "io-lifetimes", "libc", "linux-raw-sys 0.3.8", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4613,7 +5054,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.10", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4725,6 +5166,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +[[package]] +name = "safetensors" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93279b86b3de76f820a8854dd06cbc33cfa57a417b19c47f6a25280112fb1df" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "samael" version = "0.0.12" @@ -4773,7 +5224,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4895,6 +5346,12 @@ dependencies = [ "pest", ] +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + [[package]] name = "serde" version = "1.0.189" @@ -5252,7 +5709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -5306,6 +5763,18 @@ dependencies = [ "der 0.7.8", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sql-builder" version = "3.1.1" @@ -6022,6 +6491,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "285ba80e733fac80aa4270fbcdf83772a79b80aa35c97075320abfee4a915b06" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", + "unicode-xid", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -6059,7 +6540,7 @@ dependencies = [ "fastrand 2.0.1", "redox_syscall 0.3.5", "rustix 0.38.20", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6185,6 +6666,54 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tinyvector" +version = "0.1.0" +source = "git+https://github.com/windmill-labs/tinyvector#20823b94c20f2b9093f318badd24026cf54dcc85" +dependencies = [ + "anyhow", + "bincode", + "lazy_static", + "rayon", + "schemars", + "serde", + "thiserror", + "tracing", +] + +[[package]] +name = "tokenizers" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c" +dependencies = [ + "aho-corasick", + "clap", + "derive_builder", + "esaxx-rs", + "getrandom 0.2.10", + "indicatif", + "itertools 0.11.0", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.8.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax 0.7.5", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.32.0" @@ -6202,7 +6731,7 @@ dependencies = [ "socket2 0.5.4", "tokio-macros", "tracing", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -6740,6 +7269,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-segmentation" version = "1.10.1" @@ -6788,6 +7326,25 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5ccd538d4a604753ebc2f17cd9946e89b77bf87f6a8e2309667c6f2e87855e3" +dependencies = [ + "base64 0.21.4", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls", + "rustls-webpki", + "serde", + "serde_json", + "url", + "webpki-roots 0.25.2", +] + [[package]] name = "url" version = "2.4.1" @@ -7026,6 +7583,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wav" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609" +dependencies = [ + "riff", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -7170,6 +7736,9 @@ dependencies = [ "axum", "base64 0.21.4", "bytes", + "candle-core", + "candle-nn", + "candle-transformers", "chrono", "chrono-tz", "cookie", @@ -7177,6 +7746,7 @@ dependencies = [ "futures", "git-version", "hex", + "hf-hub", "hmac", "hyper", "itertools 0.11.0", @@ -7201,6 +7771,8 @@ dependencies = [ "sqlx", "tempfile", "time", + "tinyvector", + "tokenizers", "tokio", "tokio-tar", "tokio-util", @@ -7489,7 +8061,16 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", ] [[package]] @@ -7498,7 +8079,22 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -7507,51 +8103,93 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -7574,7 +8212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -7604,12 +8242,68 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yoke" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e38c508604d6bbbd292dadb3c02559aa7fff6b654a078a36217cad871636e4" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e19fb6ed40002bab5403ffa37e53e0e56f914a4450c8765f533018db1db35f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655b0814c5c0b19ade497851070c640773304939a6c0fd5f5fb43da0696d05b7" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6a647510471d372f2e6c2e6b7219e44d8c574d24fdc11c610a61455782f18c3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", + "synstructure", +] + [[package]] name = "zeroize" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam-utils", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 2ec3055c4d..74d8c25699 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -193,4 +193,10 @@ gcp_auth = "0.9.0" rust_decimal = { version = "1.31.0", features = ["db-postgres"]} jsonwebtoken = "8.3.0" pem = "3.0.1" -nix = { version = "0.27.1", features = ["process", "signal"] } \ No newline at end of file +nix = { version = "0.27.1", features = ["process", "signal"] } +tinyvector = { git = "https://github.com/windmill-labs/tinyvector" } +hf-hub = "0.3.2" +tokenizers = "0.14.1" +candle-core = "0.3.0" +candle-transformers = "0.3.0" +candle-nn = "0.3.0" diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 13c4b7fdee..993a336411 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -75,4 +75,10 @@ mail-send.workspace = true samael = { workspace = true, optional = true } async-recursion.workspace = true rsa.workspace = true -uuid.workspace = true \ No newline at end of file +uuid.workspace = true +tinyvector.workspace = true +hf-hub.workspace = true +tokenizers.workspace = true +candle-core.workspace = true +candle-transformers.workspace = true +candle-nn.workspace = true \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 87746e74ec..1af4190720 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2363,35 +2363,10 @@ paths: items: type: string - /resources/type/hub/list: + /embeddings/query_resource_types: get: - summary: list hub resource types - operationId: listHubResourceTypes - tags: - - resource - responses: - "200": - description: resource type details - content: - application/json: - schema: - type: array - items: - type: object - properties: - id: - type: string - name: - type: string - schema: {} - required: - - id - - name - - /resources/type/hub/query: - get: - summary: query hub resource types by similarity - operationId: queryHubResourceTypes + summary: query resource types by similarity + operationId: queryResourceTypes tags: - resource parameters: @@ -2417,56 +2392,42 @@ paths: items: type: object properties: - id: + name: type: string + score: + type: number + schema: {} required: - - id + - name + - score - /scripts/hub/list: + /integrations/hub/list: get: - summary: list all available hub scripts - operationId: listHubScripts + summary: list hub integrations + operationId: listHubIntegrations tags: - - script + - integration + parameters: + - name: kind + description: query integrations kind + in: query + required: false + schema: + type: string responses: "200": - description: hub scripts list + description: integrations details content: application/json: schema: - type: object - properties: - asks: - type: array - items: - type: object - properties: - id: - type: number - ask_id: - type: number - summary: - type: string - app: - type: string - approved: - type: boolean - kind: - type: string - enum: [script, failure, trigger, command, approval] - votes: - type: number - views: - type: number - required: - - id - - ask_id - - summary - - app - - approved - - kind - - views - - votes + type: array + items: + type: object + properties: + name: + type: string + required: + - name /flows/hub/list: get: @@ -2642,8 +2603,73 @@ paths: required: - content - language + + /scripts/hub/top: + get: + summary: get top hub scripts + operationId: getTopHubScripts + tags: + - script + parameters: + - name: limit + description: query limit + in: query + required: false + schema: + type: number + - name: app + description: query scripts app + in: query + required: false + schema: + type: string + - name: kind + description: query scripts kind + in: query + required: false + schema: + type: string + responses: + "200": + description: hub scripts list + content: + application/json: + schema: + type: object + properties: + asks: + type: array + items: + type: object + properties: + id: + type: number + ask_id: + type: number + summary: + type: string + app: + type: string + approved: + type: boolean + kind: + type: string + enum: [script, failure, trigger, command, approval] + votes: + type: number + views: + type: number + required: + - id + - ask_id + - summary + - app + - approved + - kind + - views + - votes - /scripts/hub/query: + /embeddings/query_hub_scripts: get: summary: query hub scripts by similarity operationId: queryHubScripts @@ -2668,6 +2694,12 @@ paths: required: false schema: type: number + - name: app + description: query scripts app + in: query + required: false + schema: + type: string responses: "200": description: script details @@ -2678,10 +2710,29 @@ paths: items: type: object properties: + ask_id: + type: number id: + type: number + version_id: + type: number + summary: type: string + app: + type: string + kind: + type: string + enum: [script, failure, trigger, command, approval] + score: + type: number required: + - ask_id - id + - version_id + - summary + - app + - kind + - score /w/{workspace}/scripts/list_search: get: diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs new file mode 100644 index 0000000000..c487e23452 --- /dev/null +++ b/backend/windmill-api/src/embeddings.rs @@ -0,0 +1,462 @@ +use std::{collections::HashMap, sync::Arc}; + +use anyhow::{self, Error, Result}; +use axum::{extract::Query, routing::get, Extension, Json, Router}; +use candle_core::{Device, Tensor}; +use candle_nn::VarBuilder; +use candle_transformers::models::bert::{BertModel, Config, DTYPE}; +use hf_hub::{api::sync::Api, Cache, Repo}; +use serde::{Deserialize, Serialize}; +use sqlx::{Pool, Postgres}; +use tinyvector::{ + db::{Db, Embedding}, + similarity::Distance, +}; +use tokenizers::Tokenizer; +use tokio::sync::RwLock; +use windmill_common::{error::JsonResult, utils::http_get_from_hub}; + +use crate::{resources::ResourceType, HTTP_CLIENT}; + +#[derive(Deserialize)] +struct HubScriptsQuery { + text: String, + limit: Option, + kind: Option, + app: Option, +} + +#[derive(Serialize)] +pub struct HubScriptResult { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + score: f32, +} + +async fn query_hub_scripts( + Query(query): Query, + Extension(embeddings_db): Extension>>>, +) -> JsonResult> { + let embeddings_db = embeddings_db.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_hub_scripts(&query.text, query.limit, query.kind, query.app) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::InternalErr( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[derive(Deserialize)] +struct ResourceTypesQuery { + text: String, + limit: Option, +} + +#[derive(Serialize)] +pub struct ResourceTypeResult { + name: String, + score: f32, + schema: Option, +} +async fn query_resource_types( + Query(query): Query, + Extension(embeddings_db): Extension>>>, +) -> JsonResult> { + let embeddings_db = embeddings_db.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_resource_types(&query.text, query.limit) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::InternalErr( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[derive(Deserialize, Debug, Clone)] +struct HubScript { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + embedding: Vec, +} + +#[derive(Deserialize, Debug)] +struct HubResourceType { + name: String, + embedding: Vec, +} + +pub struct ModelInstance { + model: BertModel, + tokenizer: Tokenizer, +} + +impl ModelInstance { + pub async fn new() -> Result { + let device = Device::Cpu; + let model_id = "thenlper/gte-small".to_string(); + + let repo = Repo::model(model_id); + + let cache = Cache::default().repo(repo.clone()); + + let api = Api::new()?; + let api = api.repo(repo); + + let (config_filename, tokenizer_filename, weights_filename) = ( + cache + .get("config.json") + .or_else(|| api.get("config.json").ok()) + .ok_or(Error::msg("could not get config.json"))?, + cache + .get("tokenizer.json") + .or_else(|| api.get("tokenizer.json").ok()) + .ok_or(Error::msg("could not get tokenizer.json"))?, + cache + .get("model.safetensors") + .or_else(|| api.get("model.safetensors").ok()) + .ok_or(Error::msg("could not get model.safetensors"))?, + ); + let config = std::fs::read_to_string(config_filename)?; + let config: Config = serde_json::from_str(&config)?; + let tokenizer = Tokenizer::from( + Tokenizer::from_file(tokenizer_filename) + .map_err(Error::msg)? + .with_padding(None) + .with_truncation(None) + .map_err(Error::msg)? + .to_owned(), + ); + + let vb = + unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? }; + let model = BertModel::load(vb, &config)?; + Ok(Self { model, tokenizer }) + } + + pub async fn create_embedding(self: Arc, sentence: &str) -> Result> { + let sentence = sentence.to_owned(); + tokio::task::spawn_blocking(move || { + let tokens = self + .tokenizer + .encode(sentence, true) + .map_err(Error::msg)? + .get_ids() + .to_vec(); + + let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; + let token_type_ids = token_ids.zeros_like()?; + + let embedding = self.model.forward(&token_ids, &token_type_ids)?; + let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; + let embedding = normalize_l2(&embedding)?; + + let embedding = embedding.get(0)?.to_vec1()?; + + Ok(embedding) + }) + .await? + } +} + +pub struct EmbeddingsDb { + db: Db, + model_instance: Arc, +} + +impl EmbeddingsDb { + pub async fn new(pg_db: &Pool, model_instance: Arc) -> Result { + let db = Db::new(); + + let mut embeddings_db = Self { db, model_instance: model_instance.clone() }; + + embeddings_db.fill_db(pg_db).await?; + + Ok(embeddings_db) + } + + async fn fill_db(&mut self, pg_db: &Pool) -> Result<()> { + if self.db.get_collection("scripts").is_some() { + self.db.delete_collection("scripts")?; + } + + self.db + .create_collection("scripts".to_string(), 384, Distance::Cosine)?; + + if self.db.get_collection("resource_types").is_some() { + self.db.delete_collection("resource_types")?; + } + + self.db + .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; + + let response = http_get_from_hub( + &HTTP_CLIENT, + "https://hub.windmill.dev/scripts/embeddings", + "todo@windmill.dev", + false, + None, + ) + .await?; + let hub_scripts = response.json::>().await?; + + for script in &hub_scripts { + let mut hm = HashMap::new(); + hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); + hm.insert("summary".to_string(), script.summary.clone()); + hm.insert("app".to_string(), script.app.clone()); + hm.insert("kind".to_string(), script.kind.clone()); + hm.insert("id".to_string(), script.id.clone().to_string()); + hm.insert( + "version_id".to_string(), + script.version_id.clone().to_string(), + ); + let embedding = Embedding { + id: script.ask_id.clone().to_string(), + vector: script.embedding.clone(), + metadata: Some(hm), + }; + self.db.insert_into_collection("scripts", embedding)?; + } + + let response = http_get_from_hub( + &HTTP_CLIENT, + "https://hub.windmill.dev/resource_types/embeddings", + "todo@windmill.dev", + false, + None, + ) + .await?; + let hub_resource_types = response.json::>().await?; + + let resource_types: Vec = sqlx::query_as!( + ResourceType, + "SELECT * from resource_type ORDER \ + BY name", + ) + .fetch_all(pg_db) + .await?; + + for rt in resource_types { + let mut hm = HashMap::new(); + hm.insert("name".to_string(), rt.name.clone()); + if let Some(schema) = rt.schema.clone() { + hm.insert("schema".to_string(), serde_json::to_string(&schema)?); + } + let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name); + + let vector = if let Some(hub_rt) = hub_rt { + hub_rt.embedding.clone() + } else { + self.model_instance + .clone() + .create_embedding(&format!( + "{};{}", + rt.name, + rt.description.unwrap_or_default() + )) + .await? + }; + + let embedding = Embedding { id: rt.name.clone(), vector, metadata: Some(hm) }; + self.db + .insert_into_collection("resource_types", embedding)?; + } + + Ok(()) + } + + pub async fn query_hub_scripts( + &self, + query: &str, + limit: Option, + kind: Option, + app: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("scripts"); + + let collection = collection.ok_or(Error::msg("no collection found"))?; + + let filter = |embedding: &Embedding| { + if let Some(metadata) = embedding.metadata.as_ref() { + match ( + metadata.get("kind"), + kind.clone(), + metadata.get("app"), + app.clone(), + ) { + (Some(script_kind), Some(kind), Some(script_app), Some(app)) => { + &kind == script_kind && &app == script_app + } + (Some(script_kind), Some(kind), _, _) => &kind == script_kind, + (_, _, Some(script_app), Some(app)) => &app == script_app, + (_, None, _, None) => true, + _ => false, + } + } else { + false + } + }; + + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + Some(&filter), + Some(0.75), + ); + + let results: Result> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + + Ok(HubScriptResult { + ask_id: metadata + .get("ask_id") + .ok_or(Error::msg("no ask_id"))? + .parse::()?, + summary: metadata + .get("summary") + .ok_or(Error::msg("no summary"))? + .to_owned(), + app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), + kind: metadata + .get("kind") + .ok_or(Error::msg("no kind"))? + .to_owned(), + id: metadata + .get("id") + .ok_or(Error::msg("no id"))? + .parse::()?, + version_id: metadata + .get("version_id") + .ok_or(Error::msg("no version_id"))? + .parse::()?, + score: r.score, + }) + }) + .collect(); + + results + } + + pub async fn query_resource_types( + &self, + query: &str, + limit: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("resource_types"); + + if collection.is_none() { + return Ok(vec![]); + } + + let collection = collection.ok_or(Error::msg("no collection found"))?; + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + None, + Some(0.75), + ); + + let results: Result<_> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + Ok(ResourceTypeResult { + name: metadata + .get("name") + .ok_or(Error::msg("no name"))? + .to_owned(), + schema: match metadata.get("schema") { + Some(schema) => serde_json::from_str(schema)?, + None => None, + }, + score: r.score, + }) + }) + .collect(); + + results + } +} + +fn normalize_l2(v: &Tensor) -> Result { + Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?) +} + +pub fn global_service(db: &Pool) -> Router { + let embeddings_db: Arc>> = Arc::new(RwLock::new(None)); + + let disable_embedding = std::env::var("DISABLE_EMBEDDING") + .ok() + .map(|x| x.parse::().unwrap_or(false)) + .unwrap_or(false); + + if !disable_embedding { + let db_clone = db.clone(); + let embeddings_clone: Arc>> = embeddings_db.clone(); + tokio::spawn(async move { + let model_instance = ModelInstance::new().await; + + if let Ok(model_instance) = model_instance { + let model_instance = Arc::new(model_instance); + loop { + let new_embeddings_db = + EmbeddingsDb::new(&db_clone, model_instance.clone()).await; + if let Err(e) = new_embeddings_db.as_ref() { + tracing::error!("Failed to create embeddings db: {}", e); + } else { + let mut embeddings_db = embeddings_clone.write().await; + *embeddings_db = new_embeddings_db.ok(); + tracing::info!("Loaded embeddings DB"); + } + + tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await; + } + } else { + tracing::error!( + "Failed to initialize model instance: {}", + model_instance.err().unwrap() + ); + } + }); + } + + Router::new() + .route("/query_hub_scripts", get(query_hub_scripts)) + .route("/query_resource_types", get(query_resource_types)) + .layer(Extension(embeddings_db)) +} diff --git a/backend/windmill-api/src/integration.rs b/backend/windmill-api/src/integration.rs new file mode 100644 index 0000000000..1cdfc6890d --- /dev/null +++ b/backend/windmill-api/src/integration.rs @@ -0,0 +1,35 @@ +use crate::{db::ApiAuthed, HTTP_CLIENT}; +use axum::{body::StreamBody, extract::Query, response::IntoResponse, routing::get, Router}; +use windmill_common::{error::Error, utils::query_elems_from_hub}; + +pub fn global_service() -> Router { + Router::new().route("/hub/list", get(list_hub_integrations)) +} + +#[derive(serde::Deserialize)] +struct ListHubIntegrationsQuery { + kind: Option, +} +async fn list_hub_integrations( + ApiAuthed { email, .. }: ApiAuthed, + Query(query): Query, +) -> impl IntoResponse { + let mut query_params = vec![]; + + if let Some(kind) = query.kind { + query_params.push(("kind", kind)); + } + + let (status_code, headers, response) = query_elems_from_hub( + &HTTP_CLIENT, + "https://hub.windmill.dev/integrations/list", + &email, + Some(query_params), + ) + .await?; + Ok::<_, Error>(( + status_code, + headers, + StreamBody::new(response.bytes_stream()), + )) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 42d5317077..0e5132be43 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -48,12 +48,14 @@ mod configs; mod db; mod drafts; pub mod ee; +mod embeddings; mod favorite; mod flows; mod folders; mod granular_acls; mod groups; mod inputs; +mod integration; pub mod jobs; pub mod oauth2; mod openai; @@ -204,11 +206,12 @@ pub async fn run_server( .nest("/workers", workers::global_service()) .nest("/configs", configs::global_service()) .nest("/scripts", scripts::global_service()) - .nest("/resources", resources::global_service()) + .nest("/integrations", integration::global_service()) .nest("/groups", groups::global_service()) .nest("/flows", flows::global_service()) .nest("/apps", apps::global_service().layer(cors.clone())) .nest("/schedules", schedule::global_service()) + .nest("/embeddings", embeddings::global_service(&db)) .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/jobs", jobs::global_root_service()) diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 00e14a78d2..e8d9cefdce 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -10,12 +10,9 @@ use crate::{ db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, webhook_util::{WebhookMessage, WebhookShared}, - HTTP_CLIENT, }; use axum::{ - body::StreamBody, extract::{Extension, Path, Query}, - response::IntoResponse, routing::{delete, get, post}, Json, Router, }; @@ -30,18 +27,10 @@ use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, jobs::QueuedJob, - utils::{ - not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, - }, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, }; -pub fn global_service() -> Router { - Router::new() - .route("/type/hub/list", get(list_hub_resource_types)) - .route("/type/hub/query", get(query_hub_resource_types)) -} - pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_resources)) @@ -912,46 +901,3 @@ async fn update_resource_type( Ok(format!("resource_type {} updated", name)) } - -async fn list_hub_resource_types(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse { - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - "https://hub.windmill.dev/resource_types/list", - &email, - None, - ) - .await?; - Ok::<_, Error>(( - status_code, - headers, - StreamBody::new(response.bytes_stream()), - )) -} - -#[derive(Deserialize)] -struct HubResourceTypesQuery { - text: String, - limit: Option, -} -async fn query_hub_resource_types( - ApiAuthed { email, .. }: ApiAuthed, - Query(query): Query, -) -> impl IntoResponse { - let mut query_params = vec![("text", query.text)]; - if let Some(query_limit) = query.limit { - query_params.push(("limit", query_limit.to_string().clone())); - } - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - "https://hub.windmill.dev/resource_types/query", - &email, - Some(query_params), - ) - .await?; - - Ok::<_, Error>(( - status_code, - headers, - StreamBody::new(response.bytes_stream()), - )) -} diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 01e8cd1212..91390eac87 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -82,10 +82,9 @@ pub struct ScriptWDraft { pub fn global_service() -> Router { Router::new() - .route("/hub/list", get(list_hub_scripts)) + .route("/hub/top", get(get_top_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) - .route("/hub/query", get(query_hub_scripts)) } pub fn global_unauthed_service() -> Router { @@ -249,41 +248,31 @@ async fn list_scripts( Ok(Json(rows)) } -async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> impl IntoResponse { - let (status_code, headers, response) = query_elems_from_hub( - &HTTP_CLIENT, - "https://hub.windmill.dev/searchData?approved=true", - &email, - None, - ) - .await?; - Ok::<_, Error>(( - status_code, - headers, - StreamBody::new(response.bytes_stream()), - )) +#[derive(Deserialize)] +struct TopHubScriptsQuery { + limit: Option, + app: Option, + kind: Option, } -#[derive(Deserialize)] -struct HubScriptsQuery { - text: String, - kind: Option, - limit: Option, -} -async fn query_hub_scripts( +async fn get_top_hub_scripts( ApiAuthed { email, .. }: ApiAuthed, - Query(query): Query, + Query(query): Query, ) -> impl IntoResponse { - let mut query_params = vec![("text", query.text)]; - if let Some(query_kind) = query.kind { - query_params.push(("kind", query_kind.clone())); - } + let mut query_params = vec![]; if let Some(query_limit) = query.limit { query_params.push(("limit", query_limit.to_string().clone())); } + if let Some(query_app) = query.app { + query_params.push(("app", query_app.to_string().clone())); + } + if let Some(query_kind) = query.kind { + query_params.push(("kind", query_kind.to_string().clone())); + } + let (status_code, headers, response) = query_elems_from_hub( &HTTP_CLIENT, - "https://hub.windmill.dev/scripts/query", + "https://hub.windmill.dev/scripts/top", &email, Some(query_params), ) diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 3ce04b5c34..84192becca 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -13,7 +13,6 @@ import { copilotInfo, enterpriseLicense, - hubScripts, tutorialsToDo, userStore, workspaceStore @@ -39,7 +38,6 @@ import type { FlowEditorContext } from './flows/types' import { cleanInputs, emptyFlowModuleState } from './flows/utils' import { Pen } from 'lucide-svelte' - import { loadHubScripts } from '$lib/scripts' import { createEventDispatcher } from 'svelte' import Awareness from './Awareness.svelte' import { getAllModules } from './flows/flowExplorer' @@ -303,8 +301,6 @@ $: initialPath && $workspaceStore && loadSchedule() - loadHubScripts() - function onKeyDown(event: KeyboardEvent) { let classes = event.target?.['className'] if ( @@ -400,25 +396,23 @@ try { // make sure we display the results of the last request last const ts = Date.now() - const scriptIds = await ScriptService.queryHubScripts({ - text: `${text}`, - limit: 3, - kind: type - }) + const scripts = ( + await ScriptService.queryHubScripts({ + text: `${text}`, + limit: 3, + kind: type + }) + ).map((s) => ({ + ...s, + path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, + summary: `${s.summary} (${s.app})` + })) if (ts < doneTs) return doneTs = ts - const scripts = scriptIds - .map((qs) => { - const s = $hubScripts?.find((hs) => hs.ask_id === Number(qs.id)) - return s - }) - .filter((s) => !!s) - $copilotModulesStore[idx].hubCompletions = scripts as { path: string summary: string - approved: boolean kind: string app: string ask_id: number diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 390938951d..90d17adf95 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -1,7 +1,7 @@ - x.summary} /> +
- -{#if $hubScripts} - {#if filteredItems.length == 0} +{#if items.length > 0 && apps.length > 0} + + {#if items.length == 0} {:else}
    - {#each filteredItems.slice(0, maxItems) as item (item.path)} + {#each items as item (item.path)}
{/if} - {#if filteredItems.length > maxItems} + {#if items.length == 40}
- There are more items ({filteredItems.length}) than being displayed. Refine your search. + There are more items than being displayed. Refine your search.
{/if} {:else} diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 1cafba7e6a..968243e893 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -1,7 +1,7 @@ import { get } from 'svelte/store' import type { Schema, SupportedLanguage } from './common' import { FlowService, Script, ScriptService } from './gen' -import { workspaceStore, hubScripts } from './stores' +import { workspaceStore } from './stores' export function scriptLangToEditorLang(lang: Script.language) { if (lang == 'deno') { @@ -94,24 +94,3 @@ export async function getLatestHashForScript(path: string): Promise { }) return script.hash } - -export async function loadHubScripts() { - try { - const scripts = (await ScriptService.listHubScripts()).asks ?? [] - const processed = scripts - .map((x) => ({ - path: `hub/${x.id}/${x.app}/${x.summary.toLowerCase().replaceAll(/\s+/g, '_')}`, - summary: `${x.summary} (${x.app})`, - approved: x.approved, - kind: x.kind, - app: x.app, - views: x.views, - votes: x.votes, - ask_id: x.ask_id - })) - .sort((a, b) => b.views - a.views) - hubScripts.set(processed) - } catch { - console.error('Hub is not available') - } -} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index ba12e96dea..3dd4923307 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -58,17 +58,6 @@ export const userWorkspaces: Readable< return originalWorkspaces } }) -export const hubScripts = writable< - | Array<{ - path: string - summary: string - approved: boolean - kind: string - app: string - ask_id: number - }> - | undefined ->(undefined) export const copilotInfo = writable<{ exists_openai_resource_path: boolean code_completion_enabled: boolean