From 9ff8a85af62e56e4f44fe454ac9e4c25cd0d8897 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 7 Feb 2026 23:12:55 +0100 Subject: [PATCH] refactor: extract windmill-api into subcrates for parallel compilation (#7845) * refactor: extract windmill-api into 4 subcrates (api-auth, store, api-sse, api-jobs) Co-Authored-By: Claude Opus 4.6 * refactor: eliminate refresh_token OnceLock bridge in windmill-store Co-Authored-By: Claude Opus 4.5 * refactor: eliminate FromRequestParts OnceLock bridge in windmill-api-auth Co-Authored-By: Claude Opus 4.6 * refactor: wire subcrates into workspace and clean up unused re-exports Co-Authored-By: Claude Opus 4.6 * fix: resolve cargo check --all-features errors in subcrate wiring Co-Authored-By: Claude Opus 4.6 * sqlx * all * chore: update ee-repo-ref for warning fixes Co-Authored-By: Claude Opus 4.6 * refactor: extract windmill-trigger crate and expand windmill-api-jobs Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-trigger-kafka crate from windmill-api Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-trigger-postgres crate from windmill-api Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-trigger-websocket and windmill-trigger-mqtt crates Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-trigger-nats, sqs, gcp, and email crates Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-trigger-http crate from windmill-api Co-Authored-By: Claude Opus 4.5 * refactor: move token creation and permission helpers to windmill-api-auth Co-Authored-By: Claude Opus 4.5 * refactor: extract windmill-native-triggers crate from windmill-api Co-Authored-By: Claude Opus 4.6 * sqlx * all * refactor: extract windmill-api-embeddings crate and fix CI warnings Co-Authored-By: Claude Opus 4.5 * fix: resolve type mismatch in oauth2_oss and remaining warnings Co-Authored-By: Claude Opus 4.5 * fix: use correct HTTP_CLIENT config in embeddings crate (30s timeout, cert override) Co-Authored-By: Claude Opus 4.5 * all * fix: gate oauth_refresh_ee on oauth2 feature to fix warnings Co-Authored-By: Claude Opus 4.5 * all --------- Co-authored-by: Claude Opus 4.6 --- ...001f29e58e81b3f782887db6e0a627dd8ad75.json | 3 +- ...b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json | 15 + ...82b33947c5a5c702ce586e6e6ea51430380bf.json | 3 +- ...1f7f387f5055c47f493271d26731336257384.json | 10 +- ...52354641c2f735d071540f858bc064f2432cd.json | 28 - ...2d54f97bc2d0ef52f5812244a97c289523592.json | 39 - ...fb8c6476d6b9b00b9a336b4616d3b69a491e.json} | 4 +- ...72f455571e0dd3006fb717b36dcab36689cca.json | 35 - ...d75281dada61fd1c6da1fa52624ebe0b1c962.json | 17 + ...cd029e99b029b7f1b71eda4411d1e24e790d.json} | 4 +- ...5dc0d34fc38ee129d736b28219b84e6474229.json | 52 - ...1d6f7894c25f90e39ec8606f79b8be56d1c17.json | 23 - ...f446d88c7bc671d5e90307febfd60bb235031.json | 3 +- ...e075c6ae51404037b0a7d4d33faf6a7d6a6d8.json | 3 +- ...a94ff708ce8e0b0b4fb59e5de7c9698c75c25.json | 3 +- ...823d8e54c96889e06895c57cafae9ca27bf8b.json | 3 +- ...9109986a0803d50164a259aabcdee81014d07.json | 23 - ...748e0639c2f73cbc1d8be66420c70b14c96e1.json | 3 +- ...73b6c84867fb966df91b320c54acdd5e61315.json | 37 + ...55ddf728cd3f21ee308db7393694ccc5c7b3.json} | 7 +- ...29b496f1f2caae145c47db296b6ea0089f86c.json | 26 + ...72f20bb5a0aa921a614bd02898d3f162660c2.json | 6 +- ...2271eaaddde1f4e3a4ab62628faee2e06e947.json | 26 - ...ab0f2348646d79f5306c6868e2c3a8f389ee.json} | 7 +- ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...1a803e1bced2f8e1a12f8933b7598ee85a147.json | 51 + ...1daa788a7551b4a00dd33f78d36e246b98dd7.json | 29 + ...a165ec4046c3a7172b5635b818c0fa133411f.json | 52 + ...53492943c961f98036133437ccc1a621ee89c.json | 18 - ...98d365acec09f2f102b7de71dcc9d356900e.json} | 7 +- ...26dd1665b3434f8624710ea5931958f459707.json | 23 + ...03041f8a893fe9ed2830aecaa4d245b07d739.json | 16 - ...3dc360b88429f03dd1e2b1b55cfabe208cb7.json} | 7 +- ...023c813e349956487f7b6599da452c068e9b9.json | 28 + ...2ddc8e84eacc633e361e49a5219d0bec94b9.json} | 4 +- ...04e38ab8474849e0cacb3be1443d7f3265d30.json | 46 - ...0cb549a34b96554ae1872355b90304f5dcb76.json | 4 +- ...0d16ee847294fadc138593a1baa417cbb3652.json | 4 +- ...2eb5ac25e84f3619a0b403a2d619a0eb51432.json | 29 - ...76d4bf44cb4397dd0a722e7c9d035b58a09c1.json | 35 + ...1e58936138d80a97b423c17338ba069b6a3aa.json | 29 - ...f892bf40e629ee11ee4845199481755f2789d.json | 6 +- ...579a6e821c432eac38ad877d9ff362d8ab916.json | 23 + ...746147aa9b8d0a81642a7b7cb4738a0cad66.json} | 4 +- ...671dd89e10000d3313884a663852337a76b2d.json | 17 - ...d434f860922fbc021185beffb9827647feb8e.json | 46 + ...3bf624a1f50d06c7de75157420d97cf144763.json | 37 + ...384514987b6a4eb807ee375fb8b5cd4e92307.json | 3 +- ...3f1f882b695f896f94e5cf91d205633355a1.json} | 4 +- ...d2ca5449dad0fe361bee1a9834989bfeabbae.json | 3 +- ...456ec563d346d99256ded52756cdb4a0d7ffd.json | 37 - ...1ea2fe96a1b94f5a92953fb1c1642d15c016.json} | 4 +- ...a8a9cc35f362197c0766a5787436af141106.json} | 4 +- ...9e78ca5db2c6677109b04dc477faf48de033.json} | 4 +- ...2c0ce2187e47d745aa5ba96f9453c722750c.json} | 4 +- ...b61e8e4e92b80b336d1a861e68f9ef782f8a.json} | 4 +- ...e29d333ada2dcfbbddb028f37794b85778ca8.json | 39 + ...bfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json | 18 + ...03cb0bf241d959a6c8f7bb896160001a6672b.json | 26 - ...03f72ef81db302d727e18e63a307ac902022.json} | 4 +- ...0078a25f117a3bfec78d5270a4c4970f2465e.json | 37 - ...26275203fc8ecddf83ca6d05c9fb77e46e7ee.json | 6 +- ...50d6a6935420fbedc83aa3ad1e9be7465c8dd.json | 40 + ...7d42e7add6011c34d91761ba23e2c74c4032c.json | 6 +- ...b1b9b29f978c73f003dfbf81d5c487f666358.json | 26 + ...f457e07f07e44c7b693d5d755899d4210dee.json} | 4 +- ...2f03f5972394fd927b3da216de66dab9e3e5a.json | 16 + backend/Cargo.lock | 468 +++- backend/Cargo.toml | 32 + backend/ee-repo-ref.txt | 2 +- backend/windmill-api-auth/Cargo.toml | 39 + backend/windmill-api-auth/src/auth.rs | 745 ++++++ backend/windmill-api-auth/src/ee_oss.rs | 36 + backend/windmill-api-auth/src/lib.rs | 712 ++++++ .../src/scopes.rs | 16 +- backend/windmill-api-embeddings/Cargo.toml | 32 + backend/windmill-api-embeddings/src/lib.rs | 637 +++++ backend/windmill-api-jobs/Cargo.toml | 34 + .../src/concurrency_groups.rs | 364 +++ backend/windmill-api-jobs/src/execution.rs | 892 +++++++ backend/windmill-api-jobs/src/job_metrics.rs | 266 +++ backend/windmill-api-jobs/src/jobs.rs | 4 + backend/windmill-api-jobs/src/jobs_export.rs | 690 ++++++ backend/windmill-api-jobs/src/lib.rs | 18 + backend/windmill-api-jobs/src/query.rs | 373 +++ backend/windmill-api-jobs/src/types.rs | 570 +++++ backend/windmill-api-sse/Cargo.toml | 21 + backend/windmill-api-sse/src/lib.rs | 341 +++ backend/windmill-api/Cargo.toml | 83 +- backend/windmill-api/src/apps.rs | 2 +- backend/windmill-api/src/auth.rs | 768 +----- backend/windmill-api/src/bedrock.rs | 27 +- .../windmill-api/src/concurrency_groups.rs | 365 +-- backend/windmill-api/src/db.rs | 126 +- backend/windmill-api/src/debug.rs | 29 +- backend/windmill-api/src/ee_oss.rs | 49 +- backend/windmill-api/src/embeddings.rs | 631 +---- .../windmill-api/src/flow_conversations.rs | 61 +- backend/windmill-api/src/folders.rs | 20 +- backend/windmill-api/src/groups.rs | 66 +- backend/windmill-api/src/job_metrics.rs | 267 +-- backend/windmill-api/src/jobs.rs | 2068 +---------------- backend/windmill-api/src/jobs_export.rs | 828 +------ backend/windmill-api/src/lib.rs | 20 +- backend/windmill-api/src/live_migrations.rs | 1 - backend/windmill-api/src/mcp/core.rs | 33 +- .../windmill-api/src/native_triggers/mod.rs | 1054 +-------- backend/windmill-api/src/npm_proxy.rs | 48 +- backend/windmill-api/src/oauth2_oss.rs | 45 +- backend/windmill-api/src/openapi.rs | 3 +- .../windmill-api/src/public_app_rate_limit.rs | 3 +- backend/windmill-api/src/resources.rs | 1822 +-------------- backend/windmill-api/src/schedule.rs | 4 +- .../windmill-api/src/secret_backend_ext.rs | 225 +- backend/windmill-api/src/slack_approvals.rs | 31 +- backend/windmill-api/src/teams_oss.rs | 1 - .../windmill-api/src/triggers/email/mod.rs | 12 +- backend/windmill-api/src/triggers/gcp/mod.rs | 17 +- backend/windmill-api/src/triggers/handler.rs | 708 +----- .../windmill-api/src/triggers/http/handler.rs | 609 +---- .../src/triggers/http/http_trigger_args.rs | 28 +- backend/windmill-api/src/triggers/http/mod.rs | 408 +--- .../windmill-api/src/triggers/kafka/mod.rs | 16 +- backend/windmill-api/src/triggers/listener.rs | 851 +------ backend/windmill-api/src/triggers/mod.rs | 182 +- backend/windmill-api/src/triggers/mqtt/mod.rs | 347 +-- backend/windmill-api/src/triggers/nats/mod.rs | 16 +- .../windmill-api/src/triggers/postgres/mod.rs | 554 +---- backend/windmill-api/src/triggers/sqs/mod.rs | 16 +- .../src/triggers/websocket/mod.rs | 129 +- backend/windmill-api/src/users.rs | 405 +--- backend/windmill-api/src/utils.rs | 57 +- backend/windmill-api/src/variables.rs | 921 +------- backend/windmill-api/src/webhook_util.rs | 14 +- backend/windmill-api/src/workers.rs | 6 +- backend/windmill-api/src/workspaces_export.rs | 31 +- backend/windmill-common/src/agent_workers.rs | 14 +- backend/windmill-common/src/ai_bedrock.rs | 8 +- backend/windmill-common/src/ai_providers.rs | 10 +- .../windmill-common/src/flow_conversations.rs | 63 +- backend/windmill-common/src/lib.rs | 138 +- backend/windmill-common/src/notify_events.rs | 5 +- backend/windmill-common/src/s3_helpers.rs | 15 +- .../src/secret_backend/database.rs | 5 +- .../src/secret_backend/vault_oss.rs | 5 +- backend/windmill-common/src/tracing_init.rs | 4 +- backend/windmill-common/src/utils.rs | 10 + backend/windmill-common/src/webhook.rs | 154 ++ backend/windmill-mcp/Cargo.toml | 3 +- .../windmill-mcp/src/client_registration.rs | 208 ++ backend/windmill-mcp/src/lib.rs | 4 + backend/windmill-native-triggers/Cargo.toml | 42 + .../src}/handler.rs | 40 +- backend/windmill-native-triggers/src/lib.rs | 1053 +++++++++ .../src}/nextcloud/external.rs | 2 +- .../src}/nextcloud/mod.rs | 0 .../src}/nextcloud/routes.rs | 10 +- .../src}/sync.rs | 13 +- .../src}/workspace_integrations.rs | 97 +- backend/windmill-oauth/src/lib.rs | 54 +- backend/windmill-store/Cargo.toml | 51 + backend/windmill-store/src/lib.rs | 16 + .../windmill-store/src/oauth_refresh_oss.rs | 38 + backend/windmill-store/src/resources.rs | 1702 ++++++++++++++ .../windmill-store/src/secret_backend_ext.rs | 371 +++ .../src/var_resource_cache.rs | 10 +- backend/windmill-store/src/variables.rs | 926 ++++++++ backend/windmill-trigger-email/Cargo.toml | 30 + .../src}/handler_oss.rs | 13 +- backend/windmill-trigger-email/src/lib.rs | 11 + backend/windmill-trigger-gcp/Cargo.toml | 44 + .../src}/handler_oss.rs | 11 +- backend/windmill-trigger-gcp/src/lib.rs | 15 + .../src}/listener_oss.rs | 19 +- backend/windmill-trigger-http/Cargo.toml | 46 + backend/windmill-trigger-http/src/handler.rs | 583 +++++ .../src}/http_trigger_auth.rs | 0 backend/windmill-trigger-http/src/lib.rs | 413 ++++ backend/windmill-trigger-kafka/Cargo.toml | 34 + .../src}/handler_oss.rs | 13 +- backend/windmill-trigger-kafka/src/lib.rs | 23 + .../src}/listener_oss.rs | 19 +- backend/windmill-trigger-mqtt/Cargo.toml | 34 + .../src}/handler.rs | 35 +- backend/windmill-trigger-mqtt/src/lib.rs | 347 +++ .../src}/listener.rs | 19 +- backend/windmill-trigger-nats/Cargo.toml | 34 + .../src}/handler_oss.rs | 14 +- backend/windmill-trigger-nats/src/lib.rs | 15 + .../src}/listener_oss.rs | 19 +- backend/windmill-trigger-postgres/Cargo.toml | 45 + .../src}/bool.rs | 0 .../src}/converter.rs | 6 +- .../src}/handler.rs | 33 +- .../src}/hex.rs | 8 +- backend/windmill-trigger-postgres/src/lib.rs | 552 +++++ .../src}/listener.rs | 29 +- .../src}/mapper.rs | 3 +- .../src}/relation.rs | 0 .../src}/replication_message.rs | 0 backend/windmill-trigger-sqs/Cargo.toml | 37 + .../src}/handler_oss.rs | 12 +- backend/windmill-trigger-sqs/src/lib.rs | 15 + .../src}/listener_oss.rs | 19 +- backend/windmill-trigger-websocket/Cargo.toml | 33 + .../src}/handler.rs | 9 +- backend/windmill-trigger-websocket/src/lib.rs | 126 + .../src}/listener.rs | 192 +- backend/windmill-trigger/Cargo.toml | 43 + backend/windmill-trigger/src/capture.rs | 127 + .../src}/filter.rs | 12 +- .../src}/global_handler.rs | 22 +- backend/windmill-trigger/src/handler.rs | 693 ++++++ backend/windmill-trigger/src/lib.rs | 19 + backend/windmill-trigger/src/listener.rs | 864 +++++++ .../src}/trigger_helpers.rs | 17 +- backend/windmill-trigger/src/types.rs | 158 ++ 217 files changed, 16761 insertions(+), 14660 deletions(-) create mode 100644 backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json delete mode 100644 backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json delete mode 100644 backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json rename backend/.sqlx/{query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json => query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json} (50%) delete mode 100644 backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json create mode 100644 backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json rename backend/.sqlx/{query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json => query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json} (72%) delete mode 100644 backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json delete mode 100644 backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json delete mode 100644 backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json create mode 100644 backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json rename backend/.sqlx/{query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json => query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json} (52%) create mode 100644 backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json delete mode 100644 backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json rename backend/.sqlx/{query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json => query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json} (59%) create mode 100644 backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json create mode 100644 backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json create mode 100644 backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json delete mode 100644 backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json rename backend/.sqlx/{query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json => query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json} (58%) create mode 100644 backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json delete mode 100644 backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json rename backend/.sqlx/{query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json => query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json} (70%) create mode 100644 backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json rename backend/.sqlx/{query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json => query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json} (59%) delete mode 100644 backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json delete mode 100644 backend/.sqlx/query-b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432.json create mode 100644 backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json delete mode 100644 backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json create mode 100644 backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json rename backend/.sqlx/{query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json => query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json} (67%) delete mode 100644 backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json create mode 100644 backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json create mode 100644 backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json rename backend/.sqlx/{query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json => query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json} (56%) delete mode 100644 backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json rename backend/.sqlx/{query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json => query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json} (67%) rename backend/.sqlx/{query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json => query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json} (58%) rename backend/.sqlx/{query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json => query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json} (64%) rename backend/.sqlx/{query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json => query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json} (69%) rename backend/.sqlx/{query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json => query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json} (53%) create mode 100644 backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json create mode 100644 backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json delete mode 100644 backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json rename backend/.sqlx/{query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json => query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json} (58%) delete mode 100644 backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json create mode 100644 backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json create mode 100644 backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json rename backend/.sqlx/{query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json => query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json} (58%) create mode 100644 backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json create mode 100644 backend/windmill-api-auth/Cargo.toml create mode 100644 backend/windmill-api-auth/src/auth.rs create mode 100644 backend/windmill-api-auth/src/ee_oss.rs create mode 100644 backend/windmill-api-auth/src/lib.rs rename backend/{windmill-api => windmill-api-auth}/src/scopes.rs (97%) create mode 100644 backend/windmill-api-embeddings/Cargo.toml create mode 100644 backend/windmill-api-embeddings/src/lib.rs create mode 100644 backend/windmill-api-jobs/Cargo.toml create mode 100644 backend/windmill-api-jobs/src/concurrency_groups.rs create mode 100644 backend/windmill-api-jobs/src/execution.rs create mode 100644 backend/windmill-api-jobs/src/job_metrics.rs create mode 100644 backend/windmill-api-jobs/src/jobs.rs create mode 100644 backend/windmill-api-jobs/src/jobs_export.rs create mode 100644 backend/windmill-api-jobs/src/lib.rs create mode 100644 backend/windmill-api-jobs/src/query.rs create mode 100644 backend/windmill-api-jobs/src/types.rs create mode 100644 backend/windmill-api-sse/Cargo.toml create mode 100644 backend/windmill-api-sse/src/lib.rs create mode 100644 backend/windmill-common/src/webhook.rs create mode 100644 backend/windmill-mcp/src/client_registration.rs create mode 100644 backend/windmill-native-triggers/Cargo.toml rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/handler.rs (93%) create mode 100644 backend/windmill-native-triggers/src/lib.rs rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/nextcloud/external.rs (99%) rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/nextcloud/mod.rs (100%) rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/nextcloud/routes.rs (90%) rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/sync.rs (98%) rename backend/{windmill-api/src/native_triggers => windmill-native-triggers/src}/workspace_integrations.rs (87%) create mode 100644 backend/windmill-store/Cargo.toml create mode 100644 backend/windmill-store/src/lib.rs create mode 100644 backend/windmill-store/src/oauth_refresh_oss.rs create mode 100644 backend/windmill-store/src/resources.rs create mode 100644 backend/windmill-store/src/secret_backend_ext.rs rename backend/{windmill-api => windmill-store}/src/var_resource_cache.rs (96%) create mode 100644 backend/windmill-store/src/variables.rs create mode 100644 backend/windmill-trigger-email/Cargo.toml rename backend/{windmill-api/src/triggers/email => windmill-trigger-email/src}/handler_oss.rs (90%) create mode 100644 backend/windmill-trigger-email/src/lib.rs create mode 100644 backend/windmill-trigger-gcp/Cargo.toml rename backend/{windmill-api/src/triggers/gcp => windmill-trigger-gcp/src}/handler_oss.rs (91%) create mode 100644 backend/windmill-trigger-gcp/src/lib.rs rename backend/{windmill-api/src/triggers/gcp => windmill-trigger-gcp/src}/listener_oss.rs (61%) create mode 100644 backend/windmill-trigger-http/Cargo.toml create mode 100644 backend/windmill-trigger-http/src/handler.rs rename backend/{windmill-api/src/triggers/http => windmill-trigger-http/src}/http_trigger_auth.rs (100%) create mode 100644 backend/windmill-trigger-http/src/lib.rs create mode 100644 backend/windmill-trigger-kafka/Cargo.toml rename backend/{windmill-api/src/triggers/kafka => windmill-trigger-kafka/src}/handler_oss.rs (90%) create mode 100644 backend/windmill-trigger-kafka/src/lib.rs rename backend/{windmill-api/src/triggers/kafka => windmill-trigger-kafka/src}/listener_oss.rs (61%) create mode 100644 backend/windmill-trigger-mqtt/Cargo.toml rename backend/{windmill-api/src/triggers/mqtt => windmill-trigger-mqtt/src}/handler.rs (92%) create mode 100644 backend/windmill-trigger-mqtt/src/lib.rs rename backend/{windmill-api/src/triggers/mqtt => windmill-trigger-mqtt/src}/listener.rs (97%) create mode 100644 backend/windmill-trigger-nats/Cargo.toml rename backend/{windmill-api/src/triggers/nats => windmill-trigger-nats/src}/handler_oss.rs (88%) create mode 100644 backend/windmill-trigger-nats/src/lib.rs rename backend/{windmill-api/src/triggers/nats => windmill-trigger-nats/src}/listener_oss.rs (61%) create mode 100644 backend/windmill-trigger-postgres/Cargo.toml rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/bool.rs (100%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/converter.rs (99%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/handler.rs (98%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/hex.rs (96%) create mode 100644 backend/windmill-trigger-postgres/src/lib.rs rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/listener.rs (95%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/mapper.rs (98%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/relation.rs (100%) rename backend/{windmill-api/src/triggers/postgres => windmill-trigger-postgres/src}/replication_message.rs (100%) create mode 100644 backend/windmill-trigger-sqs/Cargo.toml rename backend/{windmill-api/src/triggers/sqs => windmill-trigger-sqs/src}/handler_oss.rs (90%) create mode 100644 backend/windmill-trigger-sqs/src/lib.rs rename backend/{windmill-api/src/triggers/sqs => windmill-trigger-sqs/src}/listener_oss.rs (61%) create mode 100644 backend/windmill-trigger-websocket/Cargo.toml rename backend/{windmill-api/src/triggers/websocket => windmill-trigger-websocket/src}/handler.rs (98%) create mode 100644 backend/windmill-trigger-websocket/src/lib.rs rename backend/{windmill-api/src/triggers/websocket => windmill-trigger-websocket/src}/listener.rs (78%) create mode 100644 backend/windmill-trigger/Cargo.toml create mode 100644 backend/windmill-trigger/src/capture.rs rename backend/{windmill-api/src/triggers => windmill-trigger/src}/filter.rs (94%) rename backend/{windmill-api/src/triggers => windmill-trigger/src}/global_handler.rs (97%) create mode 100644 backend/windmill-trigger/src/handler.rs create mode 100644 backend/windmill-trigger/src/lib.rs create mode 100644 backend/windmill-trigger/src/listener.rs rename backend/{windmill-api/src/triggers => windmill-trigger/src}/trigger_helpers.rs (99%) create mode 100644 backend/windmill-trigger/src/types.rs diff --git a/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json b/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json index 5b58cedc72..d3f1c39c7a 100644 --- a/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json +++ b/backend/.sqlx/query-023cdbc77ea9e2c17a1aa92a5b9001f29e58e81b3f782887db6e0a627dd8ad75.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json b/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json new file mode 100644 index 0000000000..c957d28526 --- /dev/null +++ b/backend/.sqlx/query-0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM\n capture\n WHERE\n workspace_id = $1\n AND created_at <= (\n SELECT\n created_at\n FROM\n capture\n WHERE\n workspace_id = $1\n ORDER BY\n created_at DESC\n OFFSET $2\n LIMIT 1\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "0574df3e18f626dd8b3f83fbff8b0ee99cf8483a8fe66fa9311cb96e3f5a0ee2" +} diff --git a/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json b/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json index 8ecc409fc8..97e39aa0e5 100644 --- a/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json +++ b/backend/.sqlx/query-06072cfe26abe58629623a8b38382b33947c5a5c702ce586e6e6ea51430380bf.json @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index d29a18c691..e7ed0aee65 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - true, - true, - true, - true, - true, + false, + false, + false, + false, + false, true, true ] diff --git a/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json b/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json deleted file mode 100644 index a867ace2cb..0000000000 --- a/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nats_trigger \n SET \n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE \n workspace_id = $11 AND path = $12\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd" -} diff --git a/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json b/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json deleted file mode 100644 index ec215e0f29..0000000000 --- a/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE \n workspace_id = $12 AND \n path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "JsonbArray", - { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - }, - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592" -} diff --git a/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json b/backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json similarity index 50% rename from backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json rename to backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json index 9fe2580534..b02b877e92 100644 --- a/backend/.sqlx/query-73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca.json +++ b/backend/.sqlx/query-141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_completed (\n id,\n workspace_id,\n started_at,\n completed_at,\n duration_ms,\n result,\n deleted,\n canceled_by,\n canceled_reason,\n flow_status,\n memory_peak,\n status,\n worker,\n workflow_as_code_status,\n result_columns,\n retries,\n extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job_completed (\n id, workspace_id, started_at, completed_at, duration_ms, result, deleted,\n canceled_by, canceled_reason, flow_status, memory_peak, status, worker,\n workflow_as_code_status, result_columns, retries, extras\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -38,5 +38,5 @@ }, "nullable": [] }, - "hash": "73d062a512230f5b27b9a39dc4484b2c6723c5568940d2fdba85ce4800d8c5ca" + "hash": "141a428bf866f2e3d4c6eff756a5fb8c6476d6b9b00b9a336b4616d3b69a491e" } diff --git a/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json b/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json deleted file mode 100644 index a50cd946c5..0000000000 --- a/backend/.sqlx/query-1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE \n workspace_id = $10 AND path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1a85e45df7fec414e3e167bad3472f455571e0dd3006fb717b36dcab36689cca" -} diff --git a/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json b/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json new file mode 100644 index 0000000000..36707c3421 --- /dev/null +++ b/backend/.sqlx/query-1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Jsonb", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a996a7b9cd38a3b0574048a70ed75281dada61fd1c6da1fa52624ebe0b1c962" +} diff --git a/backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json b/backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json similarity index 72% rename from backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json rename to backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json index 8eb4477540..851c98634b 100644 --- a/backend/.sqlx/query-f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7.json +++ b/backend/.sqlx/query-1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n gcp_resource_path, \n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE \n ", + "query": "\n SELECT\n gcp_resource_path,\n script_path,\n is_flow,\n mode as \"mode: _\",\n workspace_id,\n path,\n edited_by,\n email,\n delivery_config AS \"delivery_config: _\",\n retry as \"retry: _\",\n error_handler_path,\n error_handler_args as \"error_handler_args: _\"\n FROM\n gcp_trigger\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n delivery_type = 'push'::DELIVERY_MODE\n ", "describe": { "columns": [ { @@ -96,5 +96,5 @@ true ] }, - "hash": "f2eebf58001aa792c5fbe9f24ac8290cc15d2865b6774f07a52dd9db7a7209c7" + "hash": "1cf2eb1426e8be89c3649272103bcd029e99b029b7f1b71eda4411d1e24e790d" } diff --git a/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json b/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json deleted file mode 100644 index fb8302c8e6..0000000000 --- a/backend/.sqlx/query-204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - }, - "Jsonb", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "204998a6136091abe6bb8a503b25dc0d34fc38ee129d736b28219b84e6474229" -} diff --git a/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json b/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json deleted file mode 100644 index 1bb99fad8e..0000000000 --- a/backend/.sqlx/query-234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1\n FROM \n http_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "234acda79d470e99e9cbde5c7401d6f7894c25f90e39ec8606f79b8be56d1c17" -} diff --git a/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json b/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json index 4e48ae281c..c51175e921 100644 --- a/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json +++ b/backend/.sqlx/query-256118bb9c87675d6a8db2d13c4f446d88c7bc671d5e90307febfd60bb235031.json @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json b/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json index 128a435c8b..5a4cd89224 100644 --- a/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json +++ b/backend/.sqlx/query-2602402bedcdbc45cdc0d64a76ce075c6ae51404037b0a7d4d33faf6a7d6a6d8.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json b/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json index 9abe8af9af..08c6c1cbc2 100644 --- a/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json +++ b/backend/.sqlx/query-26141a1c3c48184285d94a2e2aca94ff708ce8e0b0b4fb59e5de7c9698c75c25.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json b/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json index 5d7c0e2abc..1199e9441f 100644 --- a/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json +++ b/backend/.sqlx/query-27ada97cb533c8595f1d73987c7823d8e54c96889e06895c57cafae9ca27bf8b.json @@ -15,7 +15,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json b/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json deleted file mode 100644 index 6e58d3d386..0000000000 --- a/backend/.sqlx/query-28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1) \n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "28a5caa6e8a8a95d228908cc83e9109986a0803d50164a259aabcdee81014d07" -} diff --git a/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json b/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json index 8f6c8edf25..afd0f503bf 100644 --- a/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json +++ b/backend/.sqlx/query-2fd22c4ffa2d222bb116260994a748e0639c2f73cbc1d8be66420c70b14c96e1.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json b/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json new file mode 100644 index 0000000000..3f5bd4aefb --- /dev/null +++ b/backend/.sqlx/query-388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE\n workspace_id = $12 AND path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "388ff2abd495cf71e87cf0c4ddc73b6c84867fb966df91b320c54acdd5e61315" +} diff --git a/backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json b/backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json similarity index 52% rename from backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json rename to backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json index 6dd26cdb5a..0df550b57c 100644 --- a/backend/.sqlx/query-3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f.json +++ b/backend/.sqlx/query-3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE workspace_integrations \n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ", + "query": "\n UPDATE workspace_integrations\n SET oauth_data = $1, updated_at = now()\n WHERE workspace_id = $2 AND service_name = $3\n ", "describe": { "columns": [], "parameters": { @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -21,5 +22,5 @@ }, "nullable": [] }, - "hash": "3ebf94f926fff2df4268dd76773eb1996e5f0bb679dca175737ace462432721f" + "hash": "3b3f60623126626b52ca0a4a188655ddf728cd3f21ee308db7393694ccc5c7b3" } diff --git a/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json b/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json new file mode 100644 index 0000000000..f83e856cec --- /dev/null +++ b/backend/.sqlx/query-3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_queue (\n id, workspace_id, started_at, scheduled_for, running, canceled_by,\n canceled_reason, suspend, suspend_until, worker, extras, tag, priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Timestamptz", + "Timestamptz", + "Bool", + "Varchar", + "Text", + "Int4", + "Timestamptz", + "Varchar", + "Jsonb", + "Varchar", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "3d8788d42d45cf27082f031a2ce29b496f1f2caae145c47db296b6ea0089f86c" +} diff --git a/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json b/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json index e6dc6ff08b..fdcb10be1c 100644 --- a/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json +++ b/backend/.sqlx/query-3e5bdc2e071fc2f1e3c7971736272f20bb5a0aa921a614bd02898d3f162660c2.json @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -51,7 +52,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json b/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json deleted file mode 100644 index e74dc52790..0000000000 --- a/backend/.sqlx/query-3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_queue (\n id,\n workspace_id,\n started_at,\n scheduled_for,\n running,\n canceled_by,\n canceled_reason,\n suspend,\n suspend_until,\n worker,\n extras,\n tag,\n priority\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Timestamptz", - "Timestamptz", - "Bool", - "Varchar", - "Text", - "Int4", - "Timestamptz", - "Varchar", - "Jsonb", - "Varchar", - "Int2" - ] - }, - "nullable": [] - }, - "hash": "3e88cef5955beeff980135297c42271eaaddde1f4e3a4ab62628faee2e06e947" -} diff --git a/backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json b/backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json similarity index 59% rename from backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json rename to backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json index fd4e4e7f52..ebf1df39a3 100644 --- a/backend/.sqlx/query-013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e.json +++ b/backend/.sqlx/query-5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n oauth_data as \"oauth_data!: sqlx::types::Json\",\n service_name as \"service_name!: ServiceName\"\n FROM \n workspace_integrations \n WHERE \n workspace_id = $1\n ", + "query": "\n SELECT\n oauth_data as \"oauth_data!: sqlx::types::Json\",\n service_name as \"service_name!: ServiceName\"\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1\n ", "describe": { "columns": [ { @@ -16,7 +16,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -33,5 +34,5 @@ false ] }, - "hash": "013603454ee8335dc841a3637803532cf556bca5ab5a46c78fe2334c361a140e" + "hash": "5368683c19f8d6744d5dbc53e5b2ab0f2348646d79f5306c6868e2c3a8f389ee" } diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json new file mode 100644 index 0000000000..0d12be2448 --- /dev/null +++ b/backend/.sqlx/query-66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147.json @@ -0,0 +1,51 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n owner,\n email\n FROM\n capture_config\n WHERE\n workspace_id = $1\n AND path = $2\n AND is_flow = $3\n AND trigger_kind = $4\n AND last_client_ping > NOW() - INTERVAL '10 seconds'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud" + ] + } + } + } + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "66a0e51cf149ba532463e29dd361a803e1bced2f8e1a12f8933b7598ee85a147" +} diff --git a/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json b/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json new file mode 100644 index 0000000000..659a783821 --- /dev/null +++ b/backend/.sqlx/query-6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "JsonbArray", + "JsonbArray", + "Jsonb", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6c3c38ed5a0e6de0c97954fe4581daa788a7551b4a00dd33f78d36e246b98dd7" +} diff --git a/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json b/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json new file mode 100644 index 0000000000..f42d028128 --- /dev/null +++ b/backend/.sqlx/query-8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n gcp_trigger\n SET\n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n script_path = $9,\n path = $10,\n mode = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE\n workspace_id = $12 AND\n path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "delivery_mode", + "kind": { + "Enum": [ + "push", + "pull" + ] + } + } + }, + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "8642e4fc3efb011a8b7f98d2080a165ec4046c3a7172b5635b818c0fa133411f" +} diff --git a/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json b/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json deleted file mode 100644 index 5979389314..0000000000 --- a/backend/.sqlx/query-8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO job_logs (\n job_id,\n workspace_id,\n logs,\n log_offset,\n log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Int4", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "8c5a1b4ea98856466a31658cfe953492943c961f98036133437ccc1a621ee89c" -} diff --git a/backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json b/backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json similarity index 58% rename from backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json rename to backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json index 7e1e4b10ef..62f8d3100f 100644 --- a/backend/.sqlx/query-1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f.json +++ b/backend/.sqlx/query-9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n oauth_data \n FROM \n workspace_integrations \n WHERE \n workspace_id = $1 AND\n service_name = $2\n ", + "query": "\n SELECT\n oauth_data\n FROM\n workspace_integrations\n WHERE\n workspace_id = $1 AND\n service_name = $2\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -28,5 +29,5 @@ false ] }, - "hash": "1ecb8a116bbfbcae8b9c502d58d98c838bf4de8d1048889b658d7e969d70a50f" + "hash": "9052b7cd438ff029a37bd489190d98d365acec09f2f102b7de71dcc9d356900e" } diff --git a/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json b/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json new file mode 100644 index 0000000000..529824e4b2 --- /dev/null +++ b/backend/.sqlx/query-94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM\n http_trigger\n WHERE\n workspace_id = $1 AND\n path = $2\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "94bb99ee062eb72850721f121a626dd1665b3434f8624710ea5931958f459707" +} diff --git a/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json b/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json deleted file mode 100644 index 088d2317c9..0000000000 --- a/backend/.sqlx/query-95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_runtime (\n id, ping, memory_peak\n ) VALUES (\n $1, $2, $3\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Timestamptz", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "95585df7a45e61f0c31a05667b103041f8a893fe9ed2830aecaa4d245b07d739" -} diff --git a/backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json b/backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json similarity index 70% rename from backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json rename to backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json index 9cdbef80a5..a189bc4da2 100644 --- a/backend/.sqlx/query-92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5.json +++ b/backend/.sqlx/query-95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2 \n AND oauth_data IS NOT NULL\n )\n ", + "query": "\n SELECT EXISTS (\n SELECT 1\n FROM workspace_integrations\n WHERE workspace_id = $1\n AND service_name = $2\n AND oauth_data IS NOT NULL\n )\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -28,5 +29,5 @@ null ] }, - "hash": "92745b0ee2b42cc6f8ba58091b3845784ed840b2f5196b40661a453c1f2a4db5" + "hash": "95c57fb921a2e3725b92cbafac6e3dc360b88429f03dd1e2b1b55cfabe208cb7" } diff --git a/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json b/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json new file mode 100644 index 0000000000..776ff7b5e3 --- /dev/null +++ b/backend/.sqlx/query-9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE nats_trigger\n SET\n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9f41ea5cbe4cffa74e4a283fe8f023c813e349956487f7b6599da452c068e9b9" +} diff --git a/backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json b/backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json similarity index 59% rename from backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json rename to backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json index 33e0d6b0ff..24efa05c77 100644 --- a/backend/.sqlx/query-447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba.json +++ b/backend/.sqlx/query-aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", + "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n )\n VALUES (\n $1,\n $2,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", "describe": { "columns": [], "parameters": { @@ -47,5 +47,5 @@ }, "nullable": [] }, - "hash": "447457d93fa956d2f7364ff15038cc590473da374b5120921086a1ee557ac9ba" + "hash": "aa59a96bf2d7edfa7c550e66c4d52ddc8e84eacc633e361e49a5219d0bec94b9" } diff --git a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json deleted file mode 100644 index 7587b063d9..0000000000 --- a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool", - { - "Custom": { - "name": "trigger_kind", - "kind": { - "Enum": [ - "webhook", - "http", - "websocket", - "kafka", - "email", - "nats", - "postgres", - "sqs", - "mqtt", - "gcp", - "default_email", - "nextcloud" - ] - } - } - } - ] - }, - "nullable": [ - null - ] - }, - "hash": "ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30" -} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 99269c9851..54e94cfb8f 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - true, - false + false, + true ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json b/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json index d16886016b..f1ba61b895 100644 --- a/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json +++ b/backend/.sqlx/query-b4eb72b0274cbdce7490f63c36d0d16ee847294fadc138593a1baa417cbb3652.json @@ -59,9 +59,7 @@ "failure", "command", "approval", - "preprocessor", - "schedule_handler_old", - "dynamic_skip" + "preprocessor" ] } } diff --git a/backend/.sqlx/query-b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432.json b/backend/.sqlx/query-b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432.json deleted file mode 100644 index 61d48bd904..0000000000 --- a/backend/.sqlx/query-b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n variable.account as account_id,\n (now() > account.expires_at) as \"is_expired: bool\"\n FROM variable\n LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n WHERE variable.path = $1 AND variable.workspace_id = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "account_id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "is_expired: bool", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "b4fc94adfe55bb87d2c4b6b45ed2eb5ac25e84f3619a0b403a2d619a0eb51432" -} diff --git a/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json b/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json new file mode 100644 index 0000000000..e30f73a14c --- /dev/null +++ b/backend/.sqlx/query-b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE email_trigger\n SET\n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n mode = $9\n WHERE\n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb", + { + "Custom": { + "name": "trigger_mode", + "kind": { + "Enum": [ + "enabled", + "disabled", + "suspended" + ] + } + } + }, + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b5cda8eb32384f315689001f45676d4bf44cb4397dd0a722e7c9d035b58a09c1" +} diff --git a/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json b/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json deleted file mode 100644 index 1d8fa42e31..0000000000 --- a/backend/.sqlx/query-b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n initial_messages = $6,\n url_runnable_args = $7,\n edited_by = $8,\n email = $9,\n can_return_message = $10,\n can_return_error_result = $11,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "JsonbArray", - "JsonbArray", - "Jsonb", - "Varchar", - "Varchar", - "Bool", - "Bool", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "b5f6870444fc97d8beab3cf61c91e58936138d80a97b423c17338ba069b6a3aa" -} diff --git a/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json b/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json index 96f59aab3f..e6ef13f492 100644 --- a/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json +++ b/backend/.sqlx/query-b615d73ddb43e9d655b86a0cf98f892bf40e629ee11ee4845199481755f2789d.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json b/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json new file mode 100644 index 0000000000..0a314a59e3 --- /dev/null +++ b/backend/.sqlx/query-bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n ((workspaced_local_part IS TRUE AND workspace_id || '-' || local_part = $1)\n OR (workspaced_local_part IS FALSE AND local_part = $1))\n AND ($2::TEXT IS NULL OR path != $2)\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bb94e3105cb1bc5d90af2bc914b579a6e821c432eac38ad877d9ff362d8ab916" +} diff --git a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json similarity index 67% rename from backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json rename to backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 7f581135ed..0688afebd3 100644 --- a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -125,5 +125,5 @@ }, "nullable": [] }, - "hash": "fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f" + "hash": "be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66" } diff --git a/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json b/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json deleted file mode 100644 index ff4479af3f..0000000000 --- a/backend/.sqlx/query-c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job_status (\n id, flow_status, flow_leaf_jobs, workflow_as_code_status\n ) VALUES (\n $1, $2, $3, $4\n )\n ON CONFLICT (id) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Jsonb", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c18dc6715b2f895c61d4cbf2d2a671dd89e10000d3313884a663852337a76b2d" -} diff --git a/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json new file mode 100644 index 0000000000..0098e51ab2 --- /dev/null +++ b/backend/.sqlx/query-c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n capture_config\n SET\n server_id = $1,\n last_server_ping = now(),\n error = 'Connecting...'\n WHERE\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n RETURNING true\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "c3b1152b554812d65eb27f95b1fd434f860922fbc021185beffb9827647feb8e" +} diff --git a/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json b/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json new file mode 100644 index 0000000000..ef146c1cee --- /dev/null +++ b/backend/.sqlx/query-c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE sqs_trigger\n SET\n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE\n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + { + "Custom": { + "name": "aws_auth_resource_type", + "kind": { + "Enum": [ + "oidc", + "credentials" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "c723c3a5066a487b93e2642993f3bf624a1f50d06c7de75157420d97cf144763" +} diff --git a/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json b/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json index 0e338633fe..8a0dacd864 100644 --- a/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json +++ b/backend/.sqlx/query-cc5919b087e3319045fc2242f22384514987b6a4eb807ee375fb8b5cd4e92307.json @@ -11,7 +11,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json similarity index 56% rename from backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json rename to backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json index 5f7957c57f..5091b2fc69 100644 --- a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json +++ b/backend/.sqlx/query-ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = NULL\n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = $4 AND \n server_id IS NULL\n ", + "query": "\n UPDATE\n capture_config\n SET\n last_server_ping = NULL\n WHERE\n workspace_id = $1 AND\n path = $2 AND\n is_flow = $3 AND\n trigger_kind = $4 AND\n server_id IS NULL\n ", "describe": { "columns": [], "parameters": { @@ -33,5 +33,5 @@ }, "nullable": [] }, - "hash": "dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e" + "hash": "ccef7a1bde5cac6c362c5fedb6c13f1f882b695f896f94e5cf91d205633355a1" } diff --git a/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json b/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json index b0cef08da3..b71bf15f85 100644 --- a/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json +++ b/backend/.sqlx/query-d0b493da8ff5b3b78a9a0c6972cd2ca5449dad0fe361bee1a9834989bfeabbae.json @@ -12,7 +12,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json b/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json deleted file mode 100644 index 1060c7af74..0000000000 --- a/backend/.sqlx/query-d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n mode = $11\n WHERE \n workspace_id = $12 AND path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb", - { - "Custom": { - "name": "trigger_mode", - "kind": { - "Enum": [ - "enabled", - "disabled", - "suspended" - ] - } - } - }, - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d33810c502838ad7aeb1f45691a456ec563d346d99256ded52756cdb4a0d7ffd" -} diff --git a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json similarity index 67% rename from backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json rename to backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 94acdfb2d8..11271e94d8 100644 --- a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO v2_job (\n id,\n raw_code,\n raw_lock,\n raw_flow,\n tag,\n workspace_id,\n created_at,\n created_by,\n permissioned_as,\n permissioned_as_email,\n kind,\n runnable_id,\n runnable_path,\n parent_job,\n root_job,\n script_lang,\n script_entrypoint_override,\n flow_step,\n flow_step_id,\n flow_innermost_root_job,\n trigger,\n trigger_kind,\n same_worker,\n visible_to_owner,\n concurrent_limit,\n concurrency_time_window_s,\n cache_ttl,\n timeout,\n priority,\n preprocessed,\n args,\n labels,\n pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", + "query": "\n INSERT INTO v2_job (\n id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by,\n permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path,\n parent_job, root_job, script_lang, script_entrypoint_override, flow_step,\n flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker,\n visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl,\n timeout, priority, preprocessed, args, labels, pre_run_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,\n $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,\n $21, $22, $23, $24, $25, $26, $27, $28, $29, $30,\n $31, $32, $33\n )\n ON CONFLICT (id) DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -125,5 +125,5 @@ }, "nullable": [] }, - "hash": "b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b" + "hash": "d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016" } diff --git a/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json similarity index 58% rename from backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json rename to backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json index 7ba4c70a26..9cef6500d3 100644 --- a/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json +++ b/backend/.sqlx/query-d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5\n ", + "query": "\n UPDATE\n capture_config\n SET\n error = $1,\n server_id = NULL,\n last_server_ping = NULL\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5\n ", "describe": { "columns": [], "parameters": { @@ -34,5 +34,5 @@ }, "nullable": [] }, - "hash": "db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899" + "hash": "d495c94b580fd34d5ae90615ef21a8a9cc35f362197c0766a5787436af141106" } diff --git a/backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json b/backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json similarity index 64% rename from backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json rename to backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json index 1ce9e58cd4..af5ff0153c 100644 --- a/backend/.sqlx/query-679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294.json +++ b/backend/.sqlx/query-d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n route_path_key = $1\n AND workspace_id = $2 \n AND http_method = $3 \n AND ($4::TEXT IS NULL OR path != $4)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n route_path_key = $1\n AND workspace_id = $2\n AND http_method = $3\n AND ($4::TEXT IS NULL OR path != $4)\n )\n ", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "679a9159a5fca976a3de99fe26806faded2cc63e8f16c201e99ab1725dcff294" + "hash": "d5d96f93769d2f51e340422f78bd9e78ca5db2c6677109b04dc477faf48de033" } diff --git a/backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json b/backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json similarity index 69% rename from backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json rename to backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json index 1879fa69ec..f5a1215aa7 100644 --- a/backend/.sqlx/query-80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f.json +++ b/backend/.sqlx/query-e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n mode, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )", + "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path,\n script_path,\n is_flow,\n email,\n mode,\n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )", "describe": { "columns": [], "parameters": { @@ -46,5 +46,5 @@ }, "nullable": [] }, - "hash": "80c8da5b571f8da7cde6da8557898e25fdfdf19ea7a85b5e55922883764df15f" + "hash": "e25aa749cfc9f0bb1649d162e36f2c0ce2187e47d745aa5ba96f9453c722750c" } diff --git a/backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json b/backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json similarity index 53% rename from backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json rename to backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json index 7be240422d..7c994ca763 100644 --- a/backend/.sqlx/query-54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab.json +++ b/backend/.sqlx/query-e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM email_trigger \n WHERE \n workspace_id = $1\n AND local_part = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM email_trigger\n WHERE\n workspace_id = $1\n AND local_part = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", "describe": { "columns": [ { @@ -20,5 +20,5 @@ null ] }, - "hash": "54ee00c98dc50ca3da0e2195a78e6df77fd1e8c969667b267aed3347335c91ab" + "hash": "e2b362d9e36be80ee92cf9897a86b61e8e4e92b80b336d1a861e68f9ef782f8a" } diff --git a/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json b/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json new file mode 100644 index 0000000000..a9d57f3132 --- /dev/null +++ b/backend/.sqlx/query-e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n mqtt_trigger\n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(),\n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE\n workspace_id = $12 AND\n path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "JsonbArray", + { + "Custom": { + "name": "mqtt_client_version", + "kind": { + "Enum": [ + "v3", + "v5" + ] + } + } + }, + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e486a64b76da5de97e404c81dd6e29d333ada2dcfbbddb028f37794b85778ca8" +} diff --git a/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json b/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json new file mode 100644 index 0000000000..d85c08815d --- /dev/null +++ b/backend/.sqlx/query-e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_logs (\n job_id, workspace_id, logs, log_offset, log_file_index\n ) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (job_id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text", + "Int4", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "e4ffb20330f584f64d96423c0ecbfaf7f8740a02d1d0cc55aaa7d2a45acfb881" +} diff --git a/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json b/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json deleted file mode 100644 index f19d888a67..0000000000 --- a/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b" -} diff --git a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json similarity index 58% rename from backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json rename to backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json index b0da0f0e69..1e38e57fb5 100644 --- a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json +++ b/backend/.sqlx/query-e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = now(), error = $1\n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n server_id = $6 AND \n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", + "query": "\n UPDATE\n capture_config\n SET\n last_server_ping = now(), error = $1\n WHERE\n workspace_id = $2 AND\n path = $3 AND\n is_flow = $4 AND\n trigger_kind = $5 AND\n server_id = $6 AND\n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", "describe": { "columns": [ { @@ -43,5 +43,5 @@ null ] }, - "hash": "615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae" + "hash": "e80177f3ffd4c1f52cdb4757483f03f72ef81db302d727e18e63a307ac902022" } diff --git a/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json b/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json deleted file mode 100644 index 1e67891f48..0000000000 --- a/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE sqs_trigger \n SET \n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE \n workspace_id = $10 AND path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "TextArray", - { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - }, - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e" -} diff --git a/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json b/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json index f80b75b5ec..16ccdd10e9 100644 --- a/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json +++ b/backend/.sqlx/query-ecab1af12a7afa685c056b9d0e526275203fc8ecddf83ca6d05c9fb77e46e7ee.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json new file mode 100644 index 0000000000..43fd90abe5 --- /dev/null +++ b/backend/.sqlx/query-ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp", + "default_email", + "nextcloud" + ] + } + } + }, + "Jsonb", + "Jsonb", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ed8facbf29ebb670d05fe8aa34b50d6a6935420fbedc83aa3ad1e9be7465c8dd" +} diff --git a/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json b/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json index 4ce08c9e1c..536175599d 100644 --- a/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json +++ b/backend/.sqlx/query-ee537def1ead8bee48bb9f5c1f57d42e7add6011c34d91761ba23e2c74c4032c.json @@ -21,7 +21,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } @@ -71,7 +72,8 @@ "name": "native_trigger_service", "kind": { "Enum": [ - "nextcloud" + "nextcloud", + "google" ] } } diff --git a/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json b/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json new file mode 100644 index 0000000000..42f48ab33e --- /dev/null +++ b/backend/.sqlx/query-f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger\n SET\n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE\n workspace_id = $9 AND path = $10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f4340e31adddc9a37bab64957b2b1b9b29f978c73f003dfbf81d5c487f666358" +} diff --git a/backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json b/backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json similarity index 58% rename from backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json rename to backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json index 1d2f993f5d..71bb00c25e 100644 --- a/backend/.sqlx/query-0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58.json +++ b/backend/.sqlx/query-fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM http_trigger \n WHERE \n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) \n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2 \n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", + "query": "\n SELECT EXISTS(\n SELECT 1\n FROM http_trigger\n WHERE\n ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1)\n OR (workspaced_route IS FALSE AND route_path_key = $1))\n AND http_method = $2\n AND ($3::TEXT IS NULL OR path != $3)\n )\n ", "describe": { "columns": [ { @@ -33,5 +33,5 @@ null ] }, - "hash": "0d8153986cea6166820f601f80d8e67156408b08360d628300b28221ea995a58" + "hash": "fe464b8b3ade86743d82c5e3fb14f457e07f07e44c7b693d5d755899d4210dee" } diff --git a/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json b/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json new file mode 100644 index 0000000000..faed40d308 --- /dev/null +++ b/backend/.sqlx/query-fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO v2_job_runtime (id, ping, memory_peak)\n VALUES ($1, $2, $3)\n ON CONFLICT (id) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "fe71d74e996511ab7331c52704f2f03f5972394fd927b3da216de66dab9e3e5a" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 766325c988..c5b47bc8c5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15722,7 +15722,6 @@ dependencies = [ "anyhow", "argon2", "astral-tokio-tar", - "async-nats", "async-recursion", "async-stream", "async-trait", @@ -15732,25 +15731,15 @@ dependencies = [ "aws-sdk-bedrock", "aws-sdk-bedrockruntime", "aws-sdk-config", - "aws-sdk-sqs", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", "aws-sigv4", "aws-smithy-types", "axum 0.7.9", - "backon", "base32", "base64 0.22.1", - "byteorder", "bytes", - "candle-core", - "candle-nn", - "candle-transformers", "chrono", "chrono-tz", "const_format", - "constant_time_eq 0.3.1", "cookie 0.17.0", "cron", "dashmap 6.1.0", @@ -15761,10 +15750,7 @@ dependencies = [ "flate2", "futures", "git-version", - "google-cloud-googleapis", - "google-cloud-pubsub", "hex", - "hf-hub", "hmac", "http 1.4.0", "hyper 1.8.1", @@ -15778,50 +15764,36 @@ dependencies = [ "matchit 0.7.3", "mime_guess", "native-tls", - "nkeys", "object_store", "openidconnect", "openssl", - "pg_escape", "pin-project", - "postgres-native-tls 0.5.0", "postgres-native-tls 0.5.1", "prometheus", "quick_cache", "rand 0.9.0", - "rdkafka", - "rdkafka-sys", "regex", "reqwest 0.13.1", "rsa", - "rumqttc", "rust-embed", - "rust_decimal", "rustls 0.23.35", "samael", "serde", "serde_json", "serde_urlencoded", "serde_yml", - "sha1", "sha2 0.10.9", "sql-builder", "sqlx", "strum 0.27.2", "tar", "tempfile", - "thiserror 2.0.18", "time", - "tinyvector", - "tokenizers", "tokio", "tokio-native-tls", - "tokio-postgres 0.7.11", "tokio-postgres 0.7.13", "tokio-stream", - "tokio-tungstenite 0.24.0", "tokio-util", - "tonic", "tower 0.5.3", "tower-cookies", "tower-http", @@ -15831,12 +15803,17 @@ dependencies = [ "url", "urlencoding", "uuid", + "windmill-api-auth", + "windmill-api-embeddings", + "windmill-api-jobs", + "windmill-api-sse", "windmill-audit", "windmill-autoscaling", "windmill-common", "windmill-git-sync", "windmill-indexer", "windmill-mcp", + "windmill-native-triggers", "windmill-oauth", "windmill-parser", "windmill-parser-py", @@ -15844,9 +15821,46 @@ dependencies = [ "windmill-parser-sql", "windmill-parser-ts", "windmill-queue", + "windmill-store", + "windmill-trigger", + "windmill-trigger-email", + "windmill-trigger-gcp", + "windmill-trigger-http", + "windmill-trigger-kafka", + "windmill-trigger-mqtt", + "windmill-trigger-nats", + "windmill-trigger-postgres", + "windmill-trigger-sqs", + "windmill-trigger-websocket", "windmill-worker", ] +[[package]] +name = "windmill-api-auth" +version = "1.628.3" +dependencies = [ + "anyhow", + "axum 0.7.9", + "chrono", + "http 1.4.0", + "itertools 0.14.0", + "jsonwebtoken 8.3.0", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "time", + "tokio", + "tower-cookies", + "tracing", + "uuid", + "windmill-audit", + "windmill-common", + "windmill-mcp", +] + [[package]] name = "windmill-api-client" version = "1.628.3" @@ -15857,6 +15871,66 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "windmill-api-embeddings" +version = "1.628.3" +dependencies = [ + "anyhow", + "axum 0.7.9", + "candle-core", + "candle-nn", + "candle-transformers", + "hf-hub", + "lazy_static", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "tinyvector", + "tokenizers", + "tokio", + "tracing", + "windmill-common", + "windmill-store", +] + +[[package]] +name = "windmill-api-jobs" +version = "1.628.3" +dependencies = [ + "anyhow", + "axum 0.7.9", + "base64 0.22.1", + "chrono", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tower-http", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-api-sse", + "windmill-common", + "windmill-queue", +] + +[[package]] +name = "windmill-api-sse" +version = "1.628.3" +dependencies = [ + "lazy_static", + "serde", + "serde_json", + "sqlx", + "uuid", + "windmill-common", +] + [[package]] name = "windmill-audit" version = "1.628.3" @@ -16049,6 +16123,7 @@ version = "1.628.3" dependencies = [ "anyhow", "async-trait", + "chrono", "futures", "http 1.4.0", "oauth2", @@ -16063,6 +16138,37 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-native-triggers" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "backon", + "base64 0.22.1", + "chrono", + "hmac", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "reqwest 0.13.1", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx", + "strum 0.27.2", + "tokio", + "tracing", + "urlencoding", + "uuid", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-oauth", + "windmill-queue", +] + [[package]] name = "windmill-oauth" version = "1.628.3" @@ -16337,6 +16443,312 @@ dependencies = [ "windmill-parser-sql", ] +[[package]] +name = "windmill-store" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-recursion", + "axum 0.7.9", + "chrono", + "futures", + "http 1.4.0", + "hyper 1.8.1", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-mcp", + "windmill-oauth", +] + +[[package]] +name = "windmill-trigger" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "chrono", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "lazy_static", + "rand 0.9.0", + "reqwest 0.13.1", + "serde", + "serde_json", + "sql-builder", + "sqlx", + "tokio", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-api-jobs", + "windmill-api-sse", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-parser", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + +[[package]] +name = "windmill-trigger-email" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "lazy_static", + "regex", + "serde", + "serde_json", + "sqlx", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-gcp" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "bytes", + "chrono", + "google-cloud-googleapis", + "google-cloud-pubsub", + "http 1.4.0", + "itertools 0.14.0", + "jsonwebtoken 8.3.0", + "lazy_static", + "quick_cache", + "reqwest 0.13.1", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tonic", + "tower-http", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-http" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "constant_time_eq 0.3.1", + "futures", + "hex", + "hmac", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "lazy_static", + "matchit 0.7.3", + "quick_cache", + "regex", + "serde", + "serde_json", + "sha1", + "sha2 0.10.9", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-kafka" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "itertools 0.14.0", + "rdkafka", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-mqtt" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "bytes", + "itertools 0.14.0", + "rumqttc", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-nats" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-nats", + "async-trait", + "axum 0.7.9", + "base64 0.22.1", + "itertools 0.14.0", + "nkeys", + "serde", + "serde_json", + "sqlx", + "tokio", + "tokio-stream", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-postgres" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "byteorder", + "bytes", + "chrono", + "futures", + "itertools 0.14.0", + "lazy_static", + "native-tls", + "pg_escape", + "postgres-native-tls 0.5.0", + "quick_cache", + "rand 0.9.0", + "rust_decimal", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tokio-postgres 0.7.11", + "tokio-stream", + "tracing", + "uuid", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-sqs" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sqs", + "aws-sdk-sts", + "axum 0.7.9", + "backon", + "chrono", + "itertools 0.14.0", + "serde", + "serde_json", + "sqlx", + "thiserror 2.0.18", + "tokio", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-store", + "windmill-trigger", +] + +[[package]] +name = "windmill-trigger-websocket" +version = "1.628.3" +dependencies = [ + "anyhow", + "async-trait", + "axum 0.7.9", + "futures", + "http 1.4.0", + "itertools 0.14.0", + "serde", + "serde_json", + "sqlx", + "tokio", + "tokio-tungstenite 0.24.0", + "tracing", + "windmill-api-auth", + "windmill-common", + "windmill-git-sync", + "windmill-queue", + "windmill-trigger", +] + [[package]] name = "windmill-worker" version = "1.628.3" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 40063f2cda..91ac3add3e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -8,6 +8,22 @@ edition.workspace = true resolver = "2" members = [ "./windmill-api", + "./windmill-api-auth", + "./windmill-api-sse", + "./windmill-api-jobs", + "./windmill-trigger", + "./windmill-trigger-kafka", + "./windmill-trigger-postgres", + "./windmill-trigger-mqtt", + "./windmill-trigger-websocket", + "./windmill-trigger-email", + "./windmill-trigger-nats", + "./windmill-trigger-sqs", + "./windmill-trigger-gcp", + "./windmill-trigger-http", + "./windmill-native-triggers", + "./windmill-api-embeddings", + "./windmill-store", "./windmill-queue", "./windmill-worker", "./windmill-common", @@ -227,6 +243,22 @@ windmill-indexer = {path = "./windmill-indexer"} windmill-mcp = {path = "./windmill-mcp"} windmill-oauth = {path = "./windmill-oauth"} windmill-macros = {path = "./windmill-macros"} +windmill-api-auth = { path = "./windmill-api-auth" } +windmill-api-sse = { path = "./windmill-api-sse" } +windmill-api-jobs = { path = "./windmill-api-jobs" } +windmill-trigger = { path = "./windmill-trigger" } +windmill-trigger-kafka = { path = "./windmill-trigger-kafka" } +windmill-trigger-postgres = { path = "./windmill-trigger-postgres" } +windmill-trigger-mqtt = { path = "./windmill-trigger-mqtt" } +windmill-trigger-websocket = { path = "./windmill-trigger-websocket" } +windmill-trigger-email = { path = "./windmill-trigger-email" } +windmill-trigger-nats = { path = "./windmill-trigger-nats" } +windmill-trigger-sqs = { path = "./windmill-trigger-sqs" } +windmill-trigger-gcp = { path = "./windmill-trigger-gcp" } +windmill-trigger-http = { path = "./windmill-trigger-http" } +windmill-native-triggers = { path = "./windmill-native-triggers" } +windmill-api-embeddings = { path = "./windmill-api-embeddings" } +windmill-store = { path = "./windmill-store" } windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 232ee6fa8c..1b4fbeea70 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7f93a13e96c77292ed4b1e63bc1e8ff1e341d283 +ca79631be6c311c5300863634c539593f805e51e \ No newline at end of file diff --git a/backend/windmill-api-auth/Cargo.toml b/backend/windmill-api-auth/Cargo.toml new file mode 100644 index 0000000000..515a51cab7 --- /dev/null +++ b/backend/windmill-api-auth/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "windmill-api-auth" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_auth" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "dep:jsonwebtoken", "dep:reqwest"] +cloud = ["windmill-common/cloud"] +private = ["windmill-common/private"] +no_auth = [] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true +windmill-mcp = { workspace = true, optional = true } +axum.workspace = true +http.workspace = true +serde.workspace = true +serde_json.workspace = true +uuid.workspace = true +itertools.workspace = true +tower-cookies.workspace = true +tracing.workspace = true +chrono.workspace = true +quick_cache.workspace = true +sqlx.workspace = true +lazy_static.workspace = true +tokio.workspace = true +time.workspace = true +anyhow.workspace = true +jsonwebtoken = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs new file mode 100644 index 0000000000..186f7af257 --- /dev/null +++ b/backend/windmill-api-auth/src/auth.rs @@ -0,0 +1,745 @@ +#[cfg(feature = "enterprise")] +use crate::ee_oss::ExternalJwks; +use axum::{ + async_trait, + extract::{FromRequestParts, OriginalUri, Query}, + Extension, Json, +}; +use chrono::TimeZone; +use http::{request::Parts, StatusCode}; +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use tower_cookies::Cookies; +use tracing::Span; + +use crate::{ApiAuthed, OptJobAuthed}; +use std::{ + str::FromStr, + sync::{ + atomic::{AtomicI64, AtomicU64, Ordering}, + Arc, + }, +}; +#[cfg(feature = "enterprise")] +use tokio::sync::RwLock; +use windmill_common::DB; + +use windmill_common::{ + auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, + error::{Error, JsonResult}, + jwt, + users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, +}; + +lazy_static::lazy_static! { + // Global auth cache accessible from main.rs for direct invalidation + pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); + +} +// Global function to invalidate a specific token from cache +pub fn invalidate_token_from_cache(token: &str) { + // Remove all cache entries for this token (across all workspaces) + AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); + tracing::info!( + "Invalidated token from auth cache: {}...", + &token[..token.len().min(8)] + ); +} + +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: chrono::DateTime, + pub job_id: Option, +} + +pub struct AuthCache { + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] + ext_jwks: Option>>, +} + +impl AuthCache { + pub fn new( + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] ext_jwks: Option>>, + ) -> Self { + AuthCache { + db, + superadmin_secret, + #[cfg(feature = "enterprise")] + ext_jwks, + } + } + + pub async fn invalidate(&self, w_id: &str, token: String) { + AUTH_CACHE.remove(&(w_id.to_string(), token)); + } + + pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { + Some(self.get_opt_job_authed(w_id, token).await?.authed) + } + + pub async fn get_opt_job_authed( + &self, + w_id: Option, + token: &str, + ) -> Option { + let key = ( + w_id.as_ref().unwrap_or(&"".to_string()).to_string(), + token.to_string(), + ); + let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); + match s { + Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => { + Some(OptJobAuthed { authed, job_id }) + } + #[cfg(feature = "enterprise")] + _ if token.starts_with("jwt_ext_") => { + let authed_and_exp = match crate::ee_oss::jwt_ext_auth( + w_id.as_ref(), + token.trim_start_matches("jwt_ext_"), + self.ext_jwks.clone(), + &self.db, + ) + .await + { + Ok(r) => Some(r), + Err(e) => { + tracing::error!("JWT_EXT auth error: {:?}", e); + None + } + }; + + if let Some((authed, exp, job_id)) = authed_and_exp.clone() { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } else { + None + } + } + _ if token.starts_with("jwt_") => { + let jwt_token = token.trim_start_matches("jwt_"); + + let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; + + match jwt_result { + Ok(claims) => { + if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { + tracing::error!("JWT auth error: workspace_id mismatch"); + return None; + } + let username_override = username_override_from_label(claims.label); + + let authed = ApiAuthed { + email: claims.email, + username: claims.username, + is_admin: claims.is_admin, + is_operator: claims.is_operator, + groups: claims.groups, + folders: claims.folders, + scopes: None, + username_override, + token_prefix: claims.audit_span, + }; + let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc + .timestamp_nanos(claims.exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } + Err(err) => { + tracing::error!("JWT auth error: {:?}", err); + None + } + } + } + _ => { + let user_o = sqlx::query!( + "UPDATE token SET last_used_at = now() WHERE + token = $1 + AND (expiration > NOW() OR expiration IS NULL) + AND (workspace_id IS NULL OR workspace_id = $2) + RETURNING owner, email, super_admin, scopes, label", + token, + w_id.as_ref(), + ) + .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) + .fetch_optional(&self.db) + .await + .ok() + .flatten(); + + if let Some(user) = user_o { + let authed_o = { + match user { + (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + let username_override = username_override_from_label(label); + if let Some((prefix, name)) = owner.split_once('/') { + if prefix == "u" { + let (is_admin, is_operator) = if super_admin { + (true, false) + } else { + let r = sqlx::query!( + "SELECT is_admin, operator FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id.as_ref().unwrap() + ) + .fetch_one(&self.db) + .await + .ok(); + if let Some(r) = r { + (r.is_admin, r.operator) + } else { + (false, true) + } + }; + + let w_id = &w_id.unwrap(); + let groups = + get_groups_for_user(w_id, &name, &email, &self.db) + .await + .ok() + .unwrap_or_default(); + + let folders = + get_folders_for_user(w_id, &name, &groups, &self.db) + .await + .ok() + .unwrap_or_default(); + + Some(ApiAuthed { + email: email, + username: name.to_string(), + is_admin, + is_operator, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } else { + let groups = vec![name.to_string()]; + let folders = get_folders_for_user( + &w_id.unwrap(), + "", + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email: email, + username: format!("group-{name}"), + is_admin: false, + groups, + is_operator: false, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + } else { + let groups = vec![]; + let folders = vec![]; + Some(ApiAuthed { + email: email, + username: owner, + is_admin: super_admin, + is_operator: true, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }) + } + } + (_, Some(email), super_admin, scopes, label) => { + let username_override = username_override_from_label(label); + if w_id.is_some() { + let row_o = sqlx::query!( + "SELECT username, is_admin, operator FROM usr WHERE + email = $1 AND workspace_id = $2 AND disabled = false", + &email, + w_id.as_ref().unwrap() + ) + .map(|x| (x.username, x.is_admin, x.operator)) + .fetch_optional(&self.db) + .await + .unwrap_or(Some(("error".to_string(), false, false))); + + match row_o { + Some((username, is_admin, is_operator)) => { + let groups = get_groups_for_user( + &w_id.as_ref().unwrap(), + &username, + &email, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user( + &w_id.unwrap(), + &username, + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email, + username, + is_admin: is_admin || super_admin, + is_operator, + groups, + folders, + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + None if super_admin => Some(ApiAuthed { + email: email.clone(), + username: email, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }), + None => None, + } + } else { + Some(ApiAuthed { + email: email.to_string(), + username: email, + is_admin: super_admin, + is_operator: true, + groups: Vec::new(), + folders: Vec::new(), + scopes, + username_override, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }) + } + } + _ => None, + } + }; + if let Some(authed) = authed_o.as_ref() { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc::now() + + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + } + authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) + } else if self + .superadmin_secret + .as_ref() + .map(|x| x == token) + .unwrap_or(false) + { + let authed = ApiAuthed { + email: SUPERADMIN_SECRET_EMAIL.to_string(), + username: "superadmin_secret".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }; + Some(OptJobAuthed { authed, job_id: None }) + } else { + None + } + } + } + } +} + +pub(crate) async fn extract_token(parts: &mut Parts, state: &S) -> Option { + let auth_header = parts + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + + let from_cookie = match auth_header { + Some(x) => Some(x.to_owned()), + None => Extension::::from_request_parts(parts, state) + .await + .ok() + .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + }; + + #[derive(Deserialize)] + struct Token { + token: Option, + } + match from_cookie { + Some(token) => Some(token), + None => Query::::from_request_parts(parts, state) + .await + .ok() + .and_then(|token| token.token.clone()), + } +} + +#[derive(Clone, Debug)] +pub struct Tokened { + pub token: String, +} + +#[derive(Clone, Debug)] +pub struct OptTokened { + #[allow(dead_code)] + pub token: Option, +} + +struct BruteForceCounter { + counter: AtomicU64, + last_reset: AtomicI64, +} + +lazy_static::lazy_static! { + static ref BRUTE_FORCE_COUNTER: BruteForceCounter = + BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; +} + +impl BruteForceCounter { + async fn increment(&self) { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { + tracing::error!( + "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if now - self.last_reset.load(Ordering::Relaxed) > 60 { + self.counter.store(0, Ordering::Relaxed); + self.last_reset.store(now, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl FromRequestParts for Tokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(Tokened { token: "".to_string() }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(tokened.clone()) + } else { + let token_o = extract_token(parts, state).await; + if let Some(token) = token_o { + let tokened = Self { token }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) + } else { + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } + } +} + +#[async_trait] +impl FromRequestParts for OptTokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(OptTokened { token: None }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(OptTokened { token: Some(tokened.token.clone()) }) + } else { + let token_o = extract_token(parts, state).await; + Ok(OptTokened { token: token_o }) + } + } +} + +pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { + if let Some(scopes) = scopes { + for scope in scopes.iter_mut() { + if scope.starts_with("run:") { + let (_, part_scope) = scope.split_once(":").unwrap(); + + if let Some((kind, path)) = part_scope.split_once("/") { + //appending a 's' as runnable kind is singular while new scope format expect it to be plural + *scope = format!("jobs:run:{}s:{}", kind, path); + } + } else if scope.starts_with("jobs:") { + // Map old jobs scopes to new format + let new_scope = match scope.as_str() { + "jobs:listjobs" => "jobs:read", + "jobs:runscript" => "jobs:run:scripts", + "jobs:runflow" => "jobs:run:flows", + "jobs:resumeflow" => "jobs:run:flows", + "jobs:deletejob" => "jobs:write", + _ => continue, + }; + + *scope = new_scope.to_string(); + } + } + } +} + +fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[1] == "api" + && path_vec[2] == "mcp" + && path_vec[3] == "w" + { + Some(path_vec[4].to_owned()) + } else { + if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { + Some(path_vec[4].to_owned()) + } else { + None + } + }; + + workspace_id +} + +/// Resolves OptJobAuthed from request parts. +/// Takes ownership of Parts and returns them back. +#[allow(unreachable_code, unused_mut)] +pub async fn resolve_opt_job_authed( + mut parts: Parts, +) -> std::result::Result<(OptJobAuthed, Parts), (Error, Parts)> { + if parts.method == http::Method::OPTIONS { + return Ok((OptJobAuthed::default(), parts)); + }; + + #[cfg(feature = "no_auth")] + { + let authed = ApiAuthed { + email: "admin@windmill.dev".to_string(), + username: "admin".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }; + return Ok((OptJobAuthed { authed, job_id: None }, parts)); + } + + let already_authed = parts.extensions.get::().cloned(); + + if let Some(authed) = already_authed { + return Ok((authed, parts)); + } + + let already_tokened = parts.extensions.get::().cloned(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(&mut parts, &()).await + }; + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(&mut parts, &()).await + { + let original_uri = OriginalUri::from_request_parts(&mut parts, &()) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + let workspace_id = maybe_get_workspace_id_from_path(&path_vec); + + if let Some(mut opt_job_authed) = + cache.get_opt_job_authed(workspace_id.clone(), &token).await + { + let authed = &mut opt_job_authed.authed; + if authed.scopes.is_some() { + transform_old_scope_to_new_scope(authed.scopes.as_mut()); + + let path = original_uri.path(); + let method = parts.method.as_str(); + + if let Err(err) = crate::scopes::check_scopes_for_route( + authed.scopes.as_deref(), + path, + method, + ) { + BRUTE_FORCE_COUNTER.increment().await; + return Err((err, parts)); + } + } + parts.extensions.insert(authed.clone()); + + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok((opt_job_authed, parts)); + } + } + } + BRUTE_FORCE_COUNTER.increment().await; + Err((Error::NotAuthorized("Unauthorized".to_string()), parts)) +} + +fn username_override_from_label(label: Option) -> Option { + match label { + Some(label) + if label.starts_with("webhook-") + || label.starts_with("http-") + || label.starts_with("email-") + || label.starts_with("ws-") => + { + Some(label) + } + Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { + Some(format!("label-{label}")) + } + _ => None, + } +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedTokenWithEmail { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, + pub email: Option, +} + +pub async fn list_tokens_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult> { + let tokens = if is_flow { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:flows:' || $2]::text[] + OR scopes @> ARRAY['run:flow/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + } else { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] + OR scopes @> ARRAY['run:script/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + }; + + Ok(Json(tokens)) +} diff --git a/backend/windmill-api-auth/src/ee_oss.rs b/backend/windmill-api-auth/src/ee_oss.rs new file mode 100644 index 0000000000..33b1e9d41a --- /dev/null +++ b/backend/windmill-api-auth/src/ee_oss.rs @@ -0,0 +1,36 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::ee::*; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +use {std::sync::Arc, tokio::sync::RwLock}; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn jwt_ext_auth( + _w_id: Option<&String>, + _token: &str, + _external_jwks: Option>>, + _db: &windmill_common::DB, +) -> anyhow::Result<(crate::ApiAuthed, usize, Option)> { + // Implementation is not open source + Err(anyhow::anyhow!("External JWT auth is not open source")) +} + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub struct ExternalJwks; + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +impl ExternalJwks { + pub async fn load() -> Option>> { + // Implementation is not open source + None + } +} diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs new file mode 100644 index 0000000000..01ab5b7ffd --- /dev/null +++ b/backend/windmill-api-auth/src/lib.rs @@ -0,0 +1,712 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod auth; +#[cfg(feature = "private")] +pub mod ee; +pub mod ee_oss; +pub mod scopes; + +use axum::async_trait; +use axum::extract::FromRequestParts; +use http::request::Parts; + +use windmill_audit::audit_oss::AuditAuthorable; +use windmill_common::{ + auth::{fetch_authed_from_permissioned_as, is_devops_email, is_super_admin_email}, + db::{Authable, Authed, AuthedRef}, + error::{self, Error, Result}, + users::username_to_permissioned_as, + DB, +}; + +use scopes::ScopeDefinition; + +// Re-export key auth types and functions +pub use auth::{ + invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened, + TruncatedTokenWithEmail, AUTH_CACHE, +}; + +// ------------ ApiAuthed & OptJobAuthed types ------------ + +#[derive(Default, Clone, Debug)] +pub struct OptJobAuthed { + pub job_id: Option, + pub authed: ApiAuthed, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] +pub struct ApiAuthed { + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_operator: bool, + pub groups: Vec, + // (folder name, can write, is owner) + pub folders: Vec<(String, bool, bool)>, + pub scopes: Option>, + pub username_override: Option, + pub token_prefix: Option, +} + +impl ApiAuthed { + pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { + AuthedRef { + email: &self.email, + username: &self.username, + is_admin: &self.is_admin, + is_operator: &self.is_operator, + groups: &self.groups, + folders: &self.folders, + scopes: &self.scopes, + token_prefix: &self.token_prefix, + } + } + + pub fn display_username(&self) -> &str { + self.username_override.as_ref().unwrap_or(&self.username) + } +} + +impl From for Authed { + fn from(value: ApiAuthed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + token_prefix: value.token_prefix, + } + } +} + +impl From for ApiAuthed { + fn from(value: Authed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + username_override: None, + token_prefix: value.token_prefix, + } + } +} + +impl AuditAuthorable for ApiAuthed { + fn username(&self) -> &str { + self.username.as_str() + } + fn email(&self) -> &str { + self.email.as_str() + } + fn username_override(&self) -> Option<&str> { + self.username_override.as_deref() + } + fn token_prefix(&self) -> Option<&str> { + self.token_prefix.as_deref() + } +} + +impl Authable for ApiAuthed { + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[std::string::String]> { + self.scopes.as_ref().map(|x| x.as_slice()) + } + + fn email(&self) -> &str { + &self.email + } + + fn username(&self) -> &str { + &self.username + } +} + +// ------------ McpAuth impl (feature-gated) ------------ + +#[cfg(feature = "mcp")] +impl windmill_mcp::server::McpAuth for ApiAuthed { + fn username(&self) -> &str { + &self.username + } + + fn email(&self) -> &str { + &self.email + } + + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[String]> { + self.scopes.as_deref() + } +} + +// ------------ Utility functions ------------ + +pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { + let is_admin = is_super_admin_email(db, email).await?; + + if !is_admin { + Err(Error::NotAuthorized( + "This endpoint requires the caller to be a super admin".to_owned(), + )) + } else { + Ok(()) + } +} + +pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if let Some(scopes) = authed.scopes.as_ref() { + let mut is_scoped_token = false; + let required_scope = ScopeDefinition::from_scope_string(&required())?; + for scope in scopes { + if !scope.starts_with("if_jobs:filter_tags:") { + if !is_scoped_token { + is_scoped_token = true; + } + + match ScopeDefinition::from_scope_string(scope) { + Ok(scope) if scope.includes(&required_scope) => return Ok(()), + _ => {} + } + } + } + + if is_scoped_token { + return Err(Error::NotAuthorized(format!( + "Required scope: {}", + required_scope.as_string() + ))); + } + } + Ok(()) +} + +pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { + let is_devops = is_devops_email(db, email).await?; + + if is_devops { + Ok(()) + } else { + Err(Error::NotAuthorized( + "This endpoint requires the caller to have the `devops` role".to_string(), + )) + } +} + +// ------------ Folder ownership checks ------------ + +pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool { + if *is_admin { + true + } else { + folders.into_iter().any(|x| x.0 == name && x.2) + } +} + +pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> { + if is_owner(authed, name) { + Ok(()) + } else { + Err(Error::NotAuthorized(format!( + "You are not owner of the folder {}", + name + ))) + } +} + +pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + let splitted = path.split("/").collect::>(); + if splitted[0] == "u" { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "only the owner {} is authorized to perform this operation", + splitted[1] + ))) + } + } else if splitted[0] == "f" { + require_is_owner(authed, splitted[1]) + } else { + Err(Error::BadRequest(format!( + "Not recognized path kind: {}", + path + ))) + } + } else { + Err(Error::BadRequest( + "Cannot be owner of an empty path".to_string(), + )) + } +} + +// ------------ Scope tag helpers ------------ + +pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { + authed.scopes.as_ref()?.iter().find_map(|s| { + if s.starts_with("if_jobs:filter_tags:") { + Some( + s.trim_start_matches("if_jobs:filter_tags:") + .split(",") + .collect::>(), + ) + } else { + None + } + }) +} + +// ------------ Maybe refresh folders ------------ + +pub async fn maybe_refresh_folders( + path: &str, + w_id: &str, + authed: ApiAuthed, + db: &DB, +) -> ApiAuthed { + use windmill_common::auth::{get_folders_for_user, get_groups_for_user}; + + if authed.is_admin { + return authed; + } + let splitted = path.split('/').collect::>(); + if splitted.len() >= 2 + && splitted[0] == "f" + && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) + { + let name = &authed.username; + let groups = get_groups_for_user(w_id, name, &authed.email, db) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user(w_id, name, &groups, db) + .await + .ok() + .unwrap_or_default(); + ApiAuthed { folders, ..authed } + } else { + authed + } +} + +// ------------ FromRequestParts impls (direct call to auth module) ------------ + +#[async_trait] +impl FromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + _state: &S, + ) -> std::result::Result { + let opt_job_authed = OptJobAuthed::from_request_parts(parts, _state).await?; + Ok(opt_job_authed.authed) + } +} + +#[async_trait] +impl FromRequestParts for OptJobAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + _state: &S, + ) -> std::result::Result { + // Swap out parts so we can pass ownership to resolve_opt_job_authed + let owned_parts = std::mem::replace(parts, empty_parts()); + match auth::resolve_opt_job_authed(owned_parts).await { + Ok((result, returned_parts)) => { + *parts = returned_parts; + Ok(result) + } + Err((err, returned_parts)) => { + *parts = returned_parts; + Err(err) + } + } + } +} + +fn empty_parts() -> Parts { + let (parts, _body) = http::Request::new(()).into_parts(); + parts +} + +// ------------ OptAuthed (optional auth extractor) ------------ + +#[derive(Clone, Debug)] +pub struct OptAuthed(pub Option); + +#[async_trait] +impl FromRequestParts for OptAuthed +where + S: Send + Sync, +{ + type Rejection = (http::StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + ApiAuthed::from_request_parts(parts, state) + .await + .map(|authed| Self(Some(authed))) + .or_else(|_| Ok(Self(None))) + } +} + +// ------------ fetch_api_authed helpers ------------ + +lazy_static::lazy_static! { + static ref API_AUTHED_CACHE: quick_cache::sync::Cache<(String,String,String), ExpiringAuthCache> = quick_cache::sync::Cache::new(300); +} + +#[allow(unused)] +pub async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await +} + +#[allow(unused)] +pub async fn fetch_api_authed_from_permissioned_as( + permissioned_as: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> error::Result { + let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); + + let mut api_authed = match API_AUTHED_CACHE.get(&key) { + Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { + tracing::debug!("API authed cache hit for user {}", email); + expiring_authed.authed + } + _ => { + tracing::debug!("API authed cache miss for user {}", email); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + + let api_authed = ApiAuthed { + username: authed.username, + email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: None, + token_prefix: authed.token_prefix, + }; + + API_AUTHED_CACHE.insert( + key, + ExpiringAuthCache { + authed: api_authed.clone(), + expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + + api_authed + } + }; + + api_authed.username_override = username_override; + Ok(api_authed) +} + +// ------------ Token creation ------------ + +#[derive(serde::Deserialize)] +pub struct NewToken { + pub label: Option, + pub expiration: Option>, + pub impersonate_email: Option, + pub scopes: Option>, + pub workspace_id: Option, +} + +impl NewToken { + pub fn new( + label: Option, + expiration: Option>, + impersonate_email: Option, + scopes: Option>, + workspace_id: Option, + ) -> Self { + Self { label, expiration, impersonate_email, scopes, workspace_id } + } +} + +pub async fn create_token_internal( + tx: &mut sqlx::PgConnection, + db: &DB, + authed: &ApiAuthed, + token_config: NewToken, +) -> Result { + use tracing::Instrument; + use windmill_audit::{audit_oss::audit_log, ActionKind}; + use windmill_common::{utils::rd_string, worker::CLOUD_HOSTED}; + + let token = rd_string(32); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if *CLOUD_HOSTED { + let nb_tokens = + sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email) + .fetch_one(db) + .await?; + if nb_tokens.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + token, + authed.email, + token_config.label, + token_config.expiration, + is_super_admin, + token_config.scopes.as_ref().map(|x| x.as_slice()), + token_config.workspace_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + authed, + "users.token.create", + ActionKind::Create, + &"global", + Some(&token[0..10]), + None, + ) + .instrument(tracing::info_span!("token", email = &authed.email)) + .await?; + + Ok(token) +} + +// ------------ Permission helpers ------------ + +pub fn get_perm_in_extra_perms_for_authed( + v: serde_json::Value, + authed: &ApiAuthed, +) -> Option { + match v { + serde_json::Value::Object(obj) => { + let mut keys = vec![format!("u/{}", authed.username)]; + for g in authed.groups.iter() { + keys.push(format!("g/{}", g)); + } + let mut res = None; + for k in keys { + if let Some(v) = obj.get(&k) { + if let Some(v) = v.as_bool() { + if v { + return Some(true); + } + res = Some(v); + } + } + } + res + } + _ => None, + } +} + +pub async fn require_is_writer( + authed: &ApiAuthed, + path: &str, + w_id: &str, + db: DB, + query: &str, + kind: &str, +) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + if require_owner_of_path(authed, path).is_ok() { + return Ok(()); + } + if path.starts_with("f/") && path.split('/').count() >= 2 { + let folder = path.split('/').nth(1).unwrap(); + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let is_folder_writer = + get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); + if is_folder_writer { + return Ok(()); + } + } + } + let extra_perms = sqlx::query_scalar(query) + .bind(path) + .bind(w_id) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let perm = get_perm_in_extra_perms_for_authed(perms, authed); + match perm { + Some(true) => Ok(()), + Some(false) => Err(Error::BadRequest(format!( + "User {} is not a writer of {kind} path {path}", + authed.username + ))), + None => Err(Error::BadRequest(format!( + "User {} has neither read or write permission on {kind} {path}", + authed.username + ))), + } + } else { + Err(Error::BadRequest(format!( + "{path} does not exist yet and user {} is not an owner of the parent folder", + authed.username + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be writer of an empty path" + ))) + } +} + +// ------------ Preview access check ------------ + +pub fn require_path_read_access_for_preview( + authed: &ApiAuthed, + path: &Option, +) -> Result<()> { + let Some(path) = path else { + return Ok(()); + }; + + if authed.is_admin { + return Ok(()); + } + + if path.is_empty() { + return Ok(()); + } + + let splitted: Vec<&str> = path.split('/').collect(); + if splitted.len() < 2 { + return Err(Error::BadRequest(format!( + "Invalid path format for preview job: {}", + path + ))); + } + + match splitted[0] { + "u" => { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "You can only run preview jobs in your own namespace (u/{}) or in folders you have read access to", + authed.username + ))) + } + } + "f" => { + let folder = splitted[1]; + if authed.folders.iter().any(|(f, _, _)| f == folder) { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "You do not have read access to folder '{}'. Preview jobs require at least read access to the target folder.", + folder + ))) + } + } + "hub" => Ok(()), + _ => Err(Error::BadRequest(format!( + "Invalid path format for preview job: {}. Path must start with 'u/' or 'f/'", + path + ))), + } +} diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs similarity index 97% rename from backend/windmill-api/src/scopes.rs rename to backend/windmill-api-auth/src/scopes.rs index 1c1bdd4f26..26a2b3c74c 100644 --- a/backend/windmill-api/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -230,6 +230,10 @@ fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bo || matches_wildcard(accepted_resource, scope_resource) } +// ───────────────────────────────────────────────────────────────── +// Route-level scope checking +// ───────────────────────────────────────────────────────────────── + /// Available scope domains (top-level API categories) #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum ScopeDomain { @@ -883,8 +887,6 @@ mod tests { #[test] fn test_user_example_case() { - // User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"] - // Should grant access because scope "u/dieri/*" falls under required pattern "u/*" let user_scope = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()])); let required_mixed = ScopeDefinition::new( @@ -893,9 +895,8 @@ mod tests { None, Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]), ); - assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/* + assert!(user_scope.includes(&required_mixed)); - // Another example: scope covers one but not both paths let scope_specific = ScopeDefinition::new( "scripts", "read", @@ -908,9 +909,8 @@ mod tests { None, Some(vec!["folder/file1".to_string(), "other/file2".to_string()]), ); - assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly + assert!(scope_specific.includes(&required_multi)); - // Test bidirectional pattern matching more explicitly let scope_broad = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); let required_specific = ScopeDefinition::new( @@ -919,7 +919,7 @@ mod tests { None, Some(vec!["u/dieri/script.py".to_string()]), ); - assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py + assert!(scope_broad.includes(&required_specific)); let scope_specific_path = ScopeDefinition::new( "scripts", @@ -929,6 +929,6 @@ mod tests { ); let required_broad = ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); - assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/* + assert!(scope_specific_path.includes(&required_broad)); } } diff --git a/backend/windmill-api-embeddings/Cargo.toml b/backend/windmill-api-embeddings/Cargo.toml new file mode 100644 index 0000000000..ff7c3435aa --- /dev/null +++ b/backend/windmill-api-embeddings/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "windmill-api-embeddings" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_embeddings" +path = "src/lib.rs" + +[features] +default = [] +embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-store.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +reqwest.workspace = true +tokio.workspace = true +tracing.workspace = true +lazy_static.workspace = true +anyhow.workspace = true +tinyvector = { workspace = true, optional = true } +hf-hub = { workspace = true, optional = true } +tokenizers = { workspace = true, optional = true } +candle-core = { workspace = true, optional = true } +candle-transformers = { workspace = true, optional = true } +candle-nn = { workspace = true, optional = true } diff --git a/backend/windmill-api-embeddings/src/lib.rs b/backend/windmill-api-embeddings/src/lib.rs new file mode 100644 index 0000000000..1225f789c3 --- /dev/null +++ b/backend/windmill-api-embeddings/src/lib.rs @@ -0,0 +1,637 @@ +#[cfg(feature = "embedding")] +use anyhow::{anyhow, Error, Result}; +#[cfg(feature = "embedding")] +use std::{collections::HashMap, path::PathBuf, sync::Arc}; +#[cfg(feature = "embedding")] +use windmill_common::DEFAULT_HUB_BASE_URL; +#[cfg(feature = "embedding")] +use windmill_common::HUB_BASE_URL; + +use axum::Router; + +#[cfg(feature = "embedding")] +use axum::{ + extract::{Path, Query}, + Json, +}; + +#[cfg(feature = "embedding")] +use axum::routing::get; +#[cfg(feature = "embedding")] +use candle_core::{Device, Tensor}; +#[cfg(feature = "embedding")] +use candle_nn::VarBuilder; +#[cfg(feature = "embedding")] +use candle_transformers::models::bert::{BertModel, Config, DTYPE}; +#[cfg(feature = "embedding")] +use hf_hub::api::tokio::Api; +#[cfg(feature = "embedding")] +use serde::Deserialize; +#[cfg(feature = "embedding")] +use serde::Serialize; +#[cfg(feature = "embedding")] +use sqlx::{Pool, Postgres}; +#[cfg(feature = "embedding")] +use tinyvector::{ + db::{Db, Embedding}, + similarity::Distance, +}; +#[cfg(feature = "embedding")] +use tokenizers::Tokenizer; +#[cfg(feature = "embedding")] +use tokio::sync::RwLock; +#[cfg(feature = "embedding")] +use windmill_common::utils::http_get_from_hub; + +#[cfg(feature = "embedding")] +use windmill_common::error::JsonResult; + +#[cfg(feature = "embedding")] +use windmill_store::resources::ResourceType; + +#[cfg(feature = "embedding")] +lazy_static::lazy_static! { + pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); + pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); + // Match windmill-api's HTTP_CLIENT config: 30s timeout, ACCEPT_INVALID_CERTS support + static ref HTTP_CLIENT: reqwest::Client = windmill_common::utils::configure_client(reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .connect_timeout(std::time::Duration::from_secs(10)) + .timeout(std::time::Duration::from_secs(30)) + .danger_accept_invalid_certs(std::env::var("ACCEPT_INVALID_CERTS").is_ok())) + .build().unwrap(); +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize)] +struct HubScriptsQuery { + text: String, + limit: Option, + kind: Option, + app: Option, +} + +#[cfg(feature = "embedding")] +#[derive(Serialize)] +pub struct HubScriptResult { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + score: f32, +} + +#[cfg(feature = "embedding")] +async fn query_hub_scripts( + Query(query): Query, +) -> JsonResult> { + let embeddings_db = EMBEDDINGS_DB.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_hub_scripts(&query.text, query.limit, query.kind, query.app) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::internal_err( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize)] +struct ResourceTypesQuery { + text: String, + limit: Option, +} + +#[cfg(feature = "embedding")] +#[derive(Serialize)] +pub struct ResourceTypeResult { + name: String, + score: f32, + schema: Option, +} +#[cfg(feature = "embedding")] +async fn query_resource_types( + Query(query): Query, + Path(w_id): Path, +) -> JsonResult> { + let embeddings_db = EMBEDDINGS_DB.read().await; + + if let Some(embeddings_db) = embeddings_db.as_ref() { + let results = embeddings_db + .query_resource_types(w_id, &query.text, query.limit) + .await?; + + Ok(Json(results)) + } else { + Err(windmill_common::error::Error::internal_err( + "Embeddings db not initialized".to_string(), + )) + } +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize, Debug, Clone)] +struct HubScript { + ask_id: i64, + id: i64, + version_id: i64, + summary: String, + app: String, + kind: String, + embedding: Vec, +} + +#[cfg(feature = "embedding")] +#[derive(Deserialize, Debug)] +struct HubResourceType { + name: String, + embedding: Vec, +} + +#[cfg(feature = "embedding")] +pub struct ModelInstance { + model: BertModel, + tokenizer: Tokenizer, +} + +#[cfg(feature = "embedding")] +impl ModelInstance { + pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> { + let api = Api::new()?; + let repo_api = api.model("thenlper/gte-small".to_string()); + + let (config_filename, tokenizer_filename, weights_filename) = + ( + repo_api + .get("config.json") + .await + .map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?, + repo_api.get("tokenizer.json").await.map_err(|e| { + anyhow!("Failed to get tokenizer.json from hugging face: {}", e) + })?, + repo_api.get("model.safetensors").await.map_err(|e| { + anyhow!("Failed to get model.safetensors from hugging face: {}", e) + })?, + ); + + Ok((config_filename, tokenizer_filename, weights_filename)) + } + + pub async fn new() -> Result { + tracing::info!("Loading embedding model..."); + let device = Device::Cpu; + let (config_filename, tokenizer_filename, weights_filename) = + Self::load_model_files().await?; + let config = std::fs::read_to_string(config_filename)?; + let config: Config = serde_json::from_str(&config)?; + let tokenizer = Tokenizer::from( + Tokenizer::from_file(tokenizer_filename) + .map_err(Error::msg)? + .with_padding(None) + .to_owned(), + ); + + let vb = + unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? }; + let model = BertModel::load(vb, &config)?; + tracing::info!("Loaded embedding model"); + Ok(Self { model, tokenizer }) + } + + pub async fn create_embedding(self: Arc, sentence: &str) -> Result> { + let sentence = sentence.to_owned(); + tokio::task::spawn_blocking(move || { + let tokens = self + .tokenizer + .encode(sentence, true) + .map_err(Error::msg)? + .get_ids() + .to_vec(); + + let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; + let token_type_ids = token_ids.zeros_like()?; + + let embedding = self.model.forward(&token_ids, &token_type_ids, None)?; + let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; + let embedding = normalize_l2(&embedding)?; + + let embedding = embedding.get(0)?.to_vec1()?; + + Ok(embedding) + }) + .await? + } +} + +#[cfg(feature = "embedding")] +pub struct EmbeddingsDb { + db: Db, + model_instance: Arc, +} + +#[cfg(feature = "embedding")] +impl EmbeddingsDb { + pub async fn new(pg_db: &Pool, model_instance: Arc) -> Result { + let db = Db::new(); + + let mut embeddings_db = Self { db, model_instance: model_instance.clone() }; + + embeddings_db.fill_db(pg_db).await?; + + Ok(embeddings_db) + } + + async fn fill_db(&mut self, pg_db: &Pool) -> Result<()> { + if self.db.get_collection("scripts").is_some() { + self.db.delete_collection("scripts")?; + } + + self.db + .create_collection("scripts".to_string(), 384, Distance::Cosine)?; + + if self.db.get_collection("resource_types").is_some() { + self.db.delete_collection("resource_types")?; + } + + self.db + .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; + + let hub_base_url = HUB_BASE_URL.read().await.clone(); + + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") + .send() + .await; + + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } else { + response.unwrap() + } + } + _ => { + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/scripts/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } + }; + + if response.error_for_status_ref().is_err() { + return Err(anyhow!( + "Failed to get scripts embeddings from hub with error code: {}", + response.status() + )); + } + + let hub_scripts = response.json::>().await?; + + for script in &hub_scripts { + let mut hm = HashMap::new(); + hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); + hm.insert("summary".to_string(), script.summary.clone()); + hm.insert("app".to_string(), script.app.clone()); + hm.insert("kind".to_string(), script.kind.clone()); + hm.insert("id".to_string(), script.id.clone().to_string()); + hm.insert( + "version_id".to_string(), + script.version_id.clone().to_string(), + ); + let embedding = Embedding { + id: script.ask_id.clone().to_string(), + vector: script.embedding.clone(), + metadata: Some(hm), + }; + self.db.insert_into_collection("scripts", embedding)?; + } + + let response = match hub_base_url.as_str() { + DEFAULT_HUB_BASE_URL => { + let response = HTTP_CLIENT + .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") + .send() + .await; + if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { + tracing::warn!( + "Failed to get resource types embeddings from bucket, trying hub..." + ); + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } else { + response.unwrap() + } + } + _ => { + http_get_from_hub( + &HTTP_CLIENT, + &format!("{}/resource_types/embeddings", hub_base_url), + false, + None, + Some(pg_db), + ) + .await? + } + }; + + if response.error_for_status_ref().is_err() { + return Err(anyhow!( + "Failed to get resource types embeddings from hub with error code: {}", + response.status() + )); + } + let hub_resource_types = response.json::>().await?; + + let resource_types: Vec = + sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",) + .fetch_all(pg_db) + .await?; + + for rt in resource_types { + let mut hm = HashMap::new(); + hm.insert("name".to_string(), rt.name.clone()); + if let Some(schema) = rt.schema.clone() { + hm.insert("schema".to_string(), serde_json::to_string(&schema)?); + } + hm.insert("workspace".to_string(), rt.workspace_id.clone()); + let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name); + + let vector = if let Some(hub_rt) = hub_rt { + hub_rt.embedding.clone() + } else { + self.model_instance + .clone() + .create_embedding(&format!( + "{};{}", + rt.name, + rt.description.unwrap_or_default() + )) + .await? + }; + + let embedding = Embedding { + id: format!("{}_{}", rt.workspace_id, rt.name), + vector, + metadata: Some(hm), + }; + + self.db + .insert_into_collection("resource_types", embedding)?; + } + + Ok(()) + } + + pub async fn query_hub_scripts( + &self, + query: &str, + limit: Option, + kind: Option, + app: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("scripts"); + + let collection = collection.ok_or(Error::msg("no collection found"))?; + + let filter = |embedding: &Embedding| { + if let Some(metadata) = embedding.metadata.as_ref() { + match ( + metadata.get("kind"), + kind.clone(), + metadata.get("app"), + app.clone(), + ) { + (Some(script_kind), Some(kind), Some(script_app), Some(app)) => { + &kind == script_kind && &app == script_app + } + (Some(script_kind), Some(kind), _, _) => &kind == script_kind, + (_, _, Some(script_app), Some(app)) => &app == script_app, + (_, None, _, None) => true, + _ => false, + } + } else { + false + } + }; + + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + Some(&filter), + Some(0.8), + ); + + let results: Result> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + + Ok(HubScriptResult { + ask_id: metadata + .get("ask_id") + .ok_or(Error::msg("no ask_id"))? + .parse::()?, + summary: metadata + .get("summary") + .ok_or(Error::msg("no summary"))? + .to_owned(), + app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), + kind: metadata + .get("kind") + .ok_or(Error::msg("no kind"))? + .to_owned(), + id: metadata + .get("id") + .ok_or(Error::msg("no id"))? + .parse::()?, + version_id: metadata + .get("version_id") + .ok_or(Error::msg("no version_id"))? + .parse::()?, + score: r.score, + }) + }) + .collect(); + + let mut results = results?; + + if results.len() > 1 { + let top_score = results[0].score; + results = results + .into_iter() + .take_while(|r| (top_score - r.score) / top_score <= 0.05) + .collect(); + } + + Ok(results) + } + + pub async fn query_resource_types( + &self, + workspace: String, + query: &str, + limit: Option, + ) -> Result> { + let model_instance = self.model_instance.clone(); + let query_embedding = model_instance.create_embedding(query).await?; + + let collection = self.db.get_collection("resource_types"); + + if collection.is_none() { + return Ok(vec![]); + } + + let collection = collection.ok_or(Error::msg("no collection found"))?; + + let filter = |embedding: &Embedding| { + if let Some(metadata) = embedding.metadata.as_ref() { + match metadata.get("workspace").map(|x| x.as_str()) { + Some("admins") => true, + Some(rt_workspace) => &workspace == rt_workspace, + _ => false, + } + } else { + false + } + }; + + let results = collection.get_similarity( + &query_embedding, + limit.unwrap_or(10) as usize, + Some(&filter), + Some(0.75), + ); + + let results: Result<_> = results + .iter() + .map(|r| { + let metadata = r + .embedding + .metadata + .as_ref() + .ok_or(Error::msg("no metadata"))?; + Ok(ResourceTypeResult { + name: metadata + .get("name") + .ok_or(Error::msg("no name"))? + .to_owned(), + schema: match metadata.get("schema") { + Some(schema) => serde_json::from_str(schema)?, + None => None, + }, + score: r.score, + }) + }) + .collect(); + + results + } +} + +#[cfg(feature = "embedding")] +fn normalize_l2(v: &Tensor) -> Result { + Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?) +} + +#[cfg(feature = "embedding")] +pub fn load_embeddings_db(db: &Pool) -> () { + let disable_embedding = std::env::var("DISABLE_EMBEDDING") + .ok() + .map(|x| x.parse::().unwrap_or(false)) + .unwrap_or(false); + + if !disable_embedding { + let db_clone = db.clone(); + tokio::spawn(async move { + let model_instance = ModelInstance::new().await; + if let Ok(model_instance) = model_instance { + let mut model_instance_lock = MODEL_INSTANCE.write().await; + *model_instance_lock = Some(Arc::new(model_instance)); + drop(model_instance_lock); + loop { + update_embeddings_db(&db_clone).await; + tokio::time::sleep(std::time::Duration::from_secs( + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, + )) + .await; + } + } else { + tracing::error!( + "Failed to initialize model instance: {}", + model_instance.err().unwrap() + ); + } + }); + } +} + +#[cfg(feature = "embedding")] +pub async fn update_embeddings_db(db: &Pool) -> () { + if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { + tracing::info!("Creating embeddings DB..."); + let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; + if let Err(e) = new_embeddings_db.as_ref() { + tracing::error!("Failed to create embeddings db: {}", e); + } else { + let mut embeddings_db = EMBEDDINGS_DB.write().await; + *embeddings_db = new_embeddings_db.ok(); + tracing::info!("Created embeddings DB"); + } + } else { + tracing::error!("Could not update embeddings DB, model instance not initialized"); + } +} + +#[cfg(feature = "embedding")] +pub fn workspaced_service() -> Router { + Router::new().route("/query_resource_types", get(query_resource_types)) +} + +#[cfg(feature = "embedding")] +pub fn global_service() -> Router { + Router::new().route("/query_hub_scripts", get(query_hub_scripts)) +} + +#[cfg(not(feature = "embedding"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "embedding"))] +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api-jobs/Cargo.toml b/backend/windmill-api-jobs/Cargo.toml new file mode 100644 index 0000000000..d3e10205ac --- /dev/null +++ b/backend/windmill-api-jobs/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-api-jobs" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_jobs" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-api-sse.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +chrono.workspace = true +http.workspace = true +tower-http.workspace = true +base64.workspace = true +anyhow.workspace = true +tracing.workspace = true +sql-builder.workspace = true +tokio.workspace = true +hyper.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api-jobs/src/concurrency_groups.rs b/backend/windmill-api-jobs/src/concurrency_groups.rs new file mode 100644 index 0000000000..a727b7b02c --- /dev/null +++ b/backend/windmill-api-jobs/src/concurrency_groups.rs @@ -0,0 +1,364 @@ +use windmill_api_auth::{check_scopes, ApiAuthed}; +use windmill_common::{ + db::{UserDB, DB}, + error::Error::PermissionDenied, + error::{self, JsonResult}, + utils::require_admin, +}; + +use crate::query::{filter_list_completed_query, filter_list_queue_query}; +use crate::types::{ListCompletedQuery, ListQueueQuery, UnifiedJob}; +use windmill_api_sse::Job; + +use axum::extract::Path; +use axum::routing::{delete, get}; +use axum::{extract::Query, Extension, Json}; +use serde::Deserialize; + +use axum::Router; + +use serde::Serialize; +use sql_builder::bind::Bind; +use sql_builder::SqlBuilder; +use uuid::Uuid; + +pub fn global_service() -> Router { + Router::new() + .route("/list", get(list_concurrency_groups)) + .route("/prune/*concurrency_key", delete(prune_concurrency_group)) + .route("/:job_id/key", get(get_concurrency_key)) +} + +pub fn workspaced_service() -> Router { + Router::new().route("/list_jobs", get(get_concurrent_intervals)) +} + +#[derive(Serialize)] +pub struct ConcurrencyGroups { + concurrency_key: String, + total_running: i64, +} + +async fn list_concurrency_groups( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let concurrency_counts = sqlx::query_as::<_, (String, i64)>( + "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter", + ).fetch_all(&db) + .await?; + + let mut concurrency_groups: Vec = vec![]; + for (concurrency_key, count) in concurrency_counts { + concurrency_groups.push(ConcurrencyGroups { + concurrency_key: concurrency_key.clone(), + total_running: count, + }) + } + + return Ok(Json(concurrency_groups)); +} + +async fn prune_concurrency_group( + authed: ApiAuthed, + Extension(db): Extension, + Path(concurrency_key): Path, +) -> JsonResult<()> { + if !authed.is_admin { + return Err(PermissionDenied( + "Only administrators can delete concurrency groups".to_string(), + )); + } + let mut tx = db.begin().await?; + + let concurrency_group = sqlx::query_as::<_, (String, i64)>( + "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE", + ) + .bind(concurrency_key.clone()) + .fetch_optional(&mut *tx) + .await?; + + let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default(); + + if n_job_uuids > 0 { + tx.commit().await?; + return Err(error::Error::internal_err( + "Concurrency group is currently in use, unable to remove it. Retry later.".to_string(), + )); + } + + sqlx::query!( + "DELETE FROM concurrency_counter WHERE concurrency_id = $1", + concurrency_key.clone(), + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM concurrency_key WHERE key = $1", + concurrency_key.clone(), + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(())) +} + +#[derive(Serialize)] +struct ExtendedJobs { + jobs: Vec, + obscured_jobs: Vec, + omitted_obscured_jobs: bool, +} + +#[derive(Serialize)] +struct ObscuredJob { + typ: String, + started_at: Option>, + duration_ms: Option, +} +#[derive(Deserialize)] +struct ExtendedJobsParams { + row_limit: Option, +} + +pub fn join_concurrency_key<'c>( + concurrency_key: Option<&String>, + mut sqlb: SqlBuilder, +) -> SqlBuilder { + if let Some(key) = concurrency_key { + sqlb.join("concurrency_key") + .on_eq("id", "concurrency_key.job_id") + .and_where_eq("key", "?".bind(key)); + } + + sqlb +} + +async fn get_concurrent_intervals( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Query(iq): Query, + Query(lq): Query, +) -> JsonResult { + check_scopes(&authed, || format!("jobs:read"))?; + + if lq.success.is_some() && lq.running.is_some_and(|x| x) { + return Err(error::Error::BadRequest( + "cannot specify both success and running".to_string(), + )); + } + + let row_limit = iq.row_limit.unwrap_or(1000); + + let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; + let lqc = lq.clone(); + let lqq: ListQueueQuery = lqc.into(); + let mut sqlb_q = SqlBuilder::select_from("v2_job_queue") + .fields(UnifiedJob::queued_job_fields()) + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_c = SqlBuilder::select_from("v2_job_completed") + .fields(UnifiedJob::completed_job_fields()) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue") + .fields(&["id"]) + .order_by("created_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed") + .fields(&["id"]) + .order_by("completed_at", lq.order_desc.unwrap_or(true)) + .limit(row_limit) + .clone(); + + sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q); + sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c); + sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user); + sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user); + + let should_fetch_obscured_jobs = match lq { + ListCompletedQuery { + script_path_start: None, + script_path_exact: None, + script_hash: None, + created_by: None, + success: None, + running: None, + parent_job: None, + is_skipped: None | Some(false), + suspended: None, + schedule_path: None, + args: None, + result: None, + tag: None, + has_null_parent: None, + worker: None, + label: None, + trigger_path: None, + scheduled_for_before_now: _, + is_not_schedule: _, + started_before: _, + started_after: _, + created_before: _, + created_after: _, + created_before_queue: _, + created_after_queue: _, + completed_after: _, + completed_before: _, + created_or_started_before: _, + created_or_started_after: _, + created_or_started_after_completed_jobs: _, + order_desc: _, + job_kinds: _, + is_flow_step: _, + all_workspaces: _, + concurrency_key: Some(_), + allow_wildcards: None, + trigger_kind: _, + include_args: _, + } => true, + _ => false, + }; + + // When we have a concurrency key defined, fetch jobs from other workspaces + // as obscured unless we're in the admins workspace. This is to show the + // potential concurrency races without showing jobs that don't belong to + // the workspace. + // To avoid infering information through filtering, don't return obscured + // jobs if the filters are too specific + if should_fetch_obscured_jobs && w_id != "admins" { + // Get the obscured jobs from all workspaces (concurrency key could be global) + let (sqlb_q, sqlb_c) = ( + filter_list_queue_query( + sqlb_q, + &ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() }, + "admins", + true, + ), + filter_list_completed_query( + sqlb_c, + &ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() }, + "admins", + true, + ), + ); + + sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true); + sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true); + + let sql_q_user = sqlb_q_user.query()?; + let sql_c_user = sqlb_c_user.query()?; + let sql_q = sqlb_q.query()?; + let sql_c = sqlb_c.query()?; + + // This first transaction uses the user_db to know which uuids are + // accessible to the user. + let mut tx = user_db.begin(&authed).await?; + let running_jobs_user: Vec = if lq.success.is_none() { + sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await? + } else { + vec![] + }; + let completed_jobs_user: Vec = if lq.running.is_none() { + sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await? + } else { + vec![] + }; + tx.commit().await?; + + // This second transaction uses the db, so it will fetch information + // potentially forbidden to the user. It must be obscured before + // returning it + let running_jobs_db: Vec = if lq.success.is_none() { + sqlx::query_as(&sql_q).fetch_all(&db).await? + } else { + vec![] + }; + let completed_jobs_db: Vec = if lq.running.is_none() { + sqlx::query_as(&sql_c).fetch_all(&db).await? + } else { + vec![] + }; + + let obscured_jobs = running_jobs_db + .iter() + .filter(|j| !running_jobs_user.iter().any(|id| j.id == *id)) + .chain( + completed_jobs_db + .iter() + .filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)), + ) + .map(|j| ObscuredJob { + typ: j.typ.clone(), + started_at: j.started_at, + duration_ms: j.duration_ms, + }) + .collect(); + + let jobs = running_jobs_db + .into_iter() + .filter(|j| running_jobs_user.iter().any(|id| j.id == *id)) + .chain( + completed_jobs_db + .into_iter() + .filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)), + ) + .map(From::from) + .collect(); + + Ok(Json(ExtendedJobs { + jobs, + obscured_jobs, + omitted_obscured_jobs: !should_fetch_obscured_jobs, + })) + } else { + sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true); + sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true); + let sql_q = sqlb_q.query()?; + let sql_c = sqlb_c.query()?; + + let mut tx = user_db.begin(&authed).await?; + let running_jobs: Vec = if lq.success.is_none() { + sqlx::query_as(&sql_q).fetch_all(&mut *tx).await? + } else { + vec![] + }; + let completed_jobs: Vec = if lq.running.is_none() { + sqlx::query_as(&sql_c).fetch_all(&mut *tx).await? + } else { + vec![] + }; + tx.commit().await?; + + let jobs = running_jobs + .into_iter() + .chain(completed_jobs.into_iter()) + .map(From::from) + .collect(); + + Ok(Json(ExtendedJobs { + jobs, + obscured_jobs: vec![], + omitted_obscured_jobs: !should_fetch_obscured_jobs, + })) + } +} + +async fn get_concurrency_key( + Extension(db): Extension, + Path(job_id): Path, +) -> JsonResult> { + let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id) + .fetch_optional(&db) + .await?; + Ok(Json(key)) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs new file mode 100644 index 0000000000..08aebe4ddd --- /dev/null +++ b/backend/windmill-api-jobs/src/execution.rs @@ -0,0 +1,892 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Job execution primitives: push, cancel, tag/license checks, wait-for-result. + +use axum::{ + response::{IntoResponse, Response}, + Json, +}; +use http::{HeaderMap, HeaderName, HeaderValue}; +use hyper::StatusCode; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::str::FromStr; +use std::sync::Arc; +use tokio::sync::RwLock; +use uuid::Uuid; +use windmill_api_auth::{check_scopes, get_scope_tags, ApiAuthed}; +use windmill_common::{ + db::{UserDB, UserDbWithAuthed}, + error::{self, Error}, + flow_conversations::{add_message_to_conversation_tx, MessageType}, + get_latest_flow_version_info_for_path, + jobs::{ + check_tag_available_for_workspace_internal, format_result, script_path_to_payload, + JobPayload, + }, + triggers::TriggerMetadata, + users::username_to_permissioned_as, + utils::StripPath, + FlowVersionInfo, DB, +}; +use windmill_queue::{ + cancel_job, get_result_and_success_by_id_from_flow, push, PushArgs, PushArgsOwned, + PushIsolationLevel, +}; + +use crate::types::RunJobQuery; + +// --------------------------------------------------------------------------- +// Tag / license checks +// --------------------------------------------------------------------------- + +pub async fn check_tag_available_for_workspace( + db: &DB, + w_id: &str, + tag: &Option, + authed: &ApiAuthed, +) -> error::Result<()> { + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { + let tags = get_scope_tags(authed); + check_tag_available_for_workspace_internal(&db, w_id, tag, &authed.email, tags).await + } else { + Ok(()) + } +} + +#[cfg(feature = "enterprise")] +pub async fn check_license_key_valid() -> error::Result<()> { + use windmill_common::ee_oss::LICENSE_KEY_VALID; + + let valid = *LICENSE_KEY_VALID.read().await; + if !valid { + return Err(error::Error::BadRequest( + "License key is not valid. Go to your superadmin settings to update your license key." + .to_string(), + )); + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Cancel jobs +// --------------------------------------------------------------------------- + +pub async fn cancel_jobs( + jobs: Vec, + db: &DB, + username: &str, + w_id: &str, + force_cancel: bool, +) -> error::JsonResult> { + let mut uuids = vec![]; + tracing::info!("Cancelling jobs: {:?}", jobs); + let mut tx = db.begin().await?; + let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj + ( workspace_id + , id + , duration_ms + , result + , canceled_by + , canceled_reason + , flow_status + , status + , worker + ) + SELECT q.workspace_id + , q.id + , 0 + , $4 + , $1 + , 'cancel all' + , (SELECT flow_status FROM v2_job_status WHERE id = q.id) + , 'canceled'::job_status + , worker + FROM v2_job_queue q + JOIN v2_job USING (id) + WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule' + FOR UPDATE SKIP LOCKED + ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) + .fetch_all(&mut *tx) + .await?.into_iter().map(|x| x.id).collect::>(); + + sqlx::query!( + "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", + &trivial_jobs, + w_id + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + for job_id in jobs.into_iter() { + if trivial_jobs.contains(&job_id) { + continue; + } + match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { + let tx = db.begin().await?; + let (tx, _) = cancel_job( + username, + None, + job_id.clone(), + w_id, + tx, + db, + force_cancel, + false, + ) + .await?; + tx.commit().await?; + Ok::<_, anyhow::Error>(()) + }) + .await + { + Ok(result) => match result { + Ok(_) => { + uuids.push(job_id); + } + Err(e) => { + tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); + } + }, + Err(_) => { + tracing::error!( + "Timeout while trying to cancel job {:?} after 5 seconds", + job_id + ); + } + } + } + + uuids.extend(trivial_jobs); + + Ok(Json(uuids)) +} + +// --------------------------------------------------------------------------- +// Wait-for-result infrastructure +// --------------------------------------------------------------------------- + +struct Guard { + done: bool, + id: Uuid, + w_id: String, + db: DB, + username: String, +} + +impl Drop for Guard { + fn drop(&mut self) { + if !&self.done { + let id = self.id; + let w_id = self.w_id.clone(); + let db = self.db.clone(); + let username = self.username.clone(); + + tracing::info!("http connection broke, marking job {id} as canceled"); + tokio::spawn(async move { + let cancel_f = async { + let tx = db.begin().await?; + let (tx, _) = cancel_job( + &username, + Some("http connection broke".to_string()), + id, + &w_id, + tx, + &db, + false, + false, + ) + .await?; + tx.commit().await?; + Ok::<_, anyhow::Error>(()) + }; + + if let Err(e) = cancel_f.await { + tracing::error!( + "Error marking job as canceled after http connection broke: {e}" + ); + } + }); + } + } +} + +lazy_static::lazy_static! { + pub static ref TIMEOUT_WAIT_RESULT: Arc>> = Arc::new(RwLock::new( + std::env::var("TIMEOUT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse::().ok()) + )); + + pub static ref QUEUE_LIMIT_WAIT_RESULT: Option = std::env::var("QUEUE_LIMIT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse().ok()); + + pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(50); + + pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(2); + + pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(200); +} + +#[derive(Deserialize)] +pub struct WindmillCompositeResult { + windmill_status_code: Option, + windmill_content_type: Option, + windmill_headers: Option>, + result: Option>, +} + +pub async fn run_wait_result_internal( + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, +) -> error::Result<(Box, bool)> { + let mut result = None; + let mut success = false; + let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600); + let timeout_ms = if timeout <= 0 { + 2000 + } else { + (timeout * 1000) as u64 + }; + + let mut g = Guard { + done: false, + id: uuid, + w_id: w_id.to_string(), + db: db.clone(), + username: username.to_string(), + }; + + let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000; + let mut accumulated_delay = 0 as u64; + + loop { + if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() { + let result_and_success = get_result_and_success_by_id_from_flow( + &db, + w_id, + &uuid, + node_id_for_empty_return, + None, + ) + .await + .ok(); + if let Some((r, s)) = result_and_success { + result = Some(r); + success = s; + } + } + + if result.is_none() { + let row = sqlx::query!( + " + SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\" + FROM + v2_job_completed + WHERE + id = $1 AND + workspace_id = $2 + ", + uuid, + &w_id + ) + .fetch_optional(db) + .await?; + if let Some(mut raw_result) = row { + format_result( + raw_result.result_columns.as_ref(), + raw_result.result.as_mut(), + ); + result = raw_result.result.map(|x| x.0); + success = raw_result.success; + } + } + + if result.is_some() { + break; + } + + let delay = if accumulated_delay <= fast_poll_duration { + *WAIT_RESULT_FAST_POLL_INTERVAL_MS + } else { + *WAIT_RESULT_SLOW_POLL_INTERVAL_MS + }; + accumulated_delay += delay; + if accumulated_delay > timeout_ms { + break; + }; + tokio::time::sleep(core::time::Duration::from_millis(delay)).await; + } + + if let Some(result) = result { + g.done = true; + Ok((result, success)) + } else { + Err(Error::ExecutionErr(format!("timeout after {}s", timeout))) + } +} + +pub fn result_to_response(result: Box, success: bool) -> error::Result { + let composite_result = serde_json::from_str::(result.get()); + match composite_result { + Ok(WindmillCompositeResult { + windmill_status_code, + windmill_content_type, + windmill_headers, + result: result_value, + }) => { + if windmill_content_type.is_none() + && windmill_status_code.is_none() + && windmill_headers.is_none() + { + return Ok(( + if success { + StatusCode::OK + } else { + StatusCode::UNPROCESSABLE_ENTITY + }, + Json(result), + ) + .into_response()); + } + + let status_code_or_default = windmill_status_code + .map(|val| match StatusCode::from_u16(val) { + Ok(sc) => Ok(sc), + Err(_) => Err(Error::ExecutionErr("Invalid status code".to_string())), + }) + .unwrap_or_else(|| { + if !success { + Ok(StatusCode::UNPROCESSABLE_ENTITY) + } else if result_value.is_some() { + Ok(StatusCode::OK) + } else { + Ok(StatusCode::NO_CONTENT) + } + })?; + + let mut headers = HeaderMap::new(); + + if let Some(windmill_headers) = windmill_headers { + for (k, v) in windmill_headers { + let k = HeaderName::from_str(k.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid header name {k}: {err}")) + })?; + let v = HeaderValue::from_str(v.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid header value {v}: {err}")) + })?; + headers.insert(k, v); + } + } + + if let Some(content_type) = windmill_content_type { + let serialized_json_result = result_value + .map(|val| val.get().to_owned()) + .unwrap_or_else(String::new); + let serialized_result = + serde_json::from_str::(serialized_json_result.as_str()) + .ok() + .unwrap_or(serialized_json_result); + headers.insert( + http::header::CONTENT_TYPE, + HeaderValue::from_str(content_type.as_str()).map_err(|err| { + Error::internal_err(format!("Invalid content type {content_type}: {err}")) + })?, + ); + return Ok((status_code_or_default, headers, serialized_result).into_response()); + } + if let Some(result_value) = result_value { + return Ok((status_code_or_default, headers, Json(result_value)).into_response()); + } else { + Ok((status_code_or_default, headers).into_response()) + } + } + _ => Ok(( + if success { + StatusCode::OK + } else { + StatusCode::UNPROCESSABLE_ENTITY + }, + Json(result), + ) + .into_response()), + } +} + +pub async fn run_wait_result( + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, +) -> error::Result { + let (result, success) = + run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username).await?; + + result_to_response(result, success) +} + +// --------------------------------------------------------------------------- +// Delete job metadata +// --------------------------------------------------------------------------- + +pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { + sqlx::query!( + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", + job_uuid, + ) + .execute(db) + .await?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Queue limit check +// --------------------------------------------------------------------------- + +pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { + if let Some(limit) = queue_limit { + let count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + if count > queue_limit.unwrap() { + return Err(Error::internal_err(format!( + "Number of queued job is too high: {count} > {limit}" + ))); + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Flow run helpers +// --------------------------------------------------------------------------- + +pub async fn set_flow_memory_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + job_id: Uuid, + memory_id: Uuid, +) -> error::Result<()> { + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = jsonb_set( + flow_status, + '{memory_id}', + to_jsonb($2::uuid) + ) + WHERE id = $1", + job_id, + memory_id + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Apply flow-specific query parameters after job creation +pub async fn process_flow_run_query_params( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + job_id: Uuid, + run_query: &RunJobQuery, +) -> error::Result<()> { + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(tx, job_id, memory_id).await?; + } + Ok(()) +} + +pub async fn handle_chat_conversation_messages( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + authed: &ApiAuthed, + w_id: &str, + flow_path: &str, + run_query: &RunJobQuery, + user_message_raw: Option<&Box>, +) -> error::Result<()> { + let memory_id = run_query.memory_id.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "memory_id is required for chat-enabled flows".to_string(), + ) + })?; + + let user_message_raw = user_message_raw.ok_or_else(|| { + windmill_common::error::Error::BadRequest( + "user_message argument is required for chat-enabled flows".to_string(), + ) + })?; + + let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| { + windmill_common::error::Error::BadRequest(format!( + "Failed to deserialize user_message: {}", + e + )) + })?; + + windmill_common::flow_conversations::get_or_create_conversation_with_id( + tx, + w_id, + flow_path, + &authed.username, + &user_message, + memory_id, + ) + .await?; + + add_message_to_conversation_tx( + tx, + memory_id, + None, + &user_message, + MessageType::User, + None, + true, + ) + .await?; + + Ok(()) +} + +pub async fn run_flow<'c>( + authed: &ApiAuthed, + db: &DB, + tx_o: Option>, + user_db: UserDB, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + let FlowVersionInfo { + version, + tag, + dedicated_worker, + has_preprocessor, + chat_input_enabled, + on_behalf_of_email, + edited_by, + early_return, + .. + } = flow_version_info; + + let tag = run_query.tag.clone().or(tag); + + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; + + let return_tx = tx_o.is_some(); + + let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { + ( + &authed.email, + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Transaction(tx), + ) + } else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { + ( + on_behalf_of_email, + username_to_permissioned_as(&edited_by), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + &authed.email, + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), + ) + }; + + let (uuid, mut tx) = push( + &db, + tx, + &w_id, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + version, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, + PushArgs { args: &args.args, extra: args.extra }, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + scheduled_for, + None, + run_query.parent_job, + None, + run_query.root_job, + run_query.job_id, + false, + false, + None, + !run_query.invisible_to_owner.unwrap_or(false), + tag, + None, + None, + None, + push_authed.as_ref(), + false, + None, + trigger, + run_query.suspended_mode, + ) + .await?; + + // Set memory_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled.unwrap_or(false) { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path.to_string(), + &run_query, + args.args.get("user_message"), + ) + .await?; + } + + // If we were given a transaction, return it; otherwise commit it + if return_tx { + Ok((uuid, early_return, Some(tx))) + } else { + tx.commit().await?; + Ok((uuid, early_return, None)) + } +} + +pub async fn run_flow_and_wait_result( + authed: &ApiAuthed, + db: &DB, + user_db: UserDB, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result { + let (uuid, early_return, _) = run_flow( + authed, + db, + None, + user_db, + w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await?; + + run_wait_result(&db, uuid, w_id, early_return, &authed.username).await +} + +// --------------------------------------------------------------------------- +// Push job by path +// --------------------------------------------------------------------------- + +pub async fn push_flow_job_by_path_into_queue<'c>( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = flow_path.to_path(); + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + + let flow_version_info = + get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) + .await?; + + run_flow( + &authed, + &db, + tx_o, + user_db, + &w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await +} + +pub async fn push_script_job_by_path_into_queue<'c>( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( + script_path, + Some(userdb_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; + + let tag = run_query.tag.clone().or(tag); + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + + let return_tx = tx_o.is_some(); + + let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Transaction(tx), + ) + } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { + ( + on_behalf_of.email.as_str(), + on_behalf_of.permissioned_as.clone(), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db, authed.clone().into()), + ) + }; + + let (uuid, tx) = push( + &db, + tx, + &w_id, + job_payload, + PushArgs { args: &args.args, extra: args.extra }, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + scheduled_for, + None, + run_query.parent_job, + None, + run_query.root_job, + run_query.job_id, + false, + false, + None, + !run_query.invisible_to_owner.unwrap_or(false), + tag, + timeout, + None, + // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. + if run_query.parent_job.is_some() || run_query.root_job.is_some() { + Some(2) + } else { + None + }, + push_authed.as_ref(), + false, + None, + trigger, + run_query.suspended_mode, + ) + .await?; + + // If we were given a transaction, return it; otherwise commit it + if return_tx { + Ok((uuid, delete_after_use, Some(tx))) + } else { + tx.commit().await?; + Ok((uuid, delete_after_use, None)) + } +} diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs new file mode 100644 index 0000000000..aba8a5f25c --- /dev/null +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -0,0 +1,266 @@ +use windmill_common::DB; + +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use tower_http::cors::{Any, CorsLayer}; +use uuid::Uuid; +use windmill_common::{ + error::{self, Error}, + job_metrics::{ + record_metric, register_metric_for_job, JobStatsRecord, MetricKind, MetricNumericValue, + }, +}; + +pub fn workspaced_service() -> Router { + let cors = CorsLayer::new() + .allow_methods([http::Method::GET, http::Method::POST]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .allow_origin(Any); + + Router::new() + .route("/get/:id", post(get_job_metrics).layer(cors.clone())) + .route( + "/set_progress/:id", + post(set_job_progress).layer(cors.clone()), + ) + .route( + "/get_progress/:id", + get(get_job_progress).layer(cors.clone()), + ) +} + +#[derive(Deserialize)] +struct JobStatsRequest { + from_timestamp: Option>, + to_timestamp: Option>, + timeseries_max_datapoints: Option, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all +} + +#[derive(Serialize)] +struct JobStatsResponse { + metrics_metadata: Vec, + scalar_metrics: Vec, + timeseries_metrics: Vec, +} + +#[derive(Serialize)] +pub struct MetricsMetadata { + pub id: String, + pub name: Option, +} + +#[derive(Serialize)] +struct ScalarMetric { + metric_id: String, + value: f64, +} + +#[derive(Serialize)] +struct TimeseriesMetric { + metric_id: String, + values: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DataPoint { + pub timestamp: chrono::DateTime, + pub value: f64, +} + +async fn get_job_metrics( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json< + JobStatsRequest, + >, +) -> error::JsonResult { + let records = sqlx::query_as::<_, JobStatsRecord>( + "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", + ) + .bind(w_id) + .bind(job_id) + .fetch_all(&db) + .await?; + + let mut metrics_metadata: Vec = vec![]; + let mut scalar_metrics: Vec = vec![]; + let mut timeseries_metrics: Vec = vec![]; + + for record in records { + let metric_id = record.metric_id; + match record.metric_kind { + MetricKind::ScalarInt => { + let value = record.scalar_int.unwrap_or_default() as f64; + scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); + } + MetricKind::ScalarFloat => { + let value = record.scalar_float.unwrap_or_default() as f64; + scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); + } + MetricKind::TimeseriesInt => { + if record.timestamps.clone().unwrap_or_default().len() + != record.timeseries_int.clone().unwrap_or_default().len() + { + tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) + } + let (timestamps, timeseries_int) = timeseries_sample( + from_timestamp, + to_timestamp, + timeseries_max_datapoints, + record.timestamps.unwrap_or_default(), + record.timeseries_int.unwrap_or_default(), + ); + let mut values: Vec = vec![]; + for (idx, value) in timeseries_int.iter().enumerate() { + values.push(DataPoint { + timestamp: timestamps[idx], + value: value.to_owned() as f64, + }); + } + timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); + } + MetricKind::TimeseriesFloat => { + if record.timestamps.clone().unwrap_or_default().len() + != record.timeseries_int.clone().unwrap_or_default().len() + { + tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) + } + let (timestamps, timeseries_float) = timeseries_sample( + from_timestamp, + to_timestamp, + timeseries_max_datapoints, + record.timestamps.unwrap_or_default(), + record.timeseries_float.unwrap_or_default(), + ); + let mut values: Vec = vec![]; + for (idx, value) in timeseries_float.iter().enumerate() { + values.push(DataPoint { + timestamp: timestamps[idx], + value: value.to_owned() as f64, + }); + } + timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); + } + }; + metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name }); + } + + let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; + Ok(Json(response)) +} +#[derive(Deserialize)] +struct JobProgressSetRequest { + percent: i32, + /// Optional parent flow id + /// Used to modify flow status + /// Specifically `progress` field in corresponding FlowStatusModule in `InProgress` state + flow_job_id: Option, +} + +async fn set_job_progress( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Json(JobProgressSetRequest { percent, flow_job_id }): Json, +) -> error::JsonResult<()> { + // If flow_job_id exists, than we should modify flow_status of corresponding module + // Individual jobs and flows are handled differently + if let Some(flow_job_id) = flow_job_id { + // TODO: Return error if trying to set completed job? + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) + WHERE id = $2", + serde_json::json!(percent.clamp(0, 99)), + flow_job_id + ) + .execute(&db) + .await?; + } + + let record_progress = || { + record_metric( + &db, + w_id.clone(), + job_id, + "progress_perc".to_owned(), + MetricNumericValue::Integer(percent), + ) + }; + + // Try to record + if let Err(err) = record_progress().await { + if matches!(err, Error::MetricNotFound(..)) { + // Register + // TODO: Reset progress after job is finished (in case it reruns same job)? + _ = register_metric_for_job( + &db, + w_id.clone(), + job_id, + "progress_perc".to_string(), + MetricKind::ScalarInt, + Some("Job Execution Progress (%)".to_owned()), + ) + .await?; + // Retry recording progress + record_progress().await.map_err(|err| { + // If for some reason it still returns same error, this error will be converted to BadRequest and returned + if let Error::MetricNotFound(body) = err { + Error::BadRequest(body) + } else { + err + } + })?; + } else { + return Err(err); + } + }; + return Ok(Json(())); +} + +async fn get_job_progress( + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, +) -> error::JsonResult> { + let progress: Option> = sqlx::query_scalar!( + "SELECT (scalar_int)::int FROM job_stats WHERE job_id = $1 AND workspace_id = $2 AND metric_id = 'progress_perc'", + job_id, w_id) + .fetch_optional(&db) + .await?; + + let respond_value = if let Some(Some(progress)) = progress { + Some(progress.clamp(0, 99)) + } else { + None + }; + + Ok(Json(respond_value)) +} + +fn timeseries_sample( + from: Option>, + to: Option>, + _datapoints: Option, + timestamps: Vec>, + values: Vec, +) -> (Vec>, Vec) { + if timestamps.len() != values.len() { + tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len()); + return (vec![], vec![]); + } + let mut filtered_timestamp: Vec> = vec![]; + let mut filtered_values: Vec = vec![]; + for (idx, timestamp) in timestamps.iter().enumerate() { + if *timestamp > from.unwrap_or(chrono::DateTime::::MIN_UTC) + && *timestamp < to.unwrap_or(chrono::DateTime::::MAX_UTC) + { + filtered_timestamp.push(timestamps[idx]); + filtered_values.push(values[idx]); + } + } + // TODO: implement sampling + return (filtered_timestamp, filtered_values); +} diff --git a/backend/windmill-api-jobs/src/jobs.rs b/backend/windmill-api-jobs/src/jobs.rs new file mode 100644 index 0000000000..afee1ab953 --- /dev/null +++ b/backend/windmill-api-jobs/src/jobs.rs @@ -0,0 +1,4 @@ +// benchmark_modification + +// benchmark +pub fn _benchmark_test_fn() -> bool { true } diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs new file mode 100644 index 0000000000..5de2c51795 --- /dev/null +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -0,0 +1,690 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error, + jobs::{JobKind, JobStatus, JobTriggerKind}, + scripts::ScriptLang, + utils::{paginate, paginate_without_limits, require_admin, Pagination}, +}; + +use windmill_api_auth::ApiAuthed; + +#[derive(Serialize, Deserialize)] +pub struct ExportableCompletedJob { + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + pub duration_ms: i64, + pub result: Option>>, + pub deleted: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub flow_status: Option>>, + pub started_at: Option>, + pub memory_peak: Option, + pub status: JobStatus, + pub completed_at: chrono::DateTime, + pub worker: Option, + pub workflow_as_code_status: Option>>, + pub result_columns: Option>, + pub retries: Option>, + pub extras: Option>>, + pub logs: Option, + pub log_offset: Option, + pub log_file_index: Option>, +} + +#[derive(Serialize, Deserialize)] +pub struct ExportableQueuedJob { + // v2_job columns (excluding workspace_id) + pub id: Uuid, + pub raw_code: Option, + pub raw_lock: Option, + pub raw_flow: Option>>, + pub tag: String, + pub created_at: chrono::DateTime, + pub created_by: String, + pub permissioned_as: String, + pub permissioned_as_email: String, + pub kind: JobKind, + pub runnable_id: Option, + pub runnable_path: Option, + pub parent_job: Option, + pub root_job: Option, + pub script_lang: Option, + pub script_entrypoint_override: Option, + pub flow_step: Option, + pub flow_step_id: Option, + pub flow_innermost_root_job: Option, + pub trigger: Option, + pub trigger_kind: Option, + pub same_worker: bool, + pub visible_to_owner: bool, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub timeout: Option, + pub priority: Option, + pub preprocessed: Option, + pub args: Option>>, + pub labels: Option>, + pub pre_run_error: Option, + + // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) + pub started_at: Option>, + pub scheduled_for: chrono::DateTime, + pub running: bool, + pub canceled_by: Option, + pub canceled_reason: Option, + pub suspend: Option, + pub suspend_until: Option>, + pub worker: Option, + pub extras: Option>>, + + // v2_job_runtime columns (excluding id) + pub ping: Option>, + pub memory_peak: Option, + + // v2_job_status columns (excluding id) + pub flow_status: Option>>, + pub flow_leaf_jobs: Option>>, + pub workflow_as_code_status: Option>>, + + // concurrency_key table + pub concurrency_key: Option, +} + +pub async fn export_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableCompletedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.trigger, + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + v2_job_completed.duration_ms, + v2_job_completed.result as "result: _", + v2_job_completed.deleted, + v2_job_completed.canceled_by, + v2_job_completed.canceled_reason, + v2_job_completed.flow_status as "flow_status: _", + v2_job_completed.started_at, + v2_job_completed.memory_peak, + v2_job_completed.status as "status: _", + v2_job_completed.completed_at, + v2_job_completed.worker, + v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _", + v2_job_completed.result_columns, + v2_job_completed.retries, + v2_job_completed.extras as "extras: _", + job_logs.logs AS logs, + job_logs.log_offset, + job_logs.log_file_index + FROM v2_job_completed + INNER JOIN v2_job ON v2_job.id = v2_job_completed.id + LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id + LEFT JOIN job_logs ON job_logs.job_id = v2_job.id + WHERE v2_job_completed.workspace_id = $1 + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn export_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(pagination): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + + let (per_page, offset) = paginate_without_limits(pagination); + let per_page = per_page as i64; + let offset = offset as i64; + + let mut tx = user_db.begin(&authed).await?; + + let jobs = sqlx::query_as!( + ExportableQueuedJob, + r#" + SELECT + v2_job.id, + v2_job.raw_code, + v2_job.raw_lock, + v2_job.raw_flow as "raw_flow: _", + v2_job.tag, + v2_job.created_at, + v2_job.created_by, + v2_job.permissioned_as, + v2_job.permissioned_as_email, + v2_job.kind as "kind: _", + v2_job.runnable_id, + v2_job.runnable_path, + v2_job.parent_job, + v2_job.root_job, + v2_job.script_lang as "script_lang: _", + v2_job.script_entrypoint_override, + v2_job.flow_step, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job."trigger", + v2_job.trigger_kind as "trigger_kind: _", + v2_job.same_worker, + v2_job.visible_to_owner, + v2_job.concurrent_limit, + v2_job.concurrency_time_window_s, + v2_job.cache_ttl, + v2_job.timeout, + v2_job.priority, + v2_job.preprocessed, + v2_job.args as "args: _", + v2_job.labels, + v2_job.pre_run_error, + + v2_job_queue.started_at, + v2_job_queue.scheduled_for, + v2_job_queue.running, + v2_job_queue.canceled_by, + v2_job_queue.canceled_reason, + v2_job_queue.suspend, + v2_job_queue.suspend_until, + v2_job_queue.worker, + v2_job_queue.extras as "extras: _", + + v2_job_runtime.ping, + v2_job_runtime.memory_peak, + + v2_job_status.flow_status as "flow_status: _", + v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _", + v2_job_status.workflow_as_code_status as "workflow_as_code_status: _", + + concurrency_key.key as "concurrency_key?" + FROM v2_job_queue + INNER JOIN v2_job ON v2_job.id = v2_job_queue.id + LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id + LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id + LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id + WHERE v2_job_queue.workspace_id = $1 + AND v2_job_queue.running = false + AND v2_job.parent_job IS NULL + AND v2_job.trigger_kind IS DISTINCT FROM 'schedule' + ORDER BY v2_job.created_at DESC + LIMIT $2 + OFFSET $3 + "#, + w_id, + per_page, + offset + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(jobs)) +} + +pub async fn import_completed_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by, + permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path, + parent_job, root_job, script_lang, script_entrypoint_override, flow_step, + flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker, + visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, + timeout, priority, preprocessed, args, labels, pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, job.raw_code, job.raw_lock, job.raw_flow as _, &job.tag, &w_id, + job.created_at, &job.created_by, &job.permissioned_as, job.permissioned_as_email, + job.kind as _, job.runnable_id, job.runnable_path, job.parent_job, job.root_job, + job.script_lang as _, job.script_entrypoint_override, job.flow_step, + job.flow_step_id, job.flow_innermost_root_job, job.trigger, job.trigger_kind as _, + job.same_worker, job.visible_to_owner, job.concurrent_limit, + job.concurrency_time_window_s, job.cache_ttl, job.timeout, job.priority, + job.preprocessed, job.args as _, job.labels as _, job.pre_run_error + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + r#" + INSERT INTO v2_job_completed ( + id, workspace_id, started_at, completed_at, duration_ms, result, deleted, + canceled_by, canceled_reason, flow_status, memory_peak, status, worker, + workflow_as_code_status, result_columns, retries, extras + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.completed_at, + job.duration_ms, + job.result as _, + job.deleted, + job.canceled_by, + job.canceled_reason, + job.flow_status as _, + job.memory_peak, + job.status as _, + job.worker, + job.workflow_as_code_status as _, + job.result_columns as _, + job.retries as _, + job.extras as _ + ) + .execute(&mut *tx) + .await?; + + if let Some(logs) = &job.logs { + sqlx::query!( + r#" + INSERT INTO job_logs ( + job_id, workspace_id, logs, log_offset, log_file_index + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (job_id) DO NOTHING + "#, + job.id, + &w_id, + logs, + job.log_offset, + job.log_file_index as _ + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + Ok(format!("Successfully imported jobs")) +} + +pub async fn import_queued_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(jobs): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + for job in jobs { + sqlx::query!( + r#" + INSERT INTO v2_job ( + id, raw_code, raw_lock, raw_flow, tag, workspace_id, created_at, created_by, + permissioned_as, permissioned_as_email, kind, runnable_id, runnable_path, + parent_job, root_job, script_lang, script_entrypoint_override, flow_step, + flow_step_id, flow_innermost_root_job, trigger, trigger_kind, same_worker, + visible_to_owner, concurrent_limit, concurrency_time_window_s, cache_ttl, + timeout, priority, preprocessed, args, labels, pre_run_error + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, + $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, + $31, $32, $33 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.raw_code, + job.raw_lock, + job.raw_flow as _, + &job.tag, + &w_id, + job.created_at, + &job.created_by, + &job.permissioned_as, + &job.permissioned_as_email, + job.kind as _, + job.runnable_id, + job.runnable_path, + job.parent_job, + job.root_job, + job.script_lang as _, + job.script_entrypoint_override, + job.flow_step, + job.flow_step_id, + job.flow_innermost_root_job, + job.trigger, + job.trigger_kind as _, + job.same_worker, + job.visible_to_owner, + job.concurrent_limit, + job.concurrency_time_window_s, + job.cache_ttl, + job.timeout, + job.priority, + job.preprocessed, + job.args as _, + job.labels as _, + job.pre_run_error, + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + r#" + INSERT INTO v2_job_queue ( + id, workspace_id, started_at, scheduled_for, running, canceled_by, + canceled_reason, suspend, suspend_until, worker, extras, tag, priority + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 + ) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + &w_id, + job.started_at, + job.scheduled_for, + job.running, + job.canceled_by, + job.canceled_reason, + job.suspend, + job.suspend_until, + job.worker, + job.extras as _, + &job.tag, + job.priority + ) + .execute(&mut *tx) + .await?; + + if job.ping.is_some() || job.memory_peak.is_some() { + sqlx::query!( + r#" + INSERT INTO v2_job_runtime (id, ping, memory_peak) + VALUES ($1, $2, $3) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.ping, + job.memory_peak + ) + .execute(&mut *tx) + .await?; + } + + if job.flow_status.is_some() + || job.flow_leaf_jobs.is_some() + || job.workflow_as_code_status.is_some() + { + sqlx::query!( + r#" + INSERT INTO v2_job_status (id, flow_status, flow_leaf_jobs, workflow_as_code_status) + VALUES ($1, $2, $3, $4) + ON CONFLICT (id) DO NOTHING + "#, + job.id, + job.flow_status as _, + job.flow_leaf_jobs as _, + job.workflow_as_code_status as _ + ) + .execute(&mut *tx) + .await?; + } + + if let Some(ref concurrency_key) = job.concurrency_key { + sqlx::query!( + r#" + WITH inserted_concurrency_counter AS ( + INSERT INTO concurrency_counter (concurrency_id, job_uuids) + VALUES ($1, '{}'::jsonb) + ON CONFLICT DO NOTHING + ) + INSERT INTO concurrency_key(key, job_id) + VALUES ($1, $2) + ON CONFLICT (job_id) DO NOTHING + "#, + concurrency_key, + job.id + ) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + Ok(format!("Successfully imported jobs")) +} + +pub async fn delete_jobs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(job_ids): Json>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + + if job_ids.is_empty() { + return Ok("No jobs to delete".to_string()); + } + + let mut tx = user_db.begin(&authed).await?; + + let logs_deleted = sqlx::query!( + "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let perms_deleted = sqlx::query!( + "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let stats_deleted = sqlx::query!( + "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids) + .execute(&mut *tx) + .await? + .rows_affected(); + + let concurrency_key_deleted = sqlx::query!( + "DELETE FROM concurrency_key WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let queue_deleted = sqlx::query!( + "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let completed_deleted = sqlx::query!( + "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let zombie_deleted = sqlx::query!( + "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + let jobs_deleted = sqlx::query!( + "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", + &w_id, + &job_ids + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + tx.commit().await?; + + let total_rows_deleted = logs_deleted + + perms_deleted + + stats_deleted + + resume_deleted + + runtime_deleted + + status_deleted + + concurrency_key_deleted + + queue_deleted + + completed_deleted + + zombie_deleted + + jobs_deleted; + + tracing::info!( + "Successfully deleted {} jobs ({} total rows across all tables) from workspace {}", + job_ids.len(), + total_rows_deleted, + w_id + ); + + Ok(format!( + "Successfully deleted {} jobs ({} total rows across all tables)", + jobs_deleted, total_rows_deleted + )) +} diff --git a/backend/windmill-api-jobs/src/lib.rs b/backend/windmill-api-jobs/src/lib.rs new file mode 100644 index 0000000000..7479ec1496 --- /dev/null +++ b/backend/windmill-api-jobs/src/lib.rs @@ -0,0 +1,18 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod concurrency_groups; +pub mod execution; +pub mod job_metrics; +pub mod jobs_export; +pub mod query; +pub mod types; + +pub use execution::*; +pub use query::*; +pub use types::*; diff --git a/backend/windmill-api-jobs/src/query.rs b/backend/windmill-api-jobs/src/query.rs new file mode 100644 index 0000000000..a863a8157a --- /dev/null +++ b/backend/windmill-api-jobs/src/query.rs @@ -0,0 +1,373 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Query builders for filtering job lists (queue and completed). + +use sql_builder::prelude::*; +use sql_builder::SqlBuilder; +use windmill_common::utils::{paginate_without_limits, Pagination}; + +use crate::types::{ListCompletedQuery, ListQueueQuery}; + +pub fn filter_list_queue_query( + mut sqlb: SqlBuilder, + lq: &ListQueueQuery, + w_id: &str, + join_outstanding_wait_times: bool, +) -> SqlBuilder { + sqlb.join("v2_job").on_eq("v2_job_queue.id", "v2_job.id"); + + if join_outstanding_wait_times { + sqlb.left() + .join("outstanding_wait_time") + .on_eq("v2_job_queue.id", "outstanding_wait_time.job_id"); + } + + if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { + sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id)); + } + + if let Some(w) = &lq.worker { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job_queue.worker", w.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job_queue.worker", "?".bind(w)); + } + } + + if let Some(ps) = &lq.script_path_start { + sqlb.and_where_like_left("runnable_path", ps); + } + if let Some(p) = &lq.script_path_exact { + sqlb.and_where_eq("runnable_path", "?".bind(p)); + } + if let Some(p) = &lq.schedule_path { + sqlb.and_where_eq("trigger", "?".bind(p)); + sqlb.and_where_eq("trigger_kind", "'schedule'"); + } + if let Some(h) = &lq.script_hash { + sqlb.and_where_eq("runnable_id", "?".bind(h)); + } + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(t) = &lq.tag { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job.tag", "?".bind(t)); + } + } + + if let Some(r) = &lq.running { + sqlb.and_where_eq("running", &r); + } + if let Some(pj) = &lq.parent_job { + sqlb.and_where_eq("parent_job", "?".bind(pj)); + } + if let Some(dt) = &lq.started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.started_after { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(fs) = &lq.is_flow_step { + if *fs { + sqlb.and_where_is_not_null("flow_step_id"); + } else { + sqlb.and_where_is_null("flow_step_id"); + } + } + if let Some(fs) = &lq.has_null_parent { + if *fs { + sqlb.and_where_is_null("parent_job"); + } + } + + if let Some(dt) = &lq.created_before { + sqlb.and_where_le("v2_job.created_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_after { + sqlb.and_where_ge("v2_job.created_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_after { + let ts = dt.timestamp_millis(); + sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at >= to_timestamp({} / 1000.0))", ts, ts)); + } + + if let Some(dt) = &lq.created_or_started_before { + let ts = dt.timestamp_millis(); + sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at < to_timestamp({} / 1000.0))", ts, ts)); + } + + if let Some(s) = &lq.suspended { + if *s { + sqlb.and_where_gt("suspend", 0); + } else { + sqlb.and_where_is_null("suspend_until"); + } + } + + if let Some(jk) = &lq.job_kinds { + sqlb.and_where_in( + "kind", + &jk.split(',').into_iter().map(quote).collect::>(), + ); + } + + if let Some(args) = &lq.args { + sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + } + + if lq.scheduled_for_before_now.is_some_and(|x| x) { + sqlb.and_where_le("scheduled_for", "now()"); + } + + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); + } + + if let Some(tk) = &lq.trigger_kind { + sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); + } + + if let Some(tp) = &lq.trigger_path { + sqlb.and_where_eq("trigger", "?".bind(tp)); + } + + sqlb +} + +pub fn list_queue_jobs_query( + w_id: &str, + lq: &ListQueueQuery, + fields: &[&str], + pagination: Pagination, + join_outstanding_wait_times: bool, + tags: Option>, +) -> SqlBuilder { + let (limit, offset) = paginate_without_limits(pagination); + let mut sqlb = SqlBuilder::select_from("v2_job_queue") + .fields(fields) + .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) + .limit(limit) + .offset(offset) + .clone(); + + if let Some(tags) = tags { + sqlb.and_where_in( + "v2_job.tag", + &tags.iter().map(|x| quote(x)).collect::>(), + ); + } + + filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times) +} + +pub fn filter_list_completed_query( + mut sqlb: SqlBuilder, + lq: &ListCompletedQuery, + w_id: &str, + join_outstanding_wait_times: bool, +) -> SqlBuilder { + sqlb.join("v2_job") + .on_eq("v2_job_completed.id", "v2_job.id"); + + if join_outstanding_wait_times { + sqlb.left() + .join("outstanding_wait_time") + .on_eq("v2_job_completed.id", "outstanding_wait_time.job_id"); + } + + if let Some(label) = &lq.label { + if lq.allow_wildcards.unwrap_or(false) { + let wh = format!( + "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", + &label.replace("*", "%").replace("'", "''") + ); + sqlb.and_where("result ? 'wm_labels'"); + sqlb.and_where(&wh); + } else { + let mut wh = format!("result->'wm_labels' ? "); + wh.push_str(&format!("'{}'", &label.replace("'", "''"))); + sqlb.and_where("result ? 'wm_labels'"); + sqlb.and_where(&wh); + } + } + + if let Some(worker) = &lq.worker { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job_completed.worker", worker.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job_completed.worker", "?".bind(worker)); + } + } + + if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { + sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)) + .and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); + } + + if let Some(p) = &lq.schedule_path { + sqlb.and_where_eq("trigger", "?".bind(p)); + sqlb.and_where_eq("trigger_kind", "'schedule'"); + } + + if let Some(ps) = &lq.script_path_start { + sqlb.and_where_like_left("runnable_path", ps); + } + if let Some(p) = &lq.script_path_exact { + sqlb.and_where_eq("runnable_path", "?".bind(p)); + } + if let Some(h) = &lq.script_hash { + sqlb.and_where_eq("runnable_id", "?".bind(h)); + } + if let Some(t) = &lq.tag { + if lq.allow_wildcards.unwrap_or(false) { + sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); + } else { + sqlb.and_where_eq("v2_job.tag", "?".bind(t)); + } + } + + if let Some(cb) = &lq.created_by { + sqlb.and_where_eq("created_by", "?".bind(cb)); + } + if let Some(r) = &lq.success { + if *r { + sqlb.and_where_eq("status", "'success'") + .or_where_eq("status", "'skipped'"); + } else { + sqlb.and_where_eq("status", "'failure'") + .or_where_eq("status", "'canceled'"); + } + } + if let Some(pj) = &lq.parent_job { + sqlb.and_where_eq("parent_job", "?".bind(pj)); + } + if let Some(dt) = &lq.started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.started_after { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_before { + sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_or_started_after { + let ts = dt.to_rfc3339(); + sqlb.and_where(format!( + "(created_at >= '{}' OR started_at >= '{}')", + ts.replace("'", "''"), + ts.replace("'", "''") + )); + } + + if let Some(dt) = &lq.created_before { + sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.created_after { + sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.created_or_started_after_completed_jobs { + sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(dt) = &lq.completed_after { + sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339())); + } + if let Some(dt) = &lq.completed_before { + sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339())); + } + + if let Some(sk) = &lq.is_skipped { + if *sk { + sqlb.and_where_eq("status", "'skipped'"); + } else { + sqlb.and_where_ne("status", "'skipped'"); + } + } + if let Some(fs) = &lq.is_flow_step { + if *fs { + sqlb.and_where_is_not_null("flow_step_id"); + } else { + sqlb.and_where_is_null("flow_step_id"); + } + } + if let Some(fs) = &lq.has_null_parent { + if *fs { + sqlb.and_where_is_null("parent_job"); + } + } + if let Some(jk) = &lq.job_kinds { + sqlb.and_where_in( + "kind", + &jk.split(',').into_iter().map(quote).collect::>(), + ); + } + + if let Some(args) = &lq.args { + sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); + } + + if let Some(result) = &lq.result { + sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); + } + + if lq.is_not_schedule.unwrap_or(false) { + sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); + } + + if let Some(tk) = &lq.trigger_kind { + sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); + } + + if let Some(tp) = &lq.trigger_path { + sqlb.and_where_eq("trigger", "?".bind(tp)); + } + + sqlb +} + +pub fn list_completed_jobs_query( + w_id: &str, + per_page: Option, + offset: usize, + lq: &ListCompletedQuery, + fields: &[&str], + join_outstanding_wait_times: bool, + tags: Option>, +) -> SqlBuilder { + let mut sqlb = SqlBuilder::select_from("v2_job_completed") + .fields(fields) + .order_by( + if lq.completed_before.is_some() || lq.completed_after.is_some() { + "v2_job_completed.completed_at" + } else { + "v2_job.created_at" + }, + lq.order_desc.unwrap_or(true), + ) + .offset(offset) + .clone(); + if let Some(per_page) = per_page { + sqlb.limit(per_page); + } + + if let Some(tags) = tags { + sqlb.and_where_in( + "v2_job.tag", + &tags.iter().map(|x| quote(x)).collect::>(), + ); + } + + filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times) +} diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs new file mode 100644 index 0000000000..d9717b75a0 --- /dev/null +++ b/backend/windmill-api-jobs/src/types.rs @@ -0,0 +1,570 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Shared types for the jobs subsystem. + +use axum::{ + extract::{FromRequest, Json, Query, Request}, + response::{IntoResponse, Response}, +}; +use http::StatusCode; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::{ + error, + jobs::{CompletedJob, JobKind, JobTriggerKind, QueuedJob}, + scripts::{ScriptHash, ScriptLang}, + utils::now_from_db, + DB, +}; + +use windmill_api_sse::{Job, JobExtended}; + +// ------------ RunJobQuery ------------ + +#[derive(Debug, Deserialize, Clone, Default)] +pub struct RunJobQuery { + pub scheduled_for: Option>, + pub scheduled_in_secs: Option, + pub parent_job: Option, + pub root_job: Option, + pub invisible_to_owner: Option, + pub queue_limit: Option, + pub payload: Option, + pub job_id: Option, + pub tag: Option, + pub timeout: Option, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub skip_preprocessor: Option, + pub poll_delay_ms: Option, + pub memory_id: Option, + pub trigger_external_id: Option, + pub service_name: Option, + pub suspended_mode: Option, +} + +impl RunJobQuery { + pub async fn get_scheduled_for( + &self, + db: &DB, + ) -> error::Result>> { + if let Some(scheduled_for) = self.scheduled_for { + Ok(Some(scheduled_for)) + } else if let Some(scheduled_in_secs) = self.scheduled_in_secs { + let now = now_from_db(db).await?; + Ok(Some( + now + chrono::Duration::try_seconds(scheduled_in_secs).unwrap_or_default(), + )) + } else { + Ok(None) + } + } + + pub fn payload_as_args(&self) -> error::Result>> { + let payload_r = self.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| { + error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) + }) + }); + + let payload_as_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + Ok(payload_as_args) + } +} + +// ------------ List query types ------------ + +#[derive(Deserialize, Clone)] +pub struct ListQueueQuery { + pub script_path_start: Option, + pub script_path_exact: Option, + pub script_hash: Option, + pub created_by: Option, + pub started_before: Option>, + pub started_after: Option>, + pub created_before: Option>, + pub created_after: Option>, + pub created_or_started_before: Option>, + pub created_or_started_after: Option>, + pub running: Option, + pub schedule_path: Option, + pub parent_job: Option, + pub order_desc: Option, + pub job_kinds: Option, + pub suspended: Option, + pub worker: Option, + // filter by matching a subset of the args using base64 encoded json subset + pub args: Option, + pub tag: Option, + pub scheduled_for_before_now: Option, + pub all_workspaces: Option, + pub is_flow_step: Option, + pub has_null_parent: Option, + pub is_not_schedule: Option, + pub concurrency_key: Option, + pub allow_wildcards: Option, + pub trigger_kind: Option, + pub trigger_path: Option, + pub include_args: Option, +} + +#[derive(Deserialize, Clone)] +pub struct ListCompletedQuery { + pub script_path_start: Option, + pub script_path_exact: Option, + pub script_hash: Option, + pub created_by: Option, + pub started_before: Option>, + pub started_after: Option>, + pub created_before: Option>, + pub created_after: Option>, + pub created_or_started_before: Option>, + pub created_or_started_after: Option>, + pub created_or_started_after_completed_jobs: Option>, + pub created_before_queue: Option>, + pub created_after_queue: Option>, + pub completed_after: Option>, + pub completed_before: Option>, + pub success: Option, + pub running: Option, + pub parent_job: Option, + pub order_desc: Option, + pub job_kinds: Option, + pub is_skipped: Option, + pub is_flow_step: Option, + pub suspended: Option, + pub schedule_path: Option, + // filter by matching a subset of the args using base64 encoded json subset + pub args: Option, + // filter by matching a subset of the result using base64 encoded json subset + pub result: Option, + pub tag: Option, + pub scheduled_for_before_now: Option, + pub all_workspaces: Option, + pub has_null_parent: Option, + pub label: Option, + pub is_not_schedule: Option, + pub concurrency_key: Option, + pub worker: Option, + pub allow_wildcards: Option, + pub trigger_kind: Option, + pub trigger_path: Option, + pub include_args: Option, +} + +impl From for ListQueueQuery { + fn from(lcq: ListCompletedQuery) -> Self { + Self { + script_path_start: lcq.script_path_start, + script_path_exact: lcq.script_path_exact, + script_hash: lcq.script_hash, + created_by: lcq.created_by, + started_before: lcq.started_before, + started_after: lcq.started_after, + created_before: lcq.created_before_queue.or(lcq.created_before), + created_after: lcq.created_after_queue.or(lcq.created_after), + created_or_started_before: lcq.created_or_started_before, + created_or_started_after: lcq.created_or_started_after, + worker: lcq.worker, + running: lcq.running, + parent_job: lcq.parent_job, + order_desc: lcq.order_desc, + job_kinds: lcq.job_kinds, + suspended: lcq.suspended, + args: lcq.args, + tag: lcq.tag, + schedule_path: lcq.schedule_path, + scheduled_for_before_now: lcq.scheduled_for_before_now, + all_workspaces: lcq.all_workspaces, + is_flow_step: lcq.is_flow_step, + has_null_parent: lcq.has_null_parent, + is_not_schedule: lcq.is_not_schedule, + concurrency_key: lcq.concurrency_key, + allow_wildcards: lcq.allow_wildcards, + trigger_kind: lcq.trigger_kind, + trigger_path: lcq.trigger_path, + include_args: lcq.include_args, + } + } +} + +// ------------ ListableCompletedJob ------------ + +#[derive(Debug, sqlx::FromRow, Serialize)] +pub struct ListableCompletedJob { + pub r#type: String, + pub workspace_id: String, + pub id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub duration_ms: i64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub script_path: Option, + pub deleted: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_code: Option, + pub canceled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub canceled_reason: Option, + pub job_kind: JobKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule_path: Option, + pub permissioned_as: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_flow: Option, + pub is_flow_step: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + pub tag: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub args: Option, +} + +// ------------ UnifiedJob ------------ + +#[derive(sqlx::FromRow)] +pub struct UnifiedJob { + pub workspace_id: String, + pub typ: String, + pub id: Uuid, + pub parent_job: Option, + pub created_by: String, + pub created_at: chrono::DateTime, + pub started_at: Option>, + pub completed_at: Option>, + pub scheduled_for: Option>, + pub running: Option, + pub script_hash: Option, + pub script_path: Option, + pub args: Option, + pub duration_ms: Option, + pub success: Option, + pub deleted: bool, + pub canceled: bool, + pub canceled_by: Option, + pub job_kind: JobKind, + pub schedule_path: Option, + pub permissioned_as: String, + pub is_flow_step: bool, + pub language: Option, + pub is_skipped: bool, + pub email: String, + pub visible_to_owner: bool, + pub suspend: Option, + pub mem_peak: Option, + pub tag: String, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub priority: Option, + pub labels: Option, + pub self_wait_time_ms: Option, + pub aggregate_wait_time_ms: Option, + pub preprocessed: Option, + pub worker: Option, + pub runnable_settings_handle: Option, +} + +const CJ_FIELDS: &[&str] = &[ + "'CompletedJob' as typ", + "v2_job_completed.id", + "v2_job_completed.workspace_id", + "v2_job.parent_job", + "v2_job.created_by", + "v2_job.created_at", + "v2_job_completed.started_at", + "null as scheduled_for", + "v2_job_completed.completed_at", + "null as running", + "v2_job.runnable_id as script_hash", + "v2_job.runnable_path as script_path", + "null as args", + "v2_job_completed.duration_ms", + "v2_job_completed.status = 'success' OR v2_job_completed.status = 'skipped' as success", + "false as deleted", + "v2_job_completed.status = 'canceled' as canceled", + "v2_job_completed.canceled_by", + "v2_job.kind as job_kind", + "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", + "v2_job.permissioned_as", + "v2_job.flow_step_id IS NOT NULL as is_flow_step", + "v2_job.script_lang as language", + "v2_job_completed.status = 'skipped' as is_skipped", + "v2_job.permissioned_as_email as email", + "v2_job.visible_to_owner", + "null as suspend", + "v2_job_completed.memory_peak as mem_peak", + "v2_job.tag", + "null as concurrent_limit", + "null as concurrency_time_window_s", + "v2_job.priority", + "v2_job_completed.result->'wm_labels' as labels", + "self_wait_time_ms", + "aggregate_wait_time_ms", + "v2_job.preprocessed", + "v2_job_completed.worker", + "null as runnable_settings_handle", +]; + +const QJ_FIELDS: &[&str] = &[ + "'QueuedJob' as typ", + "v2_job_queue.id", + "v2_job_queue.workspace_id", + "v2_job.parent_job", + "v2_job.created_by", + "v2_job_queue.created_at", + "v2_job_queue.started_at", + "v2_job_queue.scheduled_for", + "null as completed_at", + "v2_job_queue.running", + "v2_job.runnable_id as script_hash", + "v2_job.runnable_path as script_path", + "null as args", + "null as duration_ms", + "null as success", + "false as deleted", + "v2_job_queue.canceled_by IS NOT NULL as canceled", + "v2_job_queue.canceled_by", + "v2_job.kind as job_kind", + "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", + "v2_job.permissioned_as", + "v2_job.flow_step_id IS NOT NULL as is_flow_step", + "v2_job.script_lang as language", + "false as is_skipped", + "v2_job.permissioned_as_email as email", + "v2_job.visible_to_owner", + "v2_job_queue.suspend", + "null as mem_peak", + "v2_job.tag", + "v2_job.concurrent_limit", + "v2_job.concurrency_time_window_s", + "v2_job.priority", + "null as labels", + "self_wait_time_ms", + "aggregate_wait_time_ms", + "v2_job.preprocessed", + "v2_job_queue.worker", + "v2_job_queue.runnable_settings_handle", +]; + +impl UnifiedJob { + pub fn completed_job_fields() -> &'static [&'static str] { + CJ_FIELDS + } + pub fn queued_job_fields() -> &'static [&'static str] { + QJ_FIELDS + } +} + +impl From for Job { + fn from(uj: UnifiedJob) -> Self { + let args = uj.args.and_then(|v| serde_json::from_value(v).ok()); + match uj.typ.as_ref() { + "CompletedJob" => Job::CompletedJob(JobExtended::new( + uj.self_wait_time_ms, + uj.aggregate_wait_time_ms, + CompletedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at, + completed_at: uj.completed_at, + duration_ms: uj.duration_ms.unwrap(), + success: uj.success.unwrap(), + script_hash: uj.script_hash, + script_path: uj.script_path, + args: args.clone(), + result: None, + result_columns: None, + logs: None, + flow_status: None, + workflow_as_code_status: None, + deleted: uj.deleted, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + canceled_reason: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + is_flow_step: uj.is_flow_step, + language: uj.language, + is_skipped: uj.is_skipped, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + mem_peak: uj.mem_peak, + tag: uj.tag, + priority: uj.priority, + labels: uj.labels, + preprocessed: uj.preprocessed, + }, + )), + "QueuedJob" => Job::QueuedJob(JobExtended::new( + uj.self_wait_time_ms, + uj.aggregate_wait_time_ms, + QueuedJob { + workspace_id: uj.workspace_id, + id: uj.id, + parent_job: uj.parent_job, + created_by: uj.created_by, + created_at: uj.created_at, + started_at: uj.started_at, + scheduled_for: uj.scheduled_for.unwrap(), + running: uj.running.unwrap(), + script_hash: uj.script_hash, + script_path: uj.script_path, + script_entrypoint_override: None, + args, + logs: None, + canceled: uj.canceled, + canceled_by: uj.canceled_by, + canceled_reason: None, + last_ping: None, + job_kind: uj.job_kind, + schedule_path: uj.schedule_path, + permissioned_as: uj.permissioned_as, + flow_status: None, + workflow_as_code_status: None, + is_flow_step: uj.is_flow_step, + language: uj.language, + same_worker: false, + pre_run_error: None, + email: uj.email, + visible_to_owner: uj.visible_to_owner, + suspend: uj.suspend, + mem_peak: uj.mem_peak, + root_job: None, + leaf_jobs: None, + tag: uj.tag, + concurrent_limit: uj.concurrent_limit, + concurrency_time_window_s: uj.concurrency_time_window_s, + timeout: None, + flow_step_id: None, + cache_ttl: None, + cache_ignore_s3_path: None, + priority: uj.priority, + preprocessed: uj.preprocessed, + runnable_settings_handle: uj.runnable_settings_handle, + }, + )), + t => panic!("job type {} not valid", t), + } + } +} + +// ------------ Approval types ------------ + +#[derive(Deserialize, Debug)] +pub struct QueryApprover { + pub approver: Option, + /// If true, generate/verify resume URLs for the parent flow instead of the specific step. + /// This allows pre-approvals that can be consumed by any later suspend step in the same flow. + pub flow_level: Option, +} + +#[allow(non_snake_case)] +#[derive(Serialize, Debug)] +pub struct ResumeUrls { + pub approvalPage: String, + pub cancel: String, + pub resume: String, +} + +// ------------ QueryOrBody extractor ------------ + +pub struct QueryOrBody(pub Option); + +#[axum::async_trait] +impl FromRequest for QueryOrBody +where + D: DeserializeOwned, + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request( + req: Request, + state: &S, + ) -> std::result::Result { + return if req.method() == axum::http::Method::GET { + let Query(InPayload { payload }) = Query::from_request(req, state) + .await + .map_err(IntoResponse::into_response)?; + payload + .map(|p| { + decode_payload(p) + .map(QueryOrBody) + .map_err(|err| (StatusCode::BAD_REQUEST, format!("{err:#?}"))) + .map_err(IntoResponse::into_response) + }) + .unwrap_or(Ok(QueryOrBody(None))) + } else { + Json::from_request(req, state) + .await + .map(|Json(v)| QueryOrBody(Some(v))) + .map_err(IntoResponse::into_response) + }; + + #[derive(Deserialize)] + struct InPayload { + payload: Option, + } + } +} + +// ------------ Utility functions ------------ + +pub fn decode_payload(t: String) -> anyhow::Result { + let vec = base64::engine::general_purpose::STANDARD + .decode(t) + .context("invalid base64")?; + serde_json::from_slice(vec.as_slice()).context("invalid json") +} + +pub fn add_raw_string( + raw_string: Option, + mut args: serde_json::Map, +) -> serde_json::Map { + if let Some(raw_string) = raw_string { + args.insert( + "raw_string".to_string(), + serde_json::Value::String(raw_string), + ); + } + return args; +} + +use anyhow::Context; +use base64::Engine; diff --git a/backend/windmill-api-sse/Cargo.toml b/backend/windmill-api-sse/Cargo.toml new file mode 100644 index 0000000000..b8127c3d36 --- /dev/null +++ b/backend/windmill-api-sse/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-api-sse" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_sse" +path = "src/lib.rs" + +[features] +default = [] +enterprise = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api-sse/src/lib.rs b/backend/windmill-api-sse/src/lib.rs new file mode 100644 index 0000000000..1db4aa9433 --- /dev/null +++ b/backend/windmill-api-sse/src/lib.rs @@ -0,0 +1,341 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Shared SSE types for job update streaming. +//! +//! This crate contains the types used by the SSE job update system, +//! separated out so they can be depended on without pulling in the +//! full windmill-api-jobs crate. + +use serde::Serialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::{Deref, DerefMut}; +use uuid::Uuid; +use windmill_common::flow_status::FlowStatus; +use windmill_common::jobs::{CompletedJob, JobKind, QueuedJob}; + +// ------------ Job types (from jobs.rs) ------------ + +#[derive(sqlx::FromRow, Debug, Serialize)] +pub struct JobExtended { + #[sqlx(flatten)] + #[serde(flatten)] + pub inner: T, + + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_code: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_lock: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_flow: Option>>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker: Option, + + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub self_wait_time_ms: Option, + #[sqlx(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub aggregate_wait_time_ms: Option, +} + +pub trait JobCommon { + fn job_kind(&self) -> &JobKind; +} + +impl JobCommon for QueuedJob { + fn job_kind(&self) -> &JobKind { + &self.job_kind + } +} + +impl JobCommon for CompletedJob { + fn job_kind(&self) -> &JobKind { + &self.job_kind + } +} + +impl JobExtended { + pub fn new( + self_wait_time_ms: Option, + aggregate_wait_time_ms: Option, + inner: T, + ) -> Self { + Self { + inner, + raw_code: None, + raw_lock: None, + raw_flow: None, + worker: None, + self_wait_time_ms, + aggregate_wait_time_ms, + } + } +} + +impl Deref for JobExtended { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for JobExtended { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +#[derive(Serialize, Debug)] +#[serde(tag = "type")] +pub enum Job { + QueuedJob(JobExtended), + CompletedJob(JobExtended), +} + +impl Job { + pub fn created_by(&self) -> &str { + match self { + Job::QueuedJob(job) => &job.created_by, + Job::CompletedJob(job) => &job.created_by, + } + } + + pub fn append_to_logs(&mut self, logs: &str) { + match self { + Job::QueuedJob(job) => { + if let Some(ref mut l) = job.logs { + l.push_str(logs); + } else { + job.logs = Some(logs.to_string()); + } + } + Job::CompletedJob(job) => { + if let Some(ref mut l) = job.logs { + l.push_str(logs); + } else { + job.logs = Some(logs.to_string()); + } + } + } + } + + pub fn log_len(&self) -> Option { + match self { + Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()), + Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()), + } + } + + pub fn logs(&self) -> Option { + match self { + Job::QueuedJob(job) => job.logs.clone(), + Job::CompletedJob(job) => job.logs.clone(), + } + } + + pub fn flow_status(&self) -> Option { + match self { + Job::QueuedJob(job) => job + .flow_status + .as_ref() + .and_then(|rf| serde_json::from_str(rf.0.get()).ok()), + Job::CompletedJob(job) => job + .flow_status + .as_ref() + .and_then(|rf| serde_json::from_str(rf.0.get()).ok()), + } + } + + pub fn is_flow_step(&self) -> bool { + match self { + Job::QueuedJob(job) => job.is_flow_step, + Job::CompletedJob(job) => job.is_flow_step, + } + } + + pub fn is_flow(&self) -> bool { + self.job_kind().is_flow() + } + + pub fn job_kind(&self) -> &JobKind { + match self { + Job::QueuedJob(job) => &job.job_kind, + Job::CompletedJob(job) => &job.job_kind, + } + } + + pub fn id(&self) -> Uuid { + match self { + Job::QueuedJob(job) => job.id, + Job::CompletedJob(job) => job.id, + } + } + + pub fn workspace_id(&self) -> &String { + match self { + Job::QueuedJob(job) => &job.workspace_id, + Job::CompletedJob(job) => &job.workspace_id, + } + } + + pub fn script_path(&self) -> &str { + match self { + Job::QueuedJob(job) => job.script_path.as_ref(), + Job::CompletedJob(job) => job.script_path.as_ref(), + } + .map(String::as_str) + .unwrap_or("tmp/main") + } + + pub fn args(&self) -> Option<&sqlx::types::Json>>> { + match self { + Job::QueuedJob(job) => job.args.as_ref(), + Job::CompletedJob(job) => job.args.as_ref(), + } + } + + pub fn full_path_with_workspace(&self) -> String { + format!( + "{}/{}/{}", + self.workspace_id(), + if self.is_flow() { "flow" } else { "script" }, + self.script_path(), + ) + } + + pub async fn concurrency_key( + &self, + db: &sqlx::Pool, + ) -> Result, sqlx::Error> { + sqlx::query_scalar!( + "SELECT key FROM concurrency_key WHERE job_id = $1", + self.id() + ) + .fetch_optional(db) + .await + } + + pub async fn fetch_outstanding_wait_time( + &mut self, + db: &sqlx::Pool, + ) -> Result<(), sqlx::Error> { + let r = sqlx::query!( + "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", + self.id() + ) + .fetch_optional(db) + .await?; + + let (self_wait_time, aggregate_wait_time) = r + .map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms)) + .unwrap_or((None, None)); + + match self { + Job::QueuedJob(job) => { + job.self_wait_time_ms = self_wait_time; + job.aggregate_wait_time_ms = aggregate_wait_time; + } + Job::CompletedJob(job) => { + job.self_wait_time_ms = self_wait_time; + job.aggregate_wait_time_ms = aggregate_wait_time; + } + } + Ok(()) + } +} + +// ------------ SSE types ------------ + +#[derive(Serialize, Debug)] +pub struct JobUpdate { + #[serde(skip_serializing_if = "Option::is_none")] + pub running: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub completed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_logs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_result_stream: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub log_offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub progress: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_status: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub workflow_as_code_status: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub job: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub only_result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_stream_job_id: Option, +} + +impl JobUpdate { + pub fn hash_str(&self) -> String { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + format!("{:x}", hasher.finish()) + } +} + +impl Hash for JobUpdate { + fn hash(&self, state: &mut H) { + self.running.hash(state); + self.completed.hash(state); + self.log_offset.hash(state); + self.mem_peak.hash(state); + self.progress.hash(state); + self.stream_offset.hash(state); + self.flow_stream_job_id.hash(state); + if !self.completed.unwrap_or(false) { + self.flow_status.as_ref().map(|x| x.get().hash(state)); + self.workflow_as_code_status + .as_ref() + .map(|x| x.get().hash(state)); + } + } +} + +#[derive(serde::Deserialize)] +pub struct JobUpdateQuery { + pub running: Option, + pub log_offset: Option, + pub stream_offset: Option, + pub get_progress: Option, + pub no_logs: Option, + pub only_result: Option, + pub fast: Option, + pub is_flow: Option, + pub poll_delay_ms: Option, +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum JobUpdateSSEStream { + Update(JobUpdate), + Error { error: String }, + NotFound, + Timeout, + Ping, +} + +lazy_static::lazy_static! { + pub static ref TIMEOUT_SSE_STREAM: u64 = + std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::().unwrap_or(60); +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index b33326a015..1a3b6ee1f3 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,41 +10,46 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-email?/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise"] stripe = [] agent_worker_server = [] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] -embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] +embedding = ["windmill-api-embeddings/embedding"] parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] +openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] -kafka = ["dep:rdkafka", "dep:rdkafka-sys"] -kafka-gssapi = ["kafka", "rdkafka/gssapi"] -nats = ["dep:async-nats", "dep:nkeys"] -websocket = ["dep:tokio-tungstenite"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"] +kafka = ["dep:windmill-trigger-kafka", "windmill-trigger-kafka/enterprise", "windmill-store/kafka"] +kafka-gssapi = ["kafka", "windmill-trigger-kafka/kafka-gssapi"] +nats = ["dep:windmill-trigger-nats", "windmill-trigger-nats/enterprise", "windmill-store/nats"] +websocket = ["dep:windmill-trigger-websocket", "windmill-trigger-websocket/enterprise"] +smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "dep:windmill-trigger-email", "windmill-trigger-email/enterprise"] license = ["dep:rsa"] zip = ["dep:async_zip"] -oauth2 = ["dep:windmill-oauth"] -http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq"] +oauth2 = ["dep:windmill-oauth", "windmill-store/oauth2"] +http_trigger = ["dep:matchit", "dep:windmill-trigger-http", "windmill-trigger-http/enterprise", "windmill-store/http_trigger"] static_frontend = ["dep:rust-embed"] -postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"] -mqtt_trigger = ["dep:thiserror", "dep:rumqttc"] -native_trigger = ["dep:strum", "dep:backon", "oauth2"] -sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config"] +postgres_trigger = ["dep:windmill-trigger-postgres", "windmill-trigger-postgres/enterprise", "windmill-store/postgres_trigger"] +mqtt_trigger = ["dep:windmill-trigger-mqtt", "windmill-trigger-mqtt/enterprise", "windmill-store/mqtt_trigger"] +native_trigger = ["dep:windmill-native-triggers", "windmill-native-triggers/native_trigger", "windmill-native-triggers/enterprise", "dep:strum", "oauth2"] +sqs_trigger = ["dep:windmill-trigger-sqs", "windmill-trigger-sqs/enterprise", "windmill-store/sqs_trigger"] deno_core = ["dep:deno_core", "dep:deno_error"] -gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] -cloud = ["windmill-common/cloud"] -mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] +gcp_trigger = ["dep:windmill-trigger-gcp", "windmill-trigger-gcp/enterprise", "windmill-store/gcp_trigger"] +cloud = ["windmill-common/cloud", "windmill-api-auth/cloud", "windmill-store/cloud"] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth", "windmill-api-auth/mcp", "windmill-store/mcp"] bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"] python = [] -no_auth = [] +no_auth = ["windmill-api-auth/no_auth", "windmill-store/no_auth"] [dependencies] windmill-mcp = { workspace = true, optional = true } +windmill-api-auth.workspace = true +windmill-api-sse.workspace = true +windmill-api-jobs.workspace = true +windmill-trigger.workspace = true +windmill-store.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true @@ -104,8 +109,6 @@ postgres-native-tls.workspace = true hmac.workspace = true cookie.workspace = true sha2.workspace = true -sha1 = { workspace = true, optional = true } -constant_time_eq = { workspace = true, optional = true } urlencoding.workspace = true lazy_static.workspace = true prometheus = { workspace = true, optional = true } @@ -118,40 +121,28 @@ libxml = { workspace = true, optional = true } async-recursion.workspace = true rsa = { workspace = true, optional = true} uuid.workspace = true -tinyvector = { workspace = true, optional = true} -hf-hub = { workspace = true, optional = true} -tokenizers = { workspace = true, optional = true} -candle-core = { workspace = true, optional = true} -candle-transformers = { workspace = true, optional = true} -candle-nn = { workspace = true, optional = true} datafusion = { workspace = true, optional = true} object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit = { workspace = true, optional = true } -tokio-tungstenite = { workspace = true, optional = true} -rdkafka = { workspace = true, optional = true } -rdkafka-sys = { workspace = true, optional = true } - -async-nats = { workspace = true, optional = true } -nkeys = { workspace = true, optional = true } +windmill-trigger-kafka = { workspace = true, optional = true } +windmill-trigger-postgres = { workspace = true, optional = true } +windmill-trigger-mqtt = { workspace = true, optional = true } +windmill-trigger-websocket = { workspace = true, optional = true } +windmill-trigger-email = { workspace = true, optional = true } +windmill-trigger-nats = { workspace = true, optional = true } +windmill-trigger-sqs = { workspace = true, optional = true } +windmill-trigger-gcp = { workspace = true, optional = true } +windmill-trigger-http = { workspace = true, optional = true } +windmill-native-triggers = { workspace = true, optional = true } +windmill-api-embeddings.workspace = true const_format.workspace = true pin-project.workspace = true http.workspace = true indexmap.workspace = true async-stream.workspace = true ulid.workspace = true -rust-postgres = { workspace = true, optional = true } -pg_escape = { workspace = true, optional = true } -byteorder = { workspace = true, optional = true } -thiserror = { workspace = true, optional = true } -rust_decimal = { workspace = true, optional = true } -rust-postgres-native-tls = { workspace = true, optional = true} -rumqttc = { workspace = true, optional = true } -aws-sdk-sqs = { workspace = true, optional = true } -aws-sdk-sso = { workspace = true, optional = true } -aws-sdk-ssooidc = { workspace = true, optional = true } -aws-sdk-sts = { workspace = true, optional = true } rustls = { workspace = true } aws-sigv4.workspace = true @@ -162,14 +153,10 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types.workspace = true async-trait.workspace = true -google-cloud-pubsub = { workspace = true, optional = true } -google-cloud-googleapis = { workspace = true , optional = true } -tonic = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } tar.workspace = true flate2.workspace = true -backon = {workspace = true, optional = true} strum = { workspace = true, optional = true } dashmap.workspace = true diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 9d2d6d9db3..6e9352e8ee 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1411,7 +1411,7 @@ async fn delete_app( deployed_object, Some(format!("App '{}' deleted", path)), true, - None, + None, ) .await?; diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 9383063919..144703ba5c 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,766 +1,4 @@ -#[cfg(feature = "enterprise")] -use crate::ee_oss::ExternalJwks; -use axum::{ - async_trait, - extract::{FromRequestParts, OriginalUri, Query}, - Extension, Json, +pub use windmill_api_auth::auth::{ + invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache, + ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail, }; -use chrono::TimeZone; -use http::{request::Parts, StatusCode}; -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use tower_cookies::Cookies; -use tracing::Span; - -use crate::db::{ApiAuthed, OptJobAuthed, DB}; -use std::{ - str::FromStr, - sync::{ - atomic::{AtomicI64, AtomicU64, Ordering}, - Arc, - }, -}; -#[cfg(feature = "enterprise")] -use tokio::sync::RwLock; - -use windmill_common::{ - auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, - error::{Error, JsonResult}, - jwt, - users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, -}; - -lazy_static::lazy_static! { - // Global auth cache accessible from main.rs for direct invalidation - pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); - -} -// Global function to invalidate a specific token from cache -pub fn invalidate_token_from_cache(token: &str) { - // Remove all cache entries for this token (across all workspaces) - AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); - tracing::info!( - "Invalidated token from auth cache: {}...", - &token[..token.len().min(8)] - ); -} - -#[derive(Clone)] -pub struct ExpiringAuthCache { - pub authed: ApiAuthed, - pub expiry: chrono::DateTime, - pub job_id: Option, -} - -pub struct AuthCache { - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] - ext_jwks: Option>>, -} - -impl AuthCache { - pub fn new( - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] ext_jwks: Option>>, - ) -> Self { - AuthCache { - db, - superadmin_secret, - #[cfg(feature = "enterprise")] - ext_jwks, - } - } - - pub async fn invalidate(&self, w_id: &str, token: String) { - AUTH_CACHE.remove(&(w_id.to_string(), token)); - } - - pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { - Some(self.get_opt_job_authed(w_id, token).await?.authed) - } - - pub async fn get_opt_job_authed( - &self, - w_id: Option, - token: &str, - ) -> Option { - let key = ( - w_id.as_ref().unwrap_or(&"".to_string()).to_string(), - token.to_string(), - ); - let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); - match s { - Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => { - Some(OptJobAuthed { authed, job_id }) - } - #[cfg(feature = "enterprise")] - _ if token.starts_with("jwt_ext_") => { - let authed_and_exp = match crate::ee_oss::jwt_ext_auth( - w_id.as_ref(), - token.trim_start_matches("jwt_ext_"), - self.ext_jwks.clone(), - &self.db, - ) - .await - { - Ok(r) => Some(r), - Err(e) => { - tracing::error!("JWT_EXT auth error: {:?}", e); - None - } - }; - - if let Some((authed, exp, job_id)) = authed_and_exp.clone() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } else { - None - } - } - _ if token.starts_with("jwt_") => { - let jwt_token = token.trim_start_matches("jwt_"); - - let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; - - match jwt_result { - Ok(claims) => { - if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { - tracing::error!("JWT auth error: workspace_id mismatch"); - return None; - } - let username_override = username_override_from_label(claims.label); - - let authed = crate::db::ApiAuthed { - email: claims.email, - username: claims.username, - is_admin: claims.is_admin, - is_operator: claims.is_operator, - groups: claims.groups, - folders: claims.folders, - scopes: None, - username_override, - token_prefix: claims.audit_span, - }; - let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc - .timestamp_nanos(claims.exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } - Err(err) => { - tracing::error!("JWT auth error: {:?}", err); - None - } - } - } - _ => { - let user_o = sqlx::query!( - "UPDATE token SET last_used_at = now() WHERE - token = $1 - AND (expiration > NOW() OR expiration IS NULL) - AND (workspace_id IS NULL OR workspace_id = $2) - RETURNING owner, email, super_admin, scopes, label", - token, - w_id.as_ref(), - ) - .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) - .fetch_optional(&self.db) - .await - .ok() - .flatten(); - - if let Some(user) = user_o { - let authed_o = { - match user { - (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { - let username_override = username_override_from_label(label); - if let Some((prefix, name)) = owner.split_once('/') { - if prefix == "u" { - let (is_admin, is_operator) = if super_admin { - (true, false) - } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id.as_ref().unwrap() - ) - .fetch_one(&self.db) - .await - .ok(); - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - (false, true) - } - }; - - let w_id = &w_id.unwrap(); - let groups = - get_groups_for_user(w_id, &name, &email, &self.db) - .await - .ok() - .unwrap_or_default(); - - let folders = - get_folders_for_user(w_id, &name, &groups, &self.db) - .await - .ok() - .unwrap_or_default(); - - Some(ApiAuthed { - email: email, - username: name.to_string(), - is_admin, - is_operator, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } else { - let groups = vec![name.to_string()]; - let folders = get_folders_for_user( - &w_id.unwrap(), - "", - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email: email, - username: format!("group-{name}"), - is_admin: false, - groups, - is_operator: false, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - } else { - let groups = vec![]; - let folders = vec![]; - Some(ApiAuthed { - email: email, - username: owner, - is_admin: super_admin, - is_operator: true, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - (_, Some(email), super_admin, scopes, label) => { - let username_override = username_override_from_label(label); - if w_id.is_some() { - let row_o = sqlx::query!( - "SELECT username, is_admin, operator FROM usr WHERE - email = $1 AND workspace_id = $2 AND disabled = false", - &email, - w_id.as_ref().unwrap() - ) - .map(|x| (x.username, x.is_admin, x.operator)) - .fetch_optional(&self.db) - .await - .unwrap_or(Some(("error".to_string(), false, false))); - - match row_o { - Some((username, is_admin, is_operator)) => { - let groups = get_groups_for_user( - &w_id.as_ref().unwrap(), - &username, - &email, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - - let folders = get_folders_for_user( - &w_id.unwrap(), - &username, - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email, - username, - is_admin: is_admin || super_admin, - is_operator, - groups, - folders, - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - None if super_admin => Some(ApiAuthed { - email: email.clone(), - username: email, - is_admin: super_admin, - is_operator: false, - groups: vec![], - folders: vec![], - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }), - None => None, - } - } else { - Some(ApiAuthed { - email: email.to_string(), - username: email, - is_admin: super_admin, - is_operator: true, - groups: Vec::new(), - folders: Vec::new(), - scopes, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - _ => None, - } - }; - if let Some(authed) = authed_o.as_ref() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc::now() - + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - } - authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) - } else if self - .superadmin_secret - .as_ref() - .map(|x| x == token) - .unwrap_or(false) - { - let authed = ApiAuthed { - email: SUPERADMIN_SECRET_EMAIL.to_string(), - username: "superadmin_secret".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }; - Some(OptJobAuthed { authed, job_id: None }) - } else { - None - } - } - } - } -} - -async fn extract_token(parts: &mut Parts, state: &S) -> Option { - let auth_header = parts - .headers - .get(http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|s| s.strip_prefix("Bearer ")); - - let from_cookie = match auth_header { - Some(x) => Some(x.to_owned()), - None => Extension::::from_request_parts(parts, state) - .await - .ok() - .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), - }; - - #[derive(Deserialize)] - struct Token { - token: Option, - } - match from_cookie { - Some(token) => Some(token), - None => Query::::from_request_parts(parts, state) - .await - .ok() - .and_then(|token| token.token.clone()), - } -} - -#[derive(Clone, Debug)] -pub struct Tokened { - pub token: String, -} - -#[derive(Clone, Debug)] -pub struct OptTokened { - #[allow(dead_code)] - pub token: Option, -} - -struct BruteForceCounter { - counter: AtomicU64, - last_reset: AtomicI64, -} - -lazy_static::lazy_static! { - static ref BRUTE_FORCE_COUNTER: BruteForceCounter = - BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; -} - -impl BruteForceCounter { - async fn increment(&self) { - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { - tracing::error!( - "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" - ); - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - if now - self.last_reset.load(Ordering::Relaxed) > 60 { - self.counter.store(0, Ordering::Relaxed); - self.last_reset.store(now, Ordering::Relaxed); - } - } -} - -#[async_trait] -impl FromRequestParts for Tokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(Tokened { token: "".to_string() }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(tokened.clone()) - } else { - let token_o = extract_token(parts, state).await; - if let Some(token) = token_o { - let tokened = Self { token }; - parts.extensions.insert(tokened.clone()); - Ok(tokened) - } else { - BRUTE_FORCE_COUNTER.increment().await; - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) - } - } - } -} - -#[async_trait] -impl FromRequestParts for OptTokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptTokened { token: None }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(OptTokened { token: Some(tokened.token.clone()) }) - } else { - let token_o = extract_token(parts, state).await; - Ok(OptTokened { token: token_o }) - } - } -} - -pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { - if let Some(scopes) = scopes { - for scope in scopes.iter_mut() { - if scope.starts_with("run:") { - let (_, part_scope) = scope.split_once(":").unwrap(); - - if let Some((kind, path)) = part_scope.split_once("/") { - //appending a 's' as runnable kind is singular while new scope format expect it to be plural - *scope = format!("jobs:run:{}s:{}", kind, path); - } - } else if scope.starts_with("jobs:") { - // Map old jobs scopes to new format - let new_scope = match scope.as_str() { - "jobs:listjobs" => "jobs:read", - "jobs:runscript" => "jobs:run:scripts", - "jobs:runflow" => "jobs:run:flows", - "jobs:resumeflow" => "jobs:run:flows", - "jobs:deletejob" => "jobs:write", - _ => continue, - }; - - *scope = new_scope.to_string(); - } - } - } -} - -fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { - let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { - Some(path_vec[3].to_owned()) - } else if path_vec.len() >= 5 - && path_vec[0] == "" - && path_vec[1] == "api" - && path_vec[2] == "mcp" - && path_vec[3] == "w" - { - Some(path_vec[4].to_owned()) - } else { - if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { - Some(path_vec[4].to_owned()) - } else { - None - } - }; - - workspace_id -} - -#[async_trait] -impl FromRequestParts for ApiAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - let opt_job_authed = OptJobAuthed::from_request_parts(parts, state).await?; - Ok(opt_job_authed.authed) - } -} - -#[async_trait] -impl FromRequestParts for OptJobAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptJobAuthed::default()); - }; - - #[cfg(feature = "no_auth")] - { - let authed = ApiAuthed { - email: "admin@windmill.dev".to_string(), - username: "admin".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: None, - }; - return Ok(OptJobAuthed { authed, job_id: None }); - } - - let already_authed = parts.extensions.get::(); - - if let Some(authed) = already_authed { - return Ok(authed.clone()); - } - - let already_tokened = parts.extensions.get::(); - let token_o = if let Some(token) = already_tokened { - Some(token.token.clone()) - } else { - extract_token(parts, state).await - }; - if let Some(token) = token_o { - if let Ok(Extension(cache)) = - Extension::>::from_request_parts(parts, state).await - { - let original_uri = OriginalUri::from_request_parts(parts, state) - .await - .ok() - .map(|x| x.0) - .unwrap_or_default(); - let path_vec: Vec<&str> = original_uri.path().split("/").collect(); - let workspace_id = maybe_get_workspace_id_from_path(&path_vec); - - if let Some(mut opt_job_authed) = - cache.get_opt_job_authed(workspace_id.clone(), &token).await - { - let authed = &mut opt_job_authed.authed; - if authed.scopes.is_some() { - transform_old_scope_to_new_scope(authed.scopes.as_mut()); - - let path = original_uri.path(); - let method = parts.method.as_str(); - - if let Err(err) = crate::scopes::check_scopes_for_route( - authed.scopes.as_deref(), - path, - method, - ) { - BRUTE_FORCE_COUNTER.increment().await; - return Err(err); - } - } - parts.extensions.insert(authed.clone()); - - Span::current().record("username", &authed.username.as_str()); - Span::current().record("email", &authed.email); - - if let Some(workspace_id) = workspace_id { - Span::current().record("workspace_id", &workspace_id); - } - return Ok(opt_job_authed); - } - } - } - BRUTE_FORCE_COUNTER.increment().await; - Err(Error::NotAuthorized("Unauthorized".to_string())) - } -} - -fn username_override_from_label(label: Option) -> Option { - match label { - Some(label) - if label.starts_with("webhook-") - || label.starts_with("http-") - || label.starts_with("email-") - || label.starts_with("ws-") => - { - Some(label) - } - Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( - label - .trim_start_matches("ephemeral-script-end-user-") - .to_string(), - ), - Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), - Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { - Some(format!("label-{label}")) - } - _ => None, - } -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedTokenWithEmail { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, - pub scopes: Option>, - pub email: Option, -} - -pub async fn list_tokens_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult> { - let tokens = if is_flow { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:flows:' || $2]::text[] - OR scopes @> ARRAY['run:flow/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - } else { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] - OR scopes @> ARRAY['run:script/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - }; - - Ok(Json(tokens)) -} diff --git a/backend/windmill-api/src/bedrock.rs b/backend/windmill-api/src/bedrock.rs index 4ed1f2cdfa..de0e9059d7 100644 --- a/backend/windmill-api/src/bedrock.rs +++ b/backend/windmill-api/src/bedrock.rs @@ -18,14 +18,16 @@ use axum::body::Bytes; use serde::Deserialize; +use windmill_common::ai_bedrock::build_tool_config; use windmill_common::ai_bedrock::{ bedrock_stream_event_is_block_stop, bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_start, format_bedrock_error, - BedrockClient + BedrockClient, +}; +use windmill_common::ai_types::{ + OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction, }; use windmill_common::error::{Error, Result}; -use windmill_common::ai_types::{OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, ToolDefFunction}; -use windmill_common::ai_bedrock::build_tool_config; // ============================================================================ // Shared Request Types for SDK-Based Handlers @@ -69,10 +71,7 @@ struct OpenAIToolFunction { /// Authentication configuration for Bedrock clients enum BedrockAuthConfig { BearerToken(String), - IamCredentials { - access_key_id: String, - secret_access_key: String, - }, + IamCredentials { access_key_id: String, secret_access_key: String }, Environment, } @@ -105,13 +104,8 @@ async fn create_bedrock_client( region: &str, ) -> Result { match determine_auth_config(api_key, aws_access_key_id, aws_secret_access_key) { - BedrockAuthConfig::BearerToken(key) => { - BedrockClient::from_bearer_token(key, region).await - } - BedrockAuthConfig::IamCredentials { - access_key_id, - secret_access_key, - } => { + BedrockAuthConfig::BearerToken(key) => BedrockClient::from_bearer_token(key, region).await, + BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => { BedrockClient::from_credentials(access_key_id, secret_access_key, None, region).await } BedrockAuthConfig::Environment => BedrockClient::from_env(region).await, @@ -185,10 +179,7 @@ async fn create_bedrock_control_client( .build(); Ok(aws_sdk_bedrock::Client::from_conf(config)) } - BedrockAuthConfig::IamCredentials { - access_key_id, - secret_access_key, - } => { + BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key } => { let credentials = aws_credential_types::Credentials::new( access_key_id, secret_access_key, diff --git a/backend/windmill-api/src/concurrency_groups.rs b/backend/windmill-api/src/concurrency_groups.rs index 25768c3c10..e00b4324a8 100644 --- a/backend/windmill-api/src/concurrency_groups.rs +++ b/backend/windmill-api/src/concurrency_groups.rs @@ -1,364 +1 @@ -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ - filter_list_completed_query, filter_list_queue_query, Job, ListCompletedQuery, - ListQueueQuery, UnifiedJob, - }, - utils::check_scopes, -}; -use axum::extract::Path; -use axum::routing::{delete, get}; -use axum::{extract::Query, Extension, Json}; -use serde::Deserialize; - -use axum::Router; - -use serde::Serialize; -use sql_builder::bind::Bind; -use sql_builder::SqlBuilder; -use uuid::Uuid; -use windmill_common::db::UserDB; -use windmill_common::error::Error::PermissionDenied; -use windmill_common::error::{self, JsonResult}; -use windmill_common::utils::require_admin; - -pub fn global_service() -> Router { - Router::new() - .route("/list", get(list_concurrency_groups)) - .route("/prune/*concurrency_key", delete(prune_concurrency_group)) - .route("/:job_id/key", get(get_concurrency_key)) -} - -pub fn workspaced_service() -> Router { - Router::new().route("/list_jobs", get(get_concurrent_intervals)) -} - -#[derive(Serialize)] -pub struct ConcurrencyGroups { - concurrency_key: String, - total_running: i64, -} - -async fn list_concurrency_groups( - authed: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let concurrency_counts = sqlx::query_as::<_, (String, i64)>( - "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter", - ).fetch_all(&db) - .await?; - - let mut concurrency_groups: Vec = vec![]; - for (concurrency_key, count) in concurrency_counts { - concurrency_groups.push(ConcurrencyGroups { - concurrency_key: concurrency_key.clone(), - total_running: count, - }) - } - - return Ok(Json(concurrency_groups)); -} - -async fn prune_concurrency_group( - authed: ApiAuthed, - Extension(db): Extension, - Path(concurrency_key): Path, -) -> JsonResult<()> { - if !authed.is_admin { - return Err(PermissionDenied( - "Only administrators can delete concurrency groups".to_string(), - )); - } - let mut tx = db.begin().await?; - - let concurrency_group = sqlx::query_as::<_, (String, i64)>( - "SELECT concurrency_id, (select COUNT(*) from jsonb_object_keys(job_uuids)) as n_job_uuids FROM concurrency_counter WHERE concurrency_id = $1 FOR UPDATE", - ) - .bind(concurrency_key.clone()) - .fetch_optional(&mut *tx) - .await?; - - let n_job_uuids = concurrency_group.map(|cg| cg.1).unwrap_or_default(); - - if n_job_uuids > 0 { - tx.commit().await?; - return Err(error::Error::internal_err( - "Concurrency group is currently in use, unable to remove it. Retry later.".to_string(), - )); - } - - sqlx::query!( - "DELETE FROM concurrency_counter WHERE concurrency_id = $1", - concurrency_key.clone(), - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "DELETE FROM concurrency_key WHERE key = $1", - concurrency_key.clone(), - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok(Json(())) -} - -#[derive(Serialize)] -struct ExtendedJobs { - jobs: Vec, - obscured_jobs: Vec, - omitted_obscured_jobs: bool, -} - -#[derive(Serialize)] -struct ObscuredJob { - typ: String, - started_at: Option>, - duration_ms: Option, -} -#[derive(Deserialize)] -struct ExtendedJobsParams { - row_limit: Option, -} - -pub fn join_concurrency_key<'c>( - concurrency_key: Option<&String>, - mut sqlb: SqlBuilder, -) -> SqlBuilder { - if let Some(key) = concurrency_key { - sqlb.join("concurrency_key") - .on_eq("id", "concurrency_key.job_id") - .and_where_eq("key", "?".bind(key)); - } - - sqlb -} - -async fn get_concurrent_intervals( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Query(iq): Query, - Query(lq): Query, -) -> JsonResult { - check_scopes(&authed, || format!("jobs:read"))?; - - if lq.success.is_some() && lq.running.is_some_and(|x| x) { - return Err(error::Error::BadRequest( - "cannot specify both success and running".to_string(), - )); - } - - let row_limit = iq.row_limit.unwrap_or(1000); - - let lq = ListCompletedQuery { order_desc: Some(true), ..lq }; - let lqc = lq.clone(); - let lqq: ListQueueQuery = lqc.into(); - let mut sqlb_q = SqlBuilder::select_from("v2_job_queue") - .fields(UnifiedJob::queued_job_fields()) - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_c = SqlBuilder::select_from("v2_job_completed") - .fields(UnifiedJob::completed_job_fields()) - .order_by("completed_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_q_user = SqlBuilder::select_from("v2_job_queue") - .fields(&["id"]) - .order_by("created_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - let mut sqlb_c_user = SqlBuilder::select_from("v2_job_completed") - .fields(&["id"]) - .order_by("completed_at", lq.order_desc.unwrap_or(true)) - .limit(row_limit) - .clone(); - - sqlb_q = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q); - sqlb_c = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c); - sqlb_q_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_q_user); - sqlb_c_user = join_concurrency_key(lq.concurrency_key.as_ref(), sqlb_c_user); - - let should_fetch_obscured_jobs = match lq { - ListCompletedQuery { - script_path_start: None, - script_path_exact: None, - script_hash: None, - created_by: None, - success: None, - running: None, - parent_job: None, - is_skipped: None | Some(false), - suspended: None, - schedule_path: None, - args: None, - result: None, - tag: None, - has_null_parent: None, - worker: None, - label: None, - trigger_path: None, - scheduled_for_before_now: _, - is_not_schedule: _, - started_before: _, - started_after: _, - created_before: _, - created_after: _, - created_before_queue: _, - created_after_queue: _, - completed_after: _, - completed_before: _, - created_or_started_before: _, - created_or_started_after: _, - created_or_started_after_completed_jobs: _, - order_desc: _, - job_kinds: _, - is_flow_step: _, - all_workspaces: _, - concurrency_key: Some(_), - allow_wildcards: None, - trigger_kind: _, - include_args: _, - } => true, - _ => false, - }; - - // When we have a concurrency key defined, fetch jobs from other workspaces - // as obscured unless we're in the admins workspace. This is to show the - // potential concurrency races without showing jobs that don't belong to - // the workspace. - // To avoid infering information through filtering, don't return obscured - // jobs if the filters are too specific - if should_fetch_obscured_jobs && w_id != "admins" { - // Get the obscured jobs from all workspaces (concurrency key could be global) - let (sqlb_q, sqlb_c) = ( - filter_list_queue_query( - sqlb_q, - &ListQueueQuery { all_workspaces: Some(true), ..lqq.clone() }, - "admins", - true, - ), - filter_list_completed_query( - sqlb_c, - &ListCompletedQuery { all_workspaces: Some(true), ..lq.clone() }, - "admins", - true, - ), - ); - - sqlb_q_user = filter_list_queue_query(sqlb_q_user, &lqq, w_id.as_str(), true); - sqlb_c_user = filter_list_completed_query(sqlb_c_user, &lq, w_id.as_str(), true); - - let sql_q_user = sqlb_q_user.query()?; - let sql_c_user = sqlb_c_user.query()?; - let sql_q = sqlb_q.query()?; - let sql_c = sqlb_c.query()?; - - // This first transaction uses the user_db to know which uuids are - // accessible to the user. - let mut tx = user_db.begin(&authed).await?; - let running_jobs_user: Vec = if lq.success.is_none() { - sqlx::query_scalar(&sql_q_user).fetch_all(&mut *tx).await? - } else { - vec![] - }; - let completed_jobs_user: Vec = if lq.running.is_none() { - sqlx::query_scalar(&sql_c_user).fetch_all(&mut *tx).await? - } else { - vec![] - }; - tx.commit().await?; - - // This second transaction uses the db, so it will fetch information - // potentially forbidden to the user. It must be obscured before - // returning it - let running_jobs_db: Vec = if lq.success.is_none() { - sqlx::query_as(&sql_q).fetch_all(&db).await? - } else { - vec![] - }; - let completed_jobs_db: Vec = if lq.running.is_none() { - sqlx::query_as(&sql_c).fetch_all(&db).await? - } else { - vec![] - }; - - let obscured_jobs = running_jobs_db - .iter() - .filter(|j| !running_jobs_user.iter().any(|id| j.id == *id)) - .chain( - completed_jobs_db - .iter() - .filter(|j| !completed_jobs_user.iter().any(|id| j.id == *id)), - ) - .map(|j| ObscuredJob { - typ: j.typ.clone(), - started_at: j.started_at, - duration_ms: j.duration_ms, - }) - .collect(); - - let jobs = running_jobs_db - .into_iter() - .filter(|j| running_jobs_user.iter().any(|id| j.id == *id)) - .chain( - completed_jobs_db - .into_iter() - .filter(|j| completed_jobs_user.iter().any(|id| j.id == *id)), - ) - .map(From::from) - .collect(); - - Ok(Json(ExtendedJobs { - jobs, - obscured_jobs, - omitted_obscured_jobs: !should_fetch_obscured_jobs, - })) - } else { - sqlb_q = filter_list_queue_query(sqlb_q, &lqq, w_id.as_str(), true); - sqlb_c = filter_list_completed_query(sqlb_c, &lq, w_id.as_str(), true); - let sql_q = sqlb_q.query()?; - let sql_c = sqlb_c.query()?; - - let mut tx = user_db.begin(&authed).await?; - let running_jobs: Vec = if lq.success.is_none() { - sqlx::query_as(&sql_q).fetch_all(&mut *tx).await? - } else { - vec![] - }; - let completed_jobs: Vec = if lq.running.is_none() { - sqlx::query_as(&sql_c).fetch_all(&mut *tx).await? - } else { - vec![] - }; - tx.commit().await?; - - let jobs = running_jobs - .into_iter() - .chain(completed_jobs.into_iter()) - .map(From::from) - .collect(); - - Ok(Json(ExtendedJobs { - jobs, - obscured_jobs: vec![], - omitted_obscured_jobs: !should_fetch_obscured_jobs, - })) - } -} - -async fn get_concurrency_key( - Extension(db): Extension, - Path(job_id): Path, -) -> JsonResult> { - let key = sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id) - .fetch_optional(&db) - .await?; - Ok(Json(key)) -} +pub use windmill_api_jobs::concurrency_groups::*; diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 9157c3cb01..cd84b12d81 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -14,13 +14,10 @@ use sqlx::{ }; use tokio::task::JoinHandle; -use windmill_audit::audit_oss::AuditAuthorable; pub use windmill_common::db::DB; -use windmill_common::{ - db::{Authable, Authed, AuthedRef}, - error::Error, - utils::generate_lock_id, -}; +use windmill_common::{error::Error, utils::generate_lock_id}; + +pub use windmill_api_auth::{ApiAuthed, OptJobAuthed}; async fn current_database(conn: &mut PgConnection) -> Result { // language=SQL @@ -291,120 +288,3 @@ pub async fn migrate( crate::live_migrations::custom_migrations(&mut custom_migrator, db).await?; Ok(None) } - -#[derive(Default, Clone, Debug)] -pub struct OptJobAuthed { - pub job_id: Option, - pub authed: ApiAuthed, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] -pub struct ApiAuthed { - pub email: String, - pub username: String, - pub is_admin: bool, - pub is_operator: bool, - pub groups: Vec, - // (folder name, can write, is owner) - pub folders: Vec<(String, bool, bool)>, - pub scopes: Option>, - pub username_override: Option, - pub token_prefix: Option, -} - -impl ApiAuthed { - pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { - AuthedRef { - email: &self.email, - username: &self.username, - is_admin: &self.is_admin, - is_operator: &self.is_operator, - groups: &self.groups, - folders: &self.folders, - scopes: &self.scopes, - token_prefix: &self.token_prefix, - } - } -} - -impl From for Authed { - fn from(value: ApiAuthed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - token_prefix: value.token_prefix, - } - } -} - -impl From for ApiAuthed { - fn from(value: Authed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - username_override: None, // Authed doesn't have this field, so default to None - token_prefix: value.token_prefix, - } - } -} - -impl ApiAuthed { - pub fn display_username(&self) -> &str { - self.username_override.as_ref().unwrap_or(&self.username) - } -} - -impl AuditAuthorable for ApiAuthed { - fn username(&self) -> &str { - self.username.as_str() - } - fn email(&self) -> &str { - self.email.as_str() - } - fn username_override(&self) -> Option<&str> { - self.username_override.as_deref() - } - fn token_prefix(&self) -> Option<&str> { - self.token_prefix.as_deref() - } -} - -impl Authable for ApiAuthed { - fn is_admin(&self) -> bool { - self.is_admin - } - - fn is_operator(&self) -> bool { - self.is_operator - } - - fn groups(&self) -> &[String] { - &self.groups - } - - fn folders(&self) -> &[(String, bool, bool)] { - &self.folders - } - - fn scopes(&self) -> Option<&[std::string::String]> { - self.scopes.as_ref().map(|x| x.as_slice()) - } - - fn email(&self) -> &str { - &self.email - } - - fn username(&self) -> &str { - &self.username - } -} diff --git a/backend/windmill-api/src/debug.rs b/backend/windmill-api/src/debug.rs index fc5e8084c5..bdf769833d 100644 --- a/backend/windmill-api/src/debug.rs +++ b/backend/windmill-api/src/debug.rs @@ -21,22 +21,23 @@ //! - A completed job entry in v2_job_completed //! - An audit log entry identical to script preview runs -use axum::{extract::Path, routing::{get, post}, Extension, Json, Router}; +use axum::{ + extract::Path, + routing::{get, post}, + Extension, Json, Router, +}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use chrono::Utc; -use ed25519_dalek::{SigningKey, Signer}; +use ed25519_dalek::{Signer, SigningKey}; use serde::{Deserialize, Serialize}; -use sha2::{Sha256, Digest}; +use sha2::{Digest, Sha256}; use sqlx::types::Json as SqlxJson; use std::sync::Arc; use tokio::sync::RwLock; use uuid::Uuid; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ - db::UserDB, - error::JsonResult, - jobs::JobKind, - scripts::ScriptLang, + db::UserDB, error::JsonResult, jobs::JobKind, scripts::ScriptLang, users::username_to_permissioned_as, }; @@ -279,7 +280,8 @@ async fn sign_debug_request( job_id, w_id, now, - SqlxJson(serde_json::json!({"debug_session": true, "language": request.language})) as SqlxJson, + SqlxJson(serde_json::json!({"debug_session": true, "language": request.language})) + as SqlxJson, ) .execute(&mut *tx) .await?; @@ -400,10 +402,13 @@ async fn sign_expression( ActionKind::Execute, &w_id, Some(&resource), - Some([ - ("job_id", request.job_id.as_str()), - ("expression", request.expression.as_str()), - ].into()), + Some( + [ + ("job_id", request.job_id.as_str()), + ("expression", request.expression.as_str()), + ] + .into(), + ), ) .await?; diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index 6e54b45dc5..a1f7a54d20 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -1,17 +1,11 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::ee::*; -#[cfg(all( - feature = "enterprise", - any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), - not(feature = "private") -))] -use {crate::db::ApiAuthed, windmill_common::DB}; +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub use windmill_api_auth::ee_oss::ExternalJwks; #[cfg(not(feature = "private"))] use anyhow::anyhow; -#[cfg(all(feature = "enterprise", not(feature = "private")))] -use {std::sync::Arc, tokio::sync::RwLock}; #[cfg(not(feature = "private"))] pub async fn validate_license_key( _license_key: String, @@ -21,42 +15,9 @@ pub async fn validate_license_key( Err(anyhow!("License can't be validated in Windmill CE")) } -#[cfg(all(feature = "enterprise", not(feature = "private")))] -pub async fn jwt_ext_auth( - _w_id: Option<&String>, - _token: &str, - _external_jwks: Option>>, - _db: &crate::db::DB, -) -> anyhow::Result<(crate::db::ApiAuthed, usize, Option)> { - // Implementation is not open source - - Err(anyhow!("External JWT auth is not open source")) -} - -#[cfg(all(feature = "enterprise", not(feature = "private")))] -pub struct ExternalJwks; - -#[cfg(all(feature = "enterprise", not(feature = "private")))] -impl ExternalJwks { - pub async fn load() -> Option>> { - // Implementation is not open source - None - } -} - +// interpolate moved to windmill-store/src/resources.rs #[cfg(all( feature = "enterprise", - any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), - not(feature = "private") + any(feature = "nats", feature = "kafka", feature = "sqs_trigger") ))] -pub async fn interpolate( - _authed: &ApiAuthed, - _db: &DB, - _w_id: &str, - _s: String, -) -> Result { - // Implementation is not open source - Err(anyhow!( - "Interpolation is not available in open source version" - )) -} +pub use windmill_store::resources::interpolate; diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 875bcf683a..6212d38e31 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -1,630 +1 @@ -#[cfg(feature = "embedding")] -use anyhow::{anyhow, Error, Result}; -#[cfg(feature = "embedding")] -use std::{collections::HashMap, path::PathBuf, sync::Arc}; -#[cfg(feature = "embedding")] -use windmill_common::DEFAULT_HUB_BASE_URL; -#[cfg(feature = "embedding")] -use windmill_common::HUB_BASE_URL; - -use axum::Router; - -#[cfg(feature = "embedding")] -use axum::{ - extract::{Path, Query}, - Json, -}; - -#[cfg(feature = "embedding")] -use axum::routing::get; -#[cfg(feature = "embedding")] -use candle_core::{Device, Tensor}; -#[cfg(feature = "embedding")] -use candle_nn::VarBuilder; -#[cfg(feature = "embedding")] -use candle_transformers::models::bert::{BertModel, Config, DTYPE}; -#[cfg(feature = "embedding")] -use hf_hub::api::tokio::Api; -#[cfg(feature = "embedding")] -use serde::Deserialize; -#[cfg(feature = "embedding")] -use serde::Serialize; -#[cfg(feature = "embedding")] -use sqlx::{Pool, Postgres}; -#[cfg(feature = "embedding")] -use tinyvector::{ - db::{Db, Embedding}, - similarity::Distance, -}; -#[cfg(feature = "embedding")] -use tokenizers::Tokenizer; -#[cfg(feature = "embedding")] -use tokio::sync::RwLock; -#[cfg(feature = "embedding")] -use windmill_common::utils::http_get_from_hub; - -#[cfg(feature = "embedding")] -use windmill_common::error::JsonResult; - -#[cfg(feature = "embedding")] -use crate::{resources::ResourceType, HTTP_CLIENT}; - -#[cfg(feature = "embedding")] -lazy_static::lazy_static! { - pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); - pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); - pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize)] -struct HubScriptsQuery { - text: String, - limit: Option, - kind: Option, - app: Option, -} - -#[cfg(feature = "embedding")] -#[derive(Serialize)] -pub struct HubScriptResult { - ask_id: i64, - id: i64, - version_id: i64, - summary: String, - app: String, - kind: String, - score: f32, -} - -#[cfg(feature = "embedding")] -async fn query_hub_scripts( - Query(query): Query, -) -> JsonResult> { - let embeddings_db = EMBEDDINGS_DB.read().await; - - if let Some(embeddings_db) = embeddings_db.as_ref() { - let results = embeddings_db - .query_hub_scripts(&query.text, query.limit, query.kind, query.app) - .await?; - - Ok(Json(results)) - } else { - Err(windmill_common::error::Error::internal_err( - "Embeddings db not initialized".to_string(), - )) - } -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize)] -struct ResourceTypesQuery { - text: String, - limit: Option, -} - -#[cfg(feature = "embedding")] -#[derive(Serialize)] -pub struct ResourceTypeResult { - name: String, - score: f32, - schema: Option, -} -#[cfg(feature = "embedding")] -async fn query_resource_types( - Query(query): Query, - Path(w_id): Path, -) -> JsonResult> { - let embeddings_db = EMBEDDINGS_DB.read().await; - - if let Some(embeddings_db) = embeddings_db.as_ref() { - let results = embeddings_db - .query_resource_types(w_id, &query.text, query.limit) - .await?; - - Ok(Json(results)) - } else { - Err(windmill_common::error::Error::internal_err( - "Embeddings db not initialized".to_string(), - )) - } -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize, Debug, Clone)] -struct HubScript { - ask_id: i64, - id: i64, - version_id: i64, - summary: String, - app: String, - kind: String, - embedding: Vec, -} - -#[cfg(feature = "embedding")] -#[derive(Deserialize, Debug)] -struct HubResourceType { - name: String, - embedding: Vec, -} - -#[cfg(feature = "embedding")] -pub struct ModelInstance { - model: BertModel, - tokenizer: Tokenizer, -} - -#[cfg(feature = "embedding")] -impl ModelInstance { - pub async fn load_model_files() -> Result<(PathBuf, PathBuf, PathBuf)> { - let api = Api::new()?; - let repo_api = api.model("thenlper/gte-small".to_string()); - - let (config_filename, tokenizer_filename, weights_filename) = - ( - repo_api - .get("config.json") - .await - .map_err(|e| anyhow!("Failed to get config.json from hugging face: {}", e))?, - repo_api.get("tokenizer.json").await.map_err(|e| { - anyhow!("Failed to get tokenizer.json from hugging face: {}", e) - })?, - repo_api.get("model.safetensors").await.map_err(|e| { - anyhow!("Failed to get model.safetensors from hugging face: {}", e) - })?, - ); - - Ok((config_filename, tokenizer_filename, weights_filename)) - } - - pub async fn new() -> Result { - tracing::info!("Loading embedding model..."); - let device = Device::Cpu; - let (config_filename, tokenizer_filename, weights_filename) = - Self::load_model_files().await?; - let config = std::fs::read_to_string(config_filename)?; - let config: Config = serde_json::from_str(&config)?; - let tokenizer = Tokenizer::from( - Tokenizer::from_file(tokenizer_filename) - .map_err(Error::msg)? - .with_padding(None) - .to_owned(), - ); - - let vb = - unsafe { VarBuilder::from_mmaped_safetensors(&[weights_filename], DTYPE, &device)? }; - let model = BertModel::load(vb, &config)?; - tracing::info!("Loaded embedding model"); - Ok(Self { model, tokenizer }) - } - - pub async fn create_embedding(self: Arc, sentence: &str) -> Result> { - let sentence = sentence.to_owned(); - tokio::task::spawn_blocking(move || { - let tokens = self - .tokenizer - .encode(sentence, true) - .map_err(Error::msg)? - .get_ids() - .to_vec(); - - let token_ids = Tensor::new(&tokens[..], &Device::Cpu)?.unsqueeze(0)?; - let token_type_ids = token_ids.zeros_like()?; - - let embedding = self.model.forward(&token_ids, &token_type_ids, None)?; - let embedding = (embedding.sum(1)? / embedding.dim(1)? as f64)?; - let embedding = normalize_l2(&embedding)?; - - let embedding = embedding.get(0)?.to_vec1()?; - - Ok(embedding) - }) - .await? - } -} - -#[cfg(feature = "embedding")] -pub struct EmbeddingsDb { - db: Db, - model_instance: Arc, -} - -#[cfg(feature = "embedding")] -impl EmbeddingsDb { - pub async fn new(pg_db: &Pool, model_instance: Arc) -> Result { - let db = Db::new(); - - let mut embeddings_db = Self { db, model_instance: model_instance.clone() }; - - embeddings_db.fill_db(pg_db).await?; - - Ok(embeddings_db) - } - - async fn fill_db(&mut self, pg_db: &Pool) -> Result<()> { - if self.db.get_collection("scripts").is_some() { - self.db.delete_collection("scripts")?; - } - - self.db - .create_collection("scripts".to_string(), 384, Distance::Cosine)?; - - if self.db.get_collection("resource_types").is_some() { - self.db.delete_collection("resource_types")?; - } - - self.db - .create_collection("resource_types".to_string(), 384, Distance::Cosine)?; - - let hub_base_url = HUB_BASE_URL.read().await.clone(); - - let response = match hub_base_url.as_str() { - DEFAULT_HUB_BASE_URL => { - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/scripts_embeddings.json") - .send() - .await; - - if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { - tracing::warn!("Failed to get scripts embeddings from bucket, trying hub..."); - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } else { - response.unwrap() - } - } - _ => { - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/scripts/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } - }; - - if response.error_for_status_ref().is_err() { - return Err(anyhow!( - "Failed to get scripts embeddings from hub with error code: {}", - response.status() - )); - } - - let hub_scripts = response.json::>().await?; - - for script in &hub_scripts { - let mut hm = HashMap::new(); - hm.insert("ask_id".to_string(), script.ask_id.clone().to_string()); - hm.insert("summary".to_string(), script.summary.clone()); - hm.insert("app".to_string(), script.app.clone()); - hm.insert("kind".to_string(), script.kind.clone()); - hm.insert("id".to_string(), script.id.clone().to_string()); - hm.insert( - "version_id".to_string(), - script.version_id.clone().to_string(), - ); - let embedding = Embedding { - id: script.ask_id.clone().to_string(), - vector: script.embedding.clone(), - metadata: Some(hm), - }; - self.db.insert_into_collection("scripts", embedding)?; - } - - let response = match hub_base_url.as_str() { - DEFAULT_HUB_BASE_URL => { - let response = HTTP_CLIENT - .get("https://bucket.windmillhub.com/embeddings/resource_types_embeddings.json") - .send() - .await; - if response.is_err() || response.as_ref().unwrap().error_for_status_ref().is_err() { - tracing::warn!( - "Failed to get resource types embeddings from bucket, trying hub..." - ); - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/resource_types/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } else { - response.unwrap() - } - } - _ => { - http_get_from_hub( - &HTTP_CLIENT, - &format!("{}/resource_types/embeddings", hub_base_url), - false, - None, - Some(pg_db), - ) - .await? - } - }; - - if response.error_for_status_ref().is_err() { - return Err(anyhow!( - "Failed to get resource types embeddings from hub with error code: {}", - response.status() - )); - } - let hub_resource_types = response.json::>().await?; - - let resource_types: Vec = - sqlx::query_as!(ResourceType, "SELECT * from resource_type ORDER BY name",) - .fetch_all(pg_db) - .await?; - - for rt in resource_types { - let mut hm = HashMap::new(); - hm.insert("name".to_string(), rt.name.clone()); - if let Some(schema) = rt.schema.clone() { - hm.insert("schema".to_string(), serde_json::to_string(&schema)?); - } - hm.insert("workspace".to_string(), rt.workspace_id.clone()); - let hub_rt = hub_resource_types.iter().find(|hrt| hrt.name == rt.name); - - let vector = if let Some(hub_rt) = hub_rt { - hub_rt.embedding.clone() - } else { - self.model_instance - .clone() - .create_embedding(&format!( - "{};{}", - rt.name, - rt.description.unwrap_or_default() - )) - .await? - }; - - let embedding = Embedding { - id: format!("{}_{}", rt.workspace_id, rt.name), - vector, - metadata: Some(hm), - }; - - self.db - .insert_into_collection("resource_types", embedding)?; - } - - Ok(()) - } - - pub async fn query_hub_scripts( - &self, - query: &str, - limit: Option, - kind: Option, - app: Option, - ) -> Result> { - let model_instance = self.model_instance.clone(); - let query_embedding = model_instance.create_embedding(query).await?; - - let collection = self.db.get_collection("scripts"); - - let collection = collection.ok_or(Error::msg("no collection found"))?; - - let filter = |embedding: &Embedding| { - if let Some(metadata) = embedding.metadata.as_ref() { - match ( - metadata.get("kind"), - kind.clone(), - metadata.get("app"), - app.clone(), - ) { - (Some(script_kind), Some(kind), Some(script_app), Some(app)) => { - &kind == script_kind && &app == script_app - } - (Some(script_kind), Some(kind), _, _) => &kind == script_kind, - (_, _, Some(script_app), Some(app)) => &app == script_app, - (_, None, _, None) => true, - _ => false, - } - } else { - false - } - }; - - let results = collection.get_similarity( - &query_embedding, - limit.unwrap_or(10) as usize, - Some(&filter), - Some(0.8), - ); - - let results: Result> = results - .iter() - .map(|r| { - let metadata = r - .embedding - .metadata - .as_ref() - .ok_or(Error::msg("no metadata"))?; - - Ok(HubScriptResult { - ask_id: metadata - .get("ask_id") - .ok_or(Error::msg("no ask_id"))? - .parse::()?, - summary: metadata - .get("summary") - .ok_or(Error::msg("no summary"))? - .to_owned(), - app: metadata.get("app").ok_or(Error::msg("no app"))?.to_owned(), - kind: metadata - .get("kind") - .ok_or(Error::msg("no kind"))? - .to_owned(), - id: metadata - .get("id") - .ok_or(Error::msg("no id"))? - .parse::()?, - version_id: metadata - .get("version_id") - .ok_or(Error::msg("no version_id"))? - .parse::()?, - score: r.score, - }) - }) - .collect(); - - let mut results = results?; - - if results.len() > 1 { - let top_score = results[0].score; - results = results - .into_iter() - .take_while(|r| (top_score - r.score) / top_score <= 0.05) - .collect(); - } - - Ok(results) - } - - pub async fn query_resource_types( - &self, - workspace: String, - query: &str, - limit: Option, - ) -> Result> { - let model_instance = self.model_instance.clone(); - let query_embedding = model_instance.create_embedding(query).await?; - - let collection = self.db.get_collection("resource_types"); - - if collection.is_none() { - return Ok(vec![]); - } - - let collection = collection.ok_or(Error::msg("no collection found"))?; - - let filter = |embedding: &Embedding| { - if let Some(metadata) = embedding.metadata.as_ref() { - match metadata.get("workspace").map(|x| x.as_str()) { - Some("admins") => true, - Some(rt_workspace) => &workspace == rt_workspace, - _ => false, - } - } else { - false - } - }; - - let results = collection.get_similarity( - &query_embedding, - limit.unwrap_or(10) as usize, - Some(&filter), - Some(0.75), - ); - - let results: Result<_> = results - .iter() - .map(|r| { - let metadata = r - .embedding - .metadata - .as_ref() - .ok_or(Error::msg("no metadata"))?; - Ok(ResourceTypeResult { - name: metadata - .get("name") - .ok_or(Error::msg("no name"))? - .to_owned(), - schema: match metadata.get("schema") { - Some(schema) => serde_json::from_str(schema)?, - None => None, - }, - score: r.score, - }) - }) - .collect(); - - results - } -} - -#[cfg(feature = "embedding")] -fn normalize_l2(v: &Tensor) -> Result { - Ok(v.broadcast_div(&v.sqr()?.sum_keepdim(1)?.sqrt()?)?) -} - -#[cfg(feature = "embedding")] -pub fn load_embeddings_db(db: &Pool) -> () { - let disable_embedding = std::env::var("DISABLE_EMBEDDING") - .ok() - .map(|x| x.parse::().unwrap_or(false)) - .unwrap_or(false); - - if !disable_embedding { - let db_clone = db.clone(); - tokio::spawn(async move { - let model_instance = ModelInstance::new().await; - if let Ok(model_instance) = model_instance { - let mut model_instance_lock = MODEL_INSTANCE.write().await; - *model_instance_lock = Some(Arc::new(model_instance)); - drop(model_instance_lock); - loop { - update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs( - *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, - )) - .await; - } - } else { - tracing::error!( - "Failed to initialize model instance: {}", - model_instance.err().unwrap() - ); - } - }); - } -} - -#[cfg(feature = "embedding")] -pub async fn update_embeddings_db(db: &Pool) -> () { - if let Some(model_instance) = MODEL_INSTANCE.read().await.as_ref() { - tracing::info!("Creating embeddings DB..."); - let new_embeddings_db = EmbeddingsDb::new(&db, model_instance.clone()).await; - if let Err(e) = new_embeddings_db.as_ref() { - tracing::error!("Failed to create embeddings db: {}", e); - } else { - let mut embeddings_db = EMBEDDINGS_DB.write().await; - *embeddings_db = new_embeddings_db.ok(); - tracing::info!("Created embeddings DB"); - } - } else { - tracing::error!("Could not update embeddings DB, model instance not initialized"); - } -} - -#[cfg(feature = "embedding")] -pub fn workspaced_service() -> Router { - Router::new().route("/query_resource_types", get(query_resource_types)) -} - -#[cfg(feature = "embedding")] -pub fn global_service() -> Router { - Router::new().route("/query_hub_scripts", get(query_hub_scripts)) -} - -#[cfg(not(feature = "embedding"))] -pub fn workspaced_service() -> Router { - Router::new() -} - -#[cfg(not(feature = "embedding"))] -pub fn global_service() -> Router { - Router::new() -} +pub use windmill_api_embeddings::*; diff --git a/backend/windmill-api/src/flow_conversations.rs b/backend/windmill-api/src/flow_conversations.rs index 4854615c18..4292139862 100644 --- a/backend/windmill-api/src/flow_conversations.rs +++ b/backend/windmill-api/src/flow_conversations.rs @@ -10,6 +10,7 @@ use sqlx::{FromRow, Postgres}; use uuid::Uuid; use crate::db::ApiAuthed; +pub use windmill_common::flow_conversations::FlowConversation; use windmill_common::{ db::{UserDB, DB}, error::{JsonResult, Result}, @@ -24,17 +25,6 @@ pub fn workspaced_service() -> Router { .route("/:conversation_id/messages", get(list_messages)) } -#[derive(Serialize, FromRow, Debug)] -pub struct FlowConversation { - pub id: Uuid, - pub workspace_id: String, - pub flow_path: String, - pub title: Option, - pub created_at: DateTime, - pub updated_at: DateTime, - pub created_by: String, -} - #[derive(Serialize, FromRow, Debug)] pub struct FlowConversationMessage { pub id: Uuid, @@ -104,55 +94,6 @@ async fn list_conversations( Ok(Json(conversations)) } -pub async fn get_or_create_conversation_with_id( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - w_id: &str, - flow_path: &str, - username: &str, - title: &str, - conversation_id: Uuid, -) -> Result { - // Check if conversation already exists - let existing_conversation = sqlx::query_as!( - FlowConversation, - "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by - FROM flow_conversation - WHERE id = $1 AND workspace_id = $2", - conversation_id, - w_id - ) - .fetch_optional(&mut **tx) - .await?; - - if let Some(existing) = existing_conversation { - return Ok(existing); - } - - // Truncate title to 25 char characters max - let title = if title.len() > 25 { - format!("{}...", &title[..25]) - } else { - title.to_string() - }; - - // Create new conversation with provided ID - let conversation = sqlx::query_as!( - FlowConversation, - "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) - VALUES ($1, $2, $3, $4, $5) - RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", - conversation_id, - w_id, - flow_path, - username, - title - ) - .fetch_one(&mut **tx) - .await?; - - Ok(conversation) -} - async fn delete_conversation( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index a73eb99b7e..80bb2540ca 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -306,24 +306,8 @@ pub async fn is_owner_api( Ok(Json(is_owner(&authed, &name))) } -pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool { - if *is_admin { - true - } else { - folders.into_iter().any(|x| x.0 == name && x.2) - } -} - -pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> { - if is_owner(authed, name) { - Ok(()) - } else { - Err(windmill_common::error::Error::NotAuthorized(format!( - "You are not owner of the folder {}", - name - ))) - } -} +use windmill_api_auth::is_owner; +pub use windmill_api_auth::require_is_owner; async fn update_folder( authed: ApiAuthed, diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index d67b97802b..d69f45d88f 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -271,15 +271,8 @@ async fn create_group( ) .await?; - log_group_permission_change( - &mut *tx, - &w_id, - &ng.name, - &authed.username, - "create", - None, - ) - .await?; + log_group_permission_change(&mut *tx, &w_id, &ng.name, &authed.username, "create", None) + .await?; tx.commit().await?; @@ -606,7 +599,10 @@ async fn add_user( .await?; if result.rows_affected() == 0 { - return Ok(format!("{} is already a member of group {}", user_username, name)); + return Ok(format!( + "{} is already a member of group {}", + user_username, name + )); } audit_log( @@ -693,12 +689,37 @@ async fn add_user_igroup( WHERE auto_invite->'instance_groups' ? $1 "#, &name - ).fetch_all(&mut *tx).await?; + ) + .fetch_all(&mut *tx) + .await?; for ws in workspaces { - let role = ws.instance_groups_roles.and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))).unwrap_or_else(|| "developer".to_string()); - let (is_admin, is_operator) = match role.as_str() { "admin" => (true, false), "operator" => (false, true), _ => (false, false) }; - auto_add_user(&email, &ws.workspace_id, &is_operator, &mut tx, &authed, Some(serde_json::json!({"source": "instance_group", "group": &name}))).await?; - if is_admin { sqlx::query!("UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", &ws.workspace_id, &email).execute(&mut *tx).await?; } + let role = ws + .instance_groups_roles + .and_then(|r| r.get(&name).and_then(|v| v.as_str().map(String::from))) + .unwrap_or_else(|| "developer".to_string()); + let (is_admin, is_operator) = match role.as_str() { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + auto_add_user( + &email, + &ws.workspace_id, + &is_operator, + &mut tx, + &authed, + Some(serde_json::json!({"source": "instance_group", "group": &name})), + ) + .await?; + if is_admin { + sqlx::query!( + "UPDATE usr SET is_admin = true WHERE workspace_id = $1 AND email = $2", + &ws.workspace_id, + &email + ) + .execute(&mut *tx) + .await?; + } } } @@ -741,7 +762,9 @@ async fn list_igroups(Extension(db): Extension) -> JsonResult> { return Ok(Json(groups)); } -async fn list_igroups_with_workspaces(Extension(db): Extension) -> JsonResult> { +async fn list_igroups_with_workspaces( + Extension(db): Extension, +) -> JsonResult> { let mut tx: Transaction<'_, Postgres> = db.begin().await?; // Get all instance groups with their emails first @@ -771,9 +794,11 @@ async fn list_igroups_with_workspaces(Extension(db): Extension) -> JsonResul .await?; // Create a map of group_name -> Vec - let mut workspaces_by_group: std::collections::HashMap> = std::collections::HashMap::new(); + let mut workspaces_by_group: std::collections::HashMap> = + std::collections::HashMap::new(); for mapping in workspace_mappings { - let role = mapping.role + let role = mapping + .role .and_then(|r| r.as_str().map(|s| s.to_string())) .unwrap_or_else(|| "developer".to_string()); @@ -791,7 +816,10 @@ async fn list_igroups_with_workspaces(Extension(db): Extension) -> JsonResul let mut result = Vec::new(); for group in groups { - let workspaces = workspaces_by_group.get(&group.name).cloned().unwrap_or_default(); + let workspaces = workspaces_by_group + .get(&group.name) + .cloned() + .unwrap_or_default(); result.push(IGroupWithWorkspaces { name: group.name, diff --git a/backend/windmill-api/src/job_metrics.rs b/backend/windmill-api/src/job_metrics.rs index 70e68ba3ed..c78d8472b7 100644 --- a/backend/windmill-api/src/job_metrics.rs +++ b/backend/windmill-api/src/job_metrics.rs @@ -1,266 +1 @@ -use crate::db::DB; - -use axum::{ - extract::Path, - routing::{get, post}, - Extension, Json, Router, -}; -use serde::{Deserialize, Serialize}; -use tower_http::cors::{Any, CorsLayer}; -use uuid::Uuid; -use windmill_common::{ - error::{self, Error}, - job_metrics::{ - record_metric, register_metric_for_job, JobStatsRecord, MetricKind, MetricNumericValue, - }, -}; - -pub fn workspaced_service() -> Router { - let cors = CorsLayer::new() - .allow_methods([http::Method::GET, http::Method::POST]) - .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) - .allow_origin(Any); - - Router::new() - .route("/get/:id", post(get_job_metrics).layer(cors.clone())) - .route( - "/set_progress/:id", - post(set_job_progress).layer(cors.clone()), - ) - .route( - "/get_progress/:id", - get(get_job_progress).layer(cors.clone()), - ) -} - -#[derive(Deserialize)] -struct JobStatsRequest { - from_timestamp: Option>, - to_timestamp: Option>, - timeseries_max_datapoints: Option, // default to 100, any value lower than that will be ignored. Set to 0 to retrieve all -} - -#[derive(Serialize)] -struct JobStatsResponse { - metrics_metadata: Vec, - scalar_metrics: Vec, - timeseries_metrics: Vec, -} - -#[derive(Serialize)] -pub struct MetricsMetadata { - pub id: String, - pub name: Option, -} - -#[derive(Serialize)] -struct ScalarMetric { - metric_id: String, - value: f64, -} - -#[derive(Serialize)] -struct TimeseriesMetric { - metric_id: String, - values: Vec, -} - -#[derive(Debug, Clone, Serialize)] -pub struct DataPoint { - pub timestamp: chrono::DateTime, - pub value: f64, -} - -async fn get_job_metrics( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Json(JobStatsRequest { from_timestamp, to_timestamp, timeseries_max_datapoints }): Json< - JobStatsRequest, - >, -) -> error::JsonResult { - let records = sqlx::query_as::<_, JobStatsRecord>( - "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", - ) - .bind(w_id) - .bind(job_id) - .fetch_all(&db) - .await?; - - let mut metrics_metadata: Vec = vec![]; - let mut scalar_metrics: Vec = vec![]; - let mut timeseries_metrics: Vec = vec![]; - - for record in records { - let metric_id = record.metric_id; - match record.metric_kind { - MetricKind::ScalarInt => { - let value = record.scalar_int.unwrap_or_default() as f64; - scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); - } - MetricKind::ScalarFloat => { - let value = record.scalar_float.unwrap_or_default() as f64; - scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); - } - MetricKind::TimeseriesInt => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) - } - let (timestamps, timeseries_int) = timeseries_sample( - from_timestamp, - to_timestamp, - timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_int.unwrap_or_default(), - ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_int.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } - timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); - } - MetricKind::TimeseriesFloat => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) - } - let (timestamps, timeseries_float) = timeseries_sample( - from_timestamp, - to_timestamp, - timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_float.unwrap_or_default(), - ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_float.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } - timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); - } - }; - metrics_metadata.push(MetricsMetadata { id: metric_id, name: record.metric_name }); - } - - let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; - Ok(Json(response)) -} -#[derive(Deserialize)] -struct JobProgressSetRequest { - percent: i32, - /// Optional parent flow id - /// Used to modify flow status - /// Specifically `progress` field in corresponding FlowStatusModule in `InProgress` state - flow_job_id: Option, -} - -async fn set_job_progress( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, - Json(JobProgressSetRequest { percent, flow_job_id }): Json, -) -> error::JsonResult<()> { - // If flow_job_id exists, than we should modify flow_status of corresponding module - // Individual jobs and flows are handled differently - if let Some(flow_job_id) = flow_job_id { - // TODO: Return error if trying to set completed job? - sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['modules', flow_status->>'step', 'progress'], $1) - WHERE id = $2", - serde_json::json!(percent.clamp(0, 99)), - flow_job_id - ) - .execute(&db) - .await?; - } - - let record_progress = || { - record_metric( - &db, - w_id.clone(), - job_id, - "progress_perc".to_owned(), - MetricNumericValue::Integer(percent), - ) - }; - - // Try to record - if let Err(err) = record_progress().await { - if matches!(err, Error::MetricNotFound(..)) { - // Register - // TODO: Reset progress after job is finished (in case it reruns same job)? - _ = register_metric_for_job( - &db, - w_id.clone(), - job_id, - "progress_perc".to_string(), - MetricKind::ScalarInt, - Some("Job Execution Progress (%)".to_owned()), - ) - .await?; - // Retry recording progress - record_progress().await.map_err(|err| { - // If for some reason it still returns same error, this error will be converted to BadRequest and returned - if let Error::MetricNotFound(body) = err { - Error::BadRequest(body) - } else { - err - } - })?; - } else { - return Err(err); - } - }; - return Ok(Json(())); -} - -async fn get_job_progress( - Extension(db): Extension, - Path((w_id, job_id)): Path<(String, Uuid)>, -) -> error::JsonResult> { - let progress: Option> = sqlx::query_scalar!( - "SELECT (scalar_int)::int FROM job_stats WHERE job_id = $1 AND workspace_id = $2 AND metric_id = 'progress_perc'", - job_id, w_id) - .fetch_optional(&db) - .await?; - - let respond_value = if let Some(Some(progress)) = progress { - Some(progress.clamp(0, 99)) - } else { - None - }; - - Ok(Json(respond_value)) -} - -fn timeseries_sample( - from: Option>, - to: Option>, - _datapoints: Option, - timestamps: Vec>, - values: Vec, -) -> (Vec>, Vec) { - if timestamps.len() != values.len() { - tracing::warn!("Timeseries metric has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", timestamps.len(), values.len()); - return (vec![], vec![]); - } - let mut filtered_timestamp: Vec> = vec![]; - let mut filtered_values: Vec = vec![]; - for (idx, timestamp) in timestamps.iter().enumerate() { - if *timestamp > from.unwrap_or(chrono::DateTime::::MIN_UTC) - && *timestamp < to.unwrap_or(chrono::DateTime::::MAX_UTC) - { - filtered_timestamp.push(timestamps[idx]); - filtered_values.push(values[idx]); - } - } - // TODO: implement sampling - return (filtered_timestamp, filtered_values); -} +pub use windmill_api_jobs::job_metrics::*; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 15adf46caa..ecf86079bf 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6,22 +6,22 @@ * LICENSE-AGPL for a copy of the license. */ +pub use windmill_api_jobs::execution::*; +pub use windmill_api_jobs::query::*; +pub use windmill_api_jobs::types::*; +pub use windmill_api_sse::*; + use axum::body::Body; -use axum::extract::Request; -use axum::http::HeaderValue; #[cfg(feature = "deno_core")] use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; use futures::{StreamExt, TryFutureExt}; -use http::{HeaderMap, HeaderName}; use itertools::Itertools; use quick_cache::sync::Cache; use serde_json::value::RawValue; use serde_json::Value; -use sqlx::Pool; use std::collections::HashMap; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::ops::{Deref, DerefMut}; use std::str::FromStr; +use std::sync::Arc; use std::time::Instant; use tokio::io::AsyncReadExt; use tower::ServiceBuilder; @@ -32,11 +32,10 @@ use windmill_common::auth::TOKEN_PREFIX_LEN; use windmill_common::client::AuthedClient; use windmill_common::db::UserDbWithAuthed; use windmill_common::error::JsonResult; -use windmill_common::flow_conversations::add_message_to_conversation_tx; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ - check_tag_available_for_workspace_internal, format_completed_job_result, format_result, - DynamicInput, JobTriggerKind, RunInlinePreviewScriptFnParams, ENTRYPOINT_OVERRIDE, + format_completed_job_result, format_result, DynamicInput, RunInlinePreviewScriptFnParams, + ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettings, @@ -74,16 +73,15 @@ use crate::{ }; use anyhow::Context; use axum::{ - extract::{FromRequest, Json, Path, Query}, + extract::{Json, Path, Query}, response::{IntoResponse, Response}, routing::{get, post}, Extension, Router, }; -use base64::Engine; use chrono::Utc; use hmac::Mac; use hyper::StatusCode; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::types::JsonRawValue; use sqlx::{types::Uuid, FromRow, Postgres, Transaction}; @@ -104,24 +102,15 @@ use windmill_common::{ oauth2::HmacSha256, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, - utils::{ - not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, - Pagination, StripPath, - }, + utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, }; use windmill_common::{ get_flow_version_info_from_version, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, get_script_info_for_hash, utils::empty_as_none, - FlowVersionInfo, ScriptHashInfo, BASE_URL, + ScriptHashInfo, BASE_URL, }; -use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, - PushArgsOwned, PushIsolationLevel, -}; - -use crate::flow_conversations; -use windmill_common::flow_conversations::MessageType; +use windmill_queue::{job_is_complete, push, PushArgs, PushArgsOwned, PushIsolationLevel}; pub fn workspaced_service() -> Router { let cors = CorsLayer::new() @@ -1156,7 +1145,7 @@ impl<'a> GetQuery<'a> { #[cfg(all(feature = "smtp", feature = "enterprise"))] async fn send_workspace_trigger_failure_email_notification( - db: &Pool, + db: &DB, w_id: &str, job_id: &Uuid, trigger_path: Option<&str>, @@ -1726,337 +1715,6 @@ async fn get_started_at_by_ids( Ok(Json(r)) } -#[derive(Debug, sqlx::FromRow, Serialize)] -pub struct ListableCompletedJob { - pub r#type: String, - pub workspace_id: String, - pub id: Uuid, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub duration_ms: i64, - pub success: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_hash: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub script_path: Option, - pub deleted: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_code: Option, - pub canceled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_by: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub canceled_reason: Option, - pub job_kind: JobKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub schedule_path: Option, - pub permissioned_as: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_flow: Option, - pub is_flow_step: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - pub tag: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub args: Option, -} - -#[derive(Debug, Deserialize, Clone, Default)] -pub struct RunJobQuery { - pub scheduled_for: Option>, - pub scheduled_in_secs: Option, - pub parent_job: Option, - pub root_job: Option, - pub invisible_to_owner: Option, - pub queue_limit: Option, - pub payload: Option, - pub job_id: Option, - pub tag: Option, - pub timeout: Option, - pub cache_ttl: Option, - pub cache_ignore_s3_path: Option, - pub skip_preprocessor: Option, - pub poll_delay_ms: Option, - pub memory_id: Option, - pub trigger_external_id: Option, - pub service_name: Option, - pub suspended_mode: Option, -} - -impl RunJobQuery { - async fn get_scheduled_for<'c>( - &self, - db: &DB, - ) -> error::Result>> { - if let Some(scheduled_for) = self.scheduled_for { - Ok(Some(scheduled_for)) - } else if let Some(scheduled_in_secs) = self.scheduled_in_secs { - let now = now_from_db(db).await?; - Ok(Some( - now + chrono::Duration::try_seconds(scheduled_in_secs).unwrap_or_default(), - )) - } else { - Ok(None) - } - } - - fn payload_as_args(&self) -> error::Result>> { - let payload_r = self.payload.clone().map(decode_payload).map(|x| { - x.map_err(|e| { - error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) - }) - }); - - let payload_as_args = if let Some(payload) = payload_r { - payload? - } else { - HashMap::new() - }; - - Ok(payload_as_args) - } -} - -#[derive(Deserialize, Clone)] -pub struct ListQueueQuery { - pub script_path_start: Option, - pub script_path_exact: Option, - pub script_hash: Option, - pub created_by: Option, - pub started_before: Option>, - pub started_after: Option>, - pub created_before: Option>, - pub created_after: Option>, - pub created_or_started_before: Option>, - pub created_or_started_after: Option>, - pub running: Option, - pub schedule_path: Option, - pub parent_job: Option, - pub order_desc: Option, - pub job_kinds: Option, - pub suspended: Option, - pub worker: Option, - // filter by matching a subset of the args using base64 encoded json subset - pub args: Option, - pub tag: Option, - pub scheduled_for_before_now: Option, - pub all_workspaces: Option, - pub is_flow_step: Option, - pub has_null_parent: Option, - pub is_not_schedule: Option, - pub concurrency_key: Option, - pub allow_wildcards: Option, - pub trigger_kind: Option, - pub trigger_path: Option, - pub include_args: Option, -} - -impl From for ListQueueQuery { - fn from(lcq: ListCompletedQuery) -> Self { - Self { - script_path_start: lcq.script_path_start, - script_path_exact: lcq.script_path_exact, - script_hash: lcq.script_hash, - created_by: lcq.created_by, - started_before: lcq.started_before, - started_after: lcq.started_after, - created_before: lcq.created_before_queue.or(lcq.created_before), - created_after: lcq.created_after_queue.or(lcq.created_after), - created_or_started_before: lcq.created_or_started_before, - created_or_started_after: lcq.created_or_started_after, - worker: lcq.worker, - running: lcq.running, - parent_job: lcq.parent_job, - order_desc: lcq.order_desc, - job_kinds: lcq.job_kinds, - suspended: lcq.suspended, - args: lcq.args, - tag: lcq.tag, - schedule_path: lcq.schedule_path, - scheduled_for_before_now: lcq.scheduled_for_before_now, - all_workspaces: lcq.all_workspaces, - is_flow_step: lcq.is_flow_step, - has_null_parent: lcq.has_null_parent, - is_not_schedule: lcq.is_not_schedule, - concurrency_key: lcq.concurrency_key, - allow_wildcards: lcq.allow_wildcards, - trigger_kind: lcq.trigger_kind, - trigger_path: lcq.trigger_path, - include_args: lcq.include_args, - } - } -} - -pub fn filter_list_queue_query( - mut sqlb: SqlBuilder, - lq: &ListQueueQuery, - w_id: &str, - join_outstanding_wait_times: bool, -) -> SqlBuilder { - sqlb.join("v2_job").on_eq("v2_job_queue.id", "v2_job.id"); - - if join_outstanding_wait_times { - sqlb.left() - .join("outstanding_wait_time") - .on_eq("v2_job_queue.id", "outstanding_wait_time.job_id"); - } - - if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job_queue.workspace_id", "?".bind(&w_id)); - } - - if let Some(w) = &lq.worker { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job_queue.worker", w.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job_queue.worker", "?".bind(w)); - } - } - - if let Some(ps) = &lq.script_path_start { - sqlb.and_where_like_left("runnable_path", ps); - } - if let Some(p) = &lq.script_path_exact { - sqlb.and_where_eq("runnable_path", "?".bind(p)); - } - if let Some(p) = &lq.schedule_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - sqlb.and_where_eq("trigger_kind", "'schedule'"); - } - if let Some(h) = &lq.script_hash { - sqlb.and_where_eq("runnable_id", "?".bind(h)); - } - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(t) = &lq.tag { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job.tag", "?".bind(t)); - } - } - - if let Some(r) = &lq.running { - sqlb.and_where_eq("running", &r); - } - if let Some(pj) = &lq.parent_job { - sqlb.and_where_eq("parent_job", "?".bind(pj)); - } - if let Some(dt) = &lq.started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.started_after { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(fs) = &lq.is_flow_step { - if *fs { - sqlb.and_where_is_not_null("flow_step_id"); - } else { - sqlb.and_where_is_null("flow_step_id"); - } - } - if let Some(fs) = &lq.has_null_parent { - if *fs { - sqlb.and_where_is_null("parent_job"); - } - } - - if let Some(dt) = &lq.created_before { - sqlb.and_where_le("v2_job.created_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_after { - sqlb.and_where_ge("v2_job.created_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_after { - let ts = dt.timestamp_millis(); - sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at >= to_timestamp({} / 1000.0))", ts, ts)); - } - - if let Some(dt) = &lq.created_or_started_before { - let ts = dt.timestamp_millis(); - sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({} / 1000.0)) OR (started_at IS NULL AND v2_job.created_at < to_timestamp({} / 1000.0))", ts, ts)); - } - - if let Some(s) = &lq.suspended { - if *s { - sqlb.and_where_gt("suspend", 0); - } else { - sqlb.and_where_is_null("suspend_until"); - } - } - - if let Some(jk) = &lq.job_kinds { - sqlb.and_where_in( - "kind", - &jk.split(',').into_iter().map(quote).collect::>(), - ); - } - - if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); - } - - if lq.scheduled_for_before_now.is_some_and(|x| x) { - sqlb.and_where_le("scheduled_for", "now()"); - } - - if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); - } - - if let Some(tk) = &lq.trigger_kind { - sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); - } - - if let Some(tp) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(tp)); - } - - sqlb -} - -pub fn list_queue_jobs_query( - w_id: &str, - lq: &ListQueueQuery, - fields: &[&str], - pagination: Pagination, - join_outstanding_wait_times: bool, - tags: Option>, -) -> SqlBuilder { - let (limit, offset) = paginate_without_limits(pagination); - let mut sqlb = SqlBuilder::select_from("v2_job_queue") - .fields(fields) - .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) - .limit(limit) - .offset(offset) - .clone(); - - if let Some(tags) = tags { - sqlb.and_where_in( - "v2_job.tag", - &tags.iter().map(|x| quote(x)).collect::>(), - ); - } - - filter_list_queue_query(sqlb, lq, w_id, join_outstanding_wait_times) -} - #[derive(Serialize, FromRow)] struct ListableQueuedJob { pub id: Uuid, @@ -2137,103 +1795,6 @@ async fn list_queue_jobs( Ok(Json(jobs)) } -pub async fn cancel_jobs( - jobs: Vec, - db: &DB, - username: &str, - w_id: &str, - force_cancel: bool, -) -> error::JsonResult> { - let mut uuids = vec![]; - tracing::info!("Cancelling jobs: {:?}", jobs); - let mut tx = db.begin().await?; - let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj - ( workspace_id - , id - , duration_ms - , result - , canceled_by - , canceled_reason - , flow_status - , status - , worker - ) - SELECT q.workspace_id - , q.id - , 0 - , $4 - , $1 - , 'cancel all' - , (SELECT flow_status FROM v2_job_status WHERE id = q.id) - , 'canceled'::job_status - , worker - FROM v2_job_queue q - JOIN v2_job USING (id) - WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger_kind IS DISTINCT FROM 'schedule' - FOR UPDATE SKIP LOCKED - ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) - .fetch_all(&mut *tx) - .await?.into_iter().map(|x| x.id).collect::>(); - - sqlx::query!( - "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", - &trivial_jobs, - w_id - ) - .execute(&mut *tx) - .await?; - tx.commit().await?; - - // sqlx::query!( - // "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = 'cancelled all by user' WHERE id IN (SELECT id FROM queue where id = any($2) AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED) RETURNING id", - // username, - // &jobs, - // w_id - // ).execute(db).await?; - for job_id in jobs.into_iter() { - if trivial_jobs.contains(&job_id) { - continue; - } - match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move { - let tx = db.begin().await?; - let (tx, _) = windmill_queue::cancel_job( - username, - None, - job_id.clone(), - w_id, - tx, - db, - force_cancel, - false, - ) - .await?; - tx.commit().await?; - Ok::<_, anyhow::Error>(()) - }) - .await - { - Ok(result) => match result { - Ok(_) => { - uuids.push(job_id); - } - Err(e) => { - tracing::error!("Failed to cancel job {:?}: {:?}", job_id, e); - } - }, - Err(_) => { - tracing::error!( - "Timeout while trying to cancel job {:?} after 5 seconds", - job_id - ); - } - } - } - - uuids.extend(trivial_jobs); - - Ok(Json(uuids)) -} - #[derive(Deserialize)] pub struct CancelSelectionQuery { force_cancel: Option, @@ -2948,14 +2509,6 @@ pub struct SuspendedJobFlow { pub approvers: Vec, } -#[derive(Deserialize, Debug)] -pub struct QueryApprover { - pub approver: Option, - /// If true, generate/verify resume URLs for the parent flow instead of the specific step. - /// This allows pre-approvals that can be consumed by any later suspend step in the same flow. - pub flow_level: Option, -} - pub async fn get_suspended_job_flow( authed: Option, opt_tokened: OptTokened, @@ -3170,14 +2723,6 @@ fn create_signature( Ok(hex::encode(mac.finalize().into_bytes())) } -#[allow(non_snake_case)] -#[derive(Serialize, Debug)] -pub struct ResumeUrls { - pub approvalPage: String, - pub cancel: String, - pub resume: String, -} - fn build_resume_url( op: &str, w_id: &str, @@ -3282,466 +2827,6 @@ async fn get_flow_id_for_job(db: &DB, job_id: Uuid) -> error::Result { .ok_or_else(|| anyhow::anyhow!("job {} has no parent flow", job_id).into()) } -#[derive(sqlx::FromRow, Debug, Serialize)] -pub struct JobExtended { - #[sqlx(flatten)] - #[serde(flatten)] - inner: T, - - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_code: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_lock: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub raw_flow: Option>>, - - #[serde(skip_serializing_if = "Option::is_none")] - pub worker: Option, - - #[sqlx(skip)] - #[serde(skip_serializing_if = "Option::is_none")] - pub self_wait_time_ms: Option, - #[sqlx(skip)] - #[serde(skip_serializing_if = "Option::is_none")] - pub aggregate_wait_time_ms: Option, -} - -pub trait JobCommon { - fn job_kind(&self) -> &JobKind; -} - -impl JobCommon for QueuedJob { - fn job_kind(&self) -> &JobKind { - &self.job_kind - } -} - -impl JobCommon for CompletedJob { - fn job_kind(&self) -> &JobKind { - &self.job_kind - } -} - -impl JobExtended { - pub fn new( - self_wait_time_ms: Option, - aggregate_wait_time_ms: Option, - inner: T, - ) -> Self { - Self { - inner, - raw_code: None, - raw_lock: None, - raw_flow: None, - worker: None, - self_wait_time_ms, - aggregate_wait_time_ms, - } - } -} - -impl Deref for JobExtended { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for JobExtended { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -#[derive(Serialize, Debug)] -#[serde(tag = "type")] -pub enum Job { - QueuedJob(JobExtended), - CompletedJob(JobExtended), -} - -impl Job { - pub fn created_by(&self) -> &str { - match self { - Job::QueuedJob(job) => &job.created_by, - Job::CompletedJob(job) => &job.created_by, - } - } - - pub fn append_to_logs(&mut self, logs: &str) { - match self { - Job::QueuedJob(job) => { - if let Some(ref mut l) = job.logs { - l.push_str(logs); - } else { - job.logs = Some(logs.to_string()); - } - } - Job::CompletedJob(job) => { - if let Some(ref mut l) = job.logs { - l.push_str(logs); - } else { - job.logs = Some(logs.to_string()); - } - } - } - } - - pub fn log_len(&self) -> Option { - match self { - Job::QueuedJob(job) => job.logs.as_ref().map(|l| l.len()), - Job::CompletedJob(job) => job.logs.as_ref().map(|l| l.len()), - } - } - - pub fn logs(&self) -> Option { - match self { - Job::QueuedJob(job) => job.logs.clone(), - Job::CompletedJob(job) => job.logs.clone(), - } - } - pub fn flow_status(&self) -> Option { - match self { - Job::QueuedJob(job) => job - .flow_status - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - Job::CompletedJob(job) => job - .flow_status - .as_ref() - .map(|rf| serde_json::from_str(rf.0.get()).ok()) - .flatten(), - } - } - pub fn is_flow_step(&self) -> bool { - match self { - Job::QueuedJob(job) => job.is_flow_step, - Job::CompletedJob(job) => job.is_flow_step, - } - } - - pub fn is_flow(&self) -> bool { - self.job_kind().is_flow() - } - - pub fn job_kind(&self) -> &JobKind { - match self { - Job::QueuedJob(job) => &job.job_kind, - Job::CompletedJob(job) => &job.job_kind, - } - } - - pub fn id(&self) -> Uuid { - match self { - Job::QueuedJob(job) => job.id, - Job::CompletedJob(job) => job.id, - } - } - - pub fn workspace_id(&self) -> &String { - match self { - Job::QueuedJob(job) => &job.workspace_id, - Job::CompletedJob(job) => &job.workspace_id, - } - } - - pub fn script_path(&self) -> &str { - match self { - Job::QueuedJob(job) => job.script_path.as_ref(), - Job::CompletedJob(job) => job.script_path.as_ref(), - } - .map(String::as_str) - .unwrap_or("tmp/main") - } - - pub fn args(&self) -> Option<&sqlx::types::Json>>> { - match self { - Job::QueuedJob(job) => job.args.as_ref(), - Job::CompletedJob(job) => job.args.as_ref(), - } - } - - pub fn full_path_with_workspace(&self) -> String { - format!( - "{}/{}/{}", - self.workspace_id(), - if self.is_flow() { "flow" } else { "script" }, - self.script_path(), - ) - } - - pub async fn concurrency_key( - &self, - db: &Pool, - ) -> Result, sqlx::Error> { - sqlx::query_scalar!( - "SELECT key FROM concurrency_key WHERE job_id = $1", - self.id() - ) - .fetch_optional(db) - .await - } - - pub async fn fetch_outstanding_wait_time( - &mut self, - db: &Pool, - ) -> Result<(), sqlx::Error> { - let r = sqlx::query!( - "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", - self.id() - ) - .fetch_optional(db) - .await?; - - let (self_wait_time, aggregate_wait_time) = r - .map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms)) - .unwrap_or((None, None)); - - match self { - Job::QueuedJob(job) => { - job.self_wait_time_ms = self_wait_time; - job.aggregate_wait_time_ms = aggregate_wait_time; - } - Job::CompletedJob(job) => { - job.self_wait_time_ms = self_wait_time; - job.aggregate_wait_time_ms = aggregate_wait_time; - } - } - Ok(()) - } -} - -#[derive(sqlx::FromRow)] -pub struct UnifiedJob { - pub workspace_id: String, - pub typ: String, - pub id: Uuid, - pub parent_job: Option, - pub created_by: String, - pub created_at: chrono::DateTime, - pub started_at: Option>, - pub completed_at: Option>, - pub scheduled_for: Option>, - pub running: Option, - pub script_hash: Option, - pub script_path: Option, - pub args: Option, - pub duration_ms: Option, - pub success: Option, - pub deleted: bool, - pub canceled: bool, - pub canceled_by: Option, - pub job_kind: JobKind, - pub schedule_path: Option, - pub permissioned_as: String, - pub is_flow_step: bool, - pub language: Option, - pub is_skipped: bool, - pub email: String, - pub visible_to_owner: bool, - pub suspend: Option, - pub mem_peak: Option, - pub tag: String, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub priority: Option, - pub labels: Option, - pub self_wait_time_ms: Option, - pub aggregate_wait_time_ms: Option, - pub preprocessed: Option, - pub worker: Option, - pub runnable_settings_handle: Option, -} - -const CJ_FIELDS: &[&str] = &[ - "'CompletedJob' as typ", - "v2_job_completed.id", - "v2_job_completed.workspace_id", - "v2_job.parent_job", - "v2_job.created_by", - "v2_job.created_at", - "v2_job_completed.started_at", - "null as scheduled_for", - "v2_job_completed.completed_at", - "null as running", - "v2_job.runnable_id as script_hash", - "v2_job.runnable_path as script_path", - "null as args", - "v2_job_completed.duration_ms", - "v2_job_completed.status = 'success' OR v2_job_completed.status = 'skipped' as success", - "false as deleted", - "v2_job_completed.status = 'canceled' as canceled", - "v2_job_completed.canceled_by", - "v2_job.kind as job_kind", - "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", - "v2_job.permissioned_as", - "v2_job.flow_step_id IS NOT NULL as is_flow_step", - "v2_job.script_lang as language", - "v2_job_completed.status = 'skipped' as is_skipped", - "v2_job.permissioned_as_email as email", - "v2_job.visible_to_owner", - "null as suspend", - "v2_job_completed.memory_peak as mem_peak", - "v2_job.tag", - "null as concurrent_limit", - "null as concurrency_time_window_s", - "v2_job.priority", - "v2_job_completed.result->'wm_labels' as labels", - "self_wait_time_ms", - "aggregate_wait_time_ms", - "v2_job.preprocessed", - "v2_job_completed.worker", - "null as runnable_settings_handle", -]; - -const QJ_FIELDS: &[&str] = &[ - "'QueuedJob' as typ", - "v2_job_queue.id", - "v2_job_queue.workspace_id", - "v2_job.parent_job", - "v2_job.created_by", - "v2_job_queue.created_at", - "v2_job_queue.started_at", - "v2_job_queue.scheduled_for", - "null as completed_at", - "v2_job_queue.running", - "v2_job.runnable_id as script_hash", - "v2_job.runnable_path as script_path", - "null as args", - "null as duration_ms", - "null as success", - "false as deleted", - "v2_job_queue.canceled_by IS NOT NULL as canceled", - "v2_job_queue.canceled_by", - "v2_job.kind as job_kind", - "CASE WHEN v2_job.trigger_kind = 'schedule' THEN v2_job.trigger END as schedule_path", - "v2_job.permissioned_as", - "v2_job.flow_step_id IS NOT NULL as is_flow_step", - "v2_job.script_lang as language", - "false as is_skipped", - "v2_job.permissioned_as_email as email", - "v2_job.visible_to_owner", - "v2_job_queue.suspend", - "null as mem_peak", - "v2_job.tag", - "v2_job.concurrent_limit", - "v2_job.concurrency_time_window_s", - "v2_job.priority", - "null as labels", - "self_wait_time_ms", - "aggregate_wait_time_ms", - "v2_job.preprocessed", - "v2_job_queue.worker", - "v2_job_queue.runnable_settings_handle", -]; - -impl UnifiedJob { - pub fn completed_job_fields() -> &'static [&'static str] { - CJ_FIELDS - } - pub fn queued_job_fields() -> &'static [&'static str] { - QJ_FIELDS - } -} - -impl<'a> From for Job { - fn from(uj: UnifiedJob) -> Self { - let args = uj.args.and_then(|v| serde_json::from_value(v).ok()); - match uj.typ.as_ref() { - "CompletedJob" => Job::CompletedJob(JobExtended::new( - uj.self_wait_time_ms, - uj.aggregate_wait_time_ms, - CompletedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at, - completed_at: uj.completed_at, - duration_ms: uj.duration_ms.unwrap(), - success: uj.success.unwrap(), - script_hash: uj.script_hash, - script_path: uj.script_path, - args: args.clone(), - result: None, - result_columns: None, - logs: None, - flow_status: None, - workflow_as_code_status: None, - deleted: uj.deleted, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - canceled_reason: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - is_flow_step: uj.is_flow_step, - language: uj.language, - is_skipped: uj.is_skipped, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - mem_peak: uj.mem_peak, - tag: uj.tag, - priority: uj.priority, - labels: uj.labels, - preprocessed: uj.preprocessed, - }, - )), - "QueuedJob" => Job::QueuedJob(JobExtended::new( - uj.self_wait_time_ms, - uj.aggregate_wait_time_ms, - QueuedJob { - workspace_id: uj.workspace_id, - id: uj.id, - parent_job: uj.parent_job, - created_by: uj.created_by, - created_at: uj.created_at, - started_at: uj.started_at, - scheduled_for: uj.scheduled_for.unwrap(), - running: uj.running.unwrap(), - script_hash: uj.script_hash, - script_path: uj.script_path, - script_entrypoint_override: None, - args, - logs: None, - canceled: uj.canceled, - canceled_by: uj.canceled_by, - canceled_reason: None, - last_ping: None, - job_kind: uj.job_kind, - schedule_path: uj.schedule_path, - permissioned_as: uj.permissioned_as, - flow_status: None, - workflow_as_code_status: None, - is_flow_step: uj.is_flow_step, - language: uj.language, - same_worker: false, - pre_run_error: None, - email: uj.email, - visible_to_owner: uj.visible_to_owner, - suspend: uj.suspend, - mem_peak: uj.mem_peak, - root_job: None, - leaf_jobs: None, - tag: uj.tag, - concurrent_limit: uj.concurrent_limit, - concurrency_time_window_s: uj.concurrency_time_window_s, - timeout: None, - flow_step_id: None, - cache_ttl: None, - cache_ignore_s3_path: None, - priority: uj.priority, - preprocessed: uj.preprocessed, - runnable_settings_handle: uj.runnable_settings_handle, - }, - )), - t => panic!("job type {} not valid", t), - } - } -} #[derive(Deserialize)] struct CancelJob { reason: Option, @@ -3809,94 +2894,6 @@ pub enum DynamicSelectRunnableRef { Inline { code: String, lang: Option }, } -pub struct QueryOrBody(pub Option); - -#[axum::async_trait] -impl FromRequest for QueryOrBody -where - D: DeserializeOwned, - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request( - req: Request, - state: &S, - ) -> std::result::Result { - return if req.method() == axum::http::Method::GET { - let Query(InPayload { payload }) = Query::from_request(req, state) - .await - .map_err(IntoResponse::into_response)?; - payload - .map(|p| { - decode_payload(p) - .map(QueryOrBody) - .map_err(|err| (StatusCode::BAD_REQUEST, format!("{err:#?}"))) - .map_err(IntoResponse::into_response) - }) - .unwrap_or(Ok(QueryOrBody(None))) - } else { - Json::from_request(req, state) - .await - .map(|Json(v)| QueryOrBody(Some(v))) - .map_err(IntoResponse::into_response) - }; - - #[derive(Deserialize)] - struct InPayload { - payload: Option, - } - } -} - -fn decode_payload(t: String) -> anyhow::Result { - let vec = base64::engine::general_purpose::STANDARD - .decode(t) - .context("invalid base64")?; - serde_json::from_slice(vec.as_slice()).context("invalid json") -} - -pub fn add_raw_string( - raw_string: Option, - mut args: serde_json::Map, -) -> serde_json::Map { - if let Some(raw_string) = raw_string { - args.insert( - "raw_string".to_string(), - serde_json::Value::String(raw_string), - ); - } - return args; -} - -pub async fn check_tag_available_for_workspace( - db: &DB, - w_id: &str, - tag: &Option, - authed: &ApiAuthed, -) -> error::Result<()> { - if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { - let tags = get_scope_tags(authed); - check_tag_available_for_workspace_internal(&db, w_id, tag, &authed.email, tags).await - } else { - Ok(()) - } -} - -#[cfg(feature = "enterprise")] -pub async fn check_license_key_valid() -> error::Result<()> { - use windmill_common::ee_oss::LICENSE_KEY_VALID; - - let valid = *LICENSE_KEY_VALID.read().await; - if !valid { - return Err(Error::BadRequest( - "License key is not valid. Go to your superadmin settings to update your license key." - .to_string(), - )); - } - Ok(()) -} - use windmill_common::flows::InputTransform; #[derive(Deserialize)] @@ -4169,95 +3166,6 @@ async fn batch_rerun_handle_job( )) } -/// Set the memory_id in flow_status for agent memory persistence -async fn set_flow_memory_id( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - job_id: Uuid, - memory_id: Uuid, -) -> error::Result<()> { - sqlx::query!( - "UPDATE v2_job_status - SET flow_status = jsonb_set( - flow_status, - '{memory_id}', - to_jsonb($2::uuid) - ) - WHERE id = $1", - job_id, - memory_id - ) - .execute(&mut **tx) - .await?; - Ok(()) -} - -/// Apply flow-specific query parameters after job creation -pub async fn process_flow_run_query_params( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - job_id: Uuid, - run_query: &RunJobQuery, -) -> error::Result<()> { - // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { - set_flow_memory_id(tx, job_id, memory_id).await?; - } - Ok(()) -} - -async fn handle_chat_conversation_messages( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - authed: &ApiAuthed, - w_id: &str, - flow_path: &str, - run_query: &RunJobQuery, - user_message_raw: Option<&Box>, -) -> error::Result<()> { - let memory_id = run_query.memory_id.ok_or_else(|| { - windmill_common::error::Error::BadRequest( - "memory_id is required for chat-enabled flows".to_string(), - ) - })?; - - let user_message_raw = user_message_raw.ok_or_else(|| { - windmill_common::error::Error::BadRequest( - "user_message argument is required for chat-enabled flows".to_string(), - ) - })?; - - // Deserialize the RawValue to get the actual string without quotes - let user_message: String = serde_json::from_str(user_message_raw.get()).map_err(|e| { - windmill_common::error::Error::BadRequest(format!( - "Failed to deserialize user_message: {}", - e - )) - })?; - - // Create conversation with provided ID (or get existing one) - flow_conversations::get_or_create_conversation_with_id( - tx, - w_id, - flow_path, - &authed.username, - &user_message, - memory_id, - ) - .await?; - - // Create user message - add_message_to_conversation_tx( - tx, - memory_id, - None, - &user_message, - MessageType::User, - None, - true, - ) - .await?; - - Ok(()) -} - pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -4292,199 +3200,6 @@ pub async fn run_flow_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } -pub async fn run_flow<'c>( - authed: &ApiAuthed, - db: &DB, - tx_o: Option>, - user_db: UserDB, - w_id: &str, - flow_path: &str, - flow_version_info: FlowVersionInfo, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - let FlowVersionInfo { - version, - tag, - dedicated_worker, - has_preprocessor, - chat_input_enabled, - on_behalf_of_email, - edited_by, - early_return, - .. - } = flow_version_info; - - let tag = run_query.tag.clone().or(tag); - - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - let scheduled_for = run_query.get_scheduled_for(&db).await?; - - let return_tx = tx_o.is_some(); - - let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { - ( - &authed.email, - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Transaction(tx), - ) - } else if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { - ( - on_behalf_of_email, - username_to_permissioned_as(&edited_by), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - &authed.email, - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), - ) - }; - - let (uuid, mut tx) = push( - &db, - tx, - &w_id, - JobPayload::Flow { - path: flow_path.to_string(), - dedicated_worker, - version, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - }, - PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - scheduled_for, - None, - run_query.parent_job, - None, - run_query.root_job, - run_query.job_id, - false, - false, - None, - !run_query.invisible_to_owner.unwrap_or(false), - tag, - None, - None, - None, - push_authed.as_ref(), - false, - None, - trigger, - run_query.suspended_mode, - ) - .await?; - - // Set memory_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { - set_flow_memory_id(&mut tx, uuid, memory_id).await?; - } - - // Handle conversation messages for chat-enabled flows - if chat_input_enabled.unwrap_or(false) { - handle_chat_conversation_messages( - &mut tx, - &authed, - &w_id, - &flow_path.to_string(), - &run_query, - args.args.get("user_message"), - ) - .await?; - } - - // If we were given a transaction, return it; otherwise commit it - if return_tx { - Ok((uuid, early_return, Some(tx))) - } else { - tx.commit().await?; - Ok((uuid, early_return, None)) - } -} - -pub async fn run_flow_and_wait_result( - authed: &ApiAuthed, - db: &DB, - user_db: UserDB, - w_id: &str, - flow_path: &str, - flow_version_info: FlowVersionInfo, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result { - let (uuid, early_return, _) = run_flow( - authed, - db, - None, - user_db, - w_id, - flow_path, - flow_version_info, - run_query, - args, - trigger, - ) - .await?; - - run_wait_result(&db, uuid, w_id, early_return, &authed.username).await -} - -pub async fn push_flow_job_by_path_into_queue<'c>( - authed: ApiAuthed, - db: DB, - tx_o: Option>, - user_db: UserDB, - w_id: String, - flow_path: StripPath, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; - - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - - let flow_version_info = - get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) - .await?; - - run_flow( - &authed, - &db, - tx_o, - user_db, - &w_id, - flow_path, - flow_version_info, - run_query, - args, - trigger, - ) - .await -} - pub async fn run_flow_by_version( authed: ApiAuthed, Extension(db): Extension, @@ -4745,112 +3460,6 @@ pub async fn get_args_and_trigger_metadata( Ok((args, trigger_metadata)) } -pub async fn push_script_job_by_path_into_queue<'c>( - authed: ApiAuthed, - db: DB, - tx_o: Option>, - user_db: UserDB, - w_id: String, - script_path: StripPath, - run_query: RunJobQuery, - args: PushArgsOwned, - trigger: Option, -) -> error::Result<( - Uuid, - Option, - Option>, -)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - let script_path = script_path.to_path(); - check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; - - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( - script_path, - Some(userdb_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await?; - let scheduled_for = run_query.get_scheduled_for(&db).await?; - - let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - - let return_tx = tx_o.is_some(); - - let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Transaction(tx), - ) - } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), - ) - }; - - let (uuid, tx) = push( - &db, - tx, - &w_id, - job_payload, - PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - scheduled_for, - None, - run_query.parent_job, - None, - run_query.root_job, - run_query.job_id, - false, - false, - None, - !run_query.invisible_to_owner.unwrap_or(false), - tag, - timeout, - None, - // If the job has a parent job, set priority to 2 as it may be ran synchronously and block a current worker until being executed. Flow steps have a priority of 1 so this is higher. - if run_query.parent_job.is_some() || run_query.root_job.is_some() { - Some(2) - } else { - None - }, - push_authed.as_ref(), - false, - None, - trigger, - run_query.suspended_mode, - ) - .await?; - - // If we were given a transaction, return it; otherwise commit it - if return_tx { - Ok((uuid, delete_after_use, Some(tx))) - } else { - tx.commit().await?; - Ok((uuid, delete_after_use, None)) - } -} - #[derive(Deserialize)] pub struct WorkflowAsCodeQuery { pub skip_update: Option, @@ -5062,327 +3671,7 @@ pub async fn run_workflow_as_code( Ok((StatusCode::CREATED, uuid.to_string())) } -struct Guard { - done: bool, - id: Uuid, - w_id: String, - db: DB, - username: String, -} - -impl Drop for Guard { - fn drop(&mut self) { - if !&self.done { - let id = self.id; - let w_id = self.w_id.clone(); - let db = self.db.clone(); - let username = self.username.clone(); - - tracing::info!("http connection broke, marking job {id} as canceled"); - tokio::spawn(async move { - let cancel_f = async { - let tx = db.begin().await?; - let (tx, _) = cancel_job( - &username, - Some("http connection broke".to_string()), - id, - &w_id, - tx, - &db, - false, - false, - ) - .await?; - tx.commit().await?; - Ok::<_, anyhow::Error>(()) - }; - - if let Err(e) = cancel_f.await { - tracing::error!( - "Error marking job as canceled after http connection broke: {e}" - ); - } - }); - } - } -} - -use std::sync::Arc; -use tokio::sync::RwLock; - lazy_static::lazy_static! { - pub static ref TIMEOUT_WAIT_RESULT: Arc>> = Arc::new(RwLock::new( - std::env::var("TIMEOUT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse::().ok()) - )); -} - -#[derive(Deserialize)] -pub struct WindmillCompositeResult { - windmill_status_code: Option, - windmill_content_type: Option, - windmill_headers: Option>, - result: Option>, -} - -pub async fn run_wait_result_internal( - db: &DB, - uuid: Uuid, - w_id: &str, - node_id_for_empty_return: Option, - username: &str, -) -> error::Result<(Box, bool)> { - let mut result = None; - let mut success = false; - let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600); - let timeout_ms = if timeout <= 0 { - 2000 - } else { - (timeout * 1000) as u64 - }; - - let mut g = Guard { - done: false, - id: uuid, - w_id: w_id.to_string(), - db: db.clone(), - username: username.to_string(), - }; - - let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000; - let mut accumulated_delay = 0 as u64; - - loop { - if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() { - let result_and_success = get_result_and_success_by_id_from_flow( - &db, - w_id, - &uuid, - node_id_for_empty_return, - None, - ) - .await - .ok(); - if let Some((r, s)) = result_and_success { - result = Some(r); - success = s; - } - } - - if result.is_none() { - let row = sqlx::query!( - " - SELECT - result AS \"result: sqlx::types::Json>\", - result_columns, - status = 'success' AS \"success!\" - FROM - v2_job_completed - WHERE - id = $1 AND - workspace_id = $2 - ", - uuid, - &w_id - ) - .fetch_optional(db) - .await?; - if let Some(mut raw_result) = row { - format_result( - raw_result.result_columns.as_ref(), - raw_result.result.as_mut(), - ); - result = raw_result.result.map(|x| x.0); - success = raw_result.success; - } - } - - if result.is_some() { - break; - } - - let delay = if accumulated_delay <= fast_poll_duration { - *WAIT_RESULT_FAST_POLL_INTERVAL_MS - } else { - *WAIT_RESULT_SLOW_POLL_INTERVAL_MS - }; - accumulated_delay += delay; - if accumulated_delay > timeout_ms { - break; - }; - tokio::time::sleep(core::time::Duration::from_millis(delay)).await; - } - - if let Some(result) = result { - g.done = true; - Ok((result, success)) - } else { - Err(Error::ExecutionErr(format!("timeout after {}s", timeout))) - } -} - -pub fn result_to_response(result: Box, success: bool) -> error::Result { - let composite_result = serde_json::from_str::(result.get()); - match composite_result { - Ok(WindmillCompositeResult { - windmill_status_code, - windmill_content_type, - windmill_headers, - result: result_value, - }) => { - if windmill_content_type.is_none() - && windmill_status_code.is_none() - && windmill_headers.is_none() - { - return Ok(( - if success { - StatusCode::OK - } else { - StatusCode::UNPROCESSABLE_ENTITY - }, - Json(result), - ) - .into_response()); - } - - let status_code_or_default = windmill_status_code - .map(|val| match StatusCode::from_u16(val) { - Ok(sc) => Ok(sc), - Err(_) => Err(Error::ExecutionErr("Invalid status code".to_string())), - }) - .unwrap_or_else(|| { - if !success { - Ok(StatusCode::UNPROCESSABLE_ENTITY) - } else if result_value.is_some() { - Ok(StatusCode::OK) - } else { - Ok(StatusCode::NO_CONTENT) - } - })?; - - let mut headers = HeaderMap::new(); - - if let Some(windmill_headers) = windmill_headers { - for (k, v) in windmill_headers { - let k = HeaderName::from_str(k.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid header name {k}: {err}")) - })?; - let v = HeaderValue::from_str(v.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid header value {v}: {err}")) - })?; - headers.insert(k, v); - } - } - - if let Some(content_type) = windmill_content_type { - let serialized_json_result = result_value - .map(|val| val.get().to_owned()) - .unwrap_or_else(String::new); - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let serialized_result = - serde_json::from_str::(serialized_json_result.as_str()) - .ok() - .unwrap_or(serialized_json_result); - headers.insert( - http::header::CONTENT_TYPE, - HeaderValue::from_str(content_type.as_str()).map_err(|err| { - Error::internal_err(format!("Invalid content type {content_type}: {err}")) - })?, - ); - return Ok((status_code_or_default, headers, serialized_result).into_response()); - } - if let Some(result_value) = result_value { - return Ok((status_code_or_default, headers, Json(result_value)).into_response()); - } else { - Ok((status_code_or_default, headers).into_response()) - } - } - _ => Ok(( - if success { - StatusCode::OK - } else { - StatusCode::UNPROCESSABLE_ENTITY - }, - Json(result), - ) - .into_response()), - } -} - -pub async fn run_wait_result( - db: &DB, - uuid: Uuid, - w_id: &str, - node_id_for_empty_return: Option, - username: &str, -) -> error::Result { - let (result, success) = - run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username).await?; - - result_to_response(result, success) -} - -pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { - sqlx::query!( - "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", - job_uuid, - ) - .execute(db) - .await?; - sqlx::query!( - "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", - job_uuid, - ) - .execute(db) - .await?; - sqlx::query!( - "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", - job_uuid, - ) - .execute(db) - .await?; - Ok(()) -} - -pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { - if let Some(limit) = queue_limit { - let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() - OR (q.suspend_until IS NOT NULL - AND (q.suspend <= 0 OR q.suspend_until <= now())))", - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - if count > queue_limit.unwrap() { - return Err(Error::internal_err(format!( - "Number of queued job is too high: {count} > {limit}" - ))); - } - } - Ok(()) -} - -lazy_static::lazy_static! { - pub static ref QUEUE_LIMIT_WAIT_RESULT: Option = std::env::var("QUEUE_LIMIT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse().ok()); - pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(50); - pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(2); - pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(200); - static ref JOB_VIEW_AUDIT_LOGS: bool = std::env::var("JOB_VIEW_AUDIT_LOGS") .ok() .and_then(|x| x.parse().ok()) @@ -7457,75 +5746,6 @@ pub async fn run_job_by_hash_inner( Ok((uuid, delete_after_use)) } -#[derive(Deserialize)] -pub struct JobUpdateQuery { - pub running: Option, - pub log_offset: Option, - pub stream_offset: Option, - pub get_progress: Option, - pub no_logs: Option, - pub only_result: Option, - pub fast: Option, - pub is_flow: Option, - pub poll_delay_ms: Option, -} - -#[derive(Serialize, Debug)] -pub struct JobUpdate { - #[serde(skip_serializing_if = "Option::is_none")] - pub running: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub completed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub new_logs: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub new_result_stream: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub log_offset: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_offset: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub mem_peak: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub progress: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_status: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub workflow_as_code_status: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub job: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub only_result: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub flow_stream_job_id: Option, -} - -impl JobUpdate { - pub fn hash_str(&self) -> String { - let mut hasher = DefaultHasher::new(); - self.hash(&mut hasher); - format!("{:x}", hasher.finish()) - } -} - -impl Hash for JobUpdate { - fn hash(&self, state: &mut H) { - self.running.hash(state); - self.completed.hash(state); - self.log_offset.hash(state); - self.mem_peak.hash(state); - self.progress.hash(state); - self.stream_offset.hash(state); - self.flow_stream_job_id.hash(state); - if !self.completed.unwrap_or(false) { - self.flow_status.as_ref().map(|x| x.get().hash(state)); - self.workflow_as_code_status - .as_ref() - .map(|x| x.get().hash(state)); - } - } -} - async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result { if file_p.contains("..") { return Err(error::Error::BadRequest("Invalid path".to_string())); @@ -7693,21 +5913,6 @@ async fn get_job_update_sse( .unwrap()) } -#[derive(Serialize)] -#[serde(tag = "type", rename_all = "lowercase")] -pub enum JobUpdateSSEStream { - Update(JobUpdate), - Error { error: String }, - NotFound, - Timeout, - Ping, -} - -lazy_static::lazy_static! { - pub static ref TIMEOUT_SSE_STREAM: u64 = - std::env::var("TIMEOUT_SSE_STREAM").unwrap_or("60".to_string()).parse::().unwrap_or(60); -} - pub fn start_job_update_sse_stream( opt_authed: Option, opt_tokened: OptTokened, @@ -8285,251 +6490,6 @@ async fn get_job_update_data( } } -pub fn filter_list_completed_query( - mut sqlb: SqlBuilder, - lq: &ListCompletedQuery, - w_id: &str, - join_outstanding_wait_times: bool, -) -> SqlBuilder { - sqlb.join("v2_job") - .on_eq("v2_job_completed.id", "v2_job.id"); - - if join_outstanding_wait_times { - sqlb.left() - .join("outstanding_wait_time") - .on_eq("v2_job_completed.id", "outstanding_wait_time.job_id"); - } - - if let Some(label) = &lq.label { - if lq.allow_wildcards.unwrap_or(false) { - let wh = format!( - "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", - &label.replace("*", "%").replace("'", "''") - ); - sqlb.and_where("result ? 'wm_labels'"); - sqlb.and_where(&wh); - } else { - let mut wh = format!("result->'wm_labels' ? "); - wh.push_str(&format!("'{}'", &label.replace("'", "''"))); - sqlb.and_where("result ? 'wm_labels'"); - sqlb.and_where(&wh); - } - } - - if let Some(worker) = &lq.worker { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job_completed.worker", worker.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job_completed.worker", "?".bind(worker)); - } - } - - if w_id != "admins" || !lq.all_workspaces.is_some_and(|x| x) { - sqlb.and_where_eq("v2_job_completed.workspace_id", "?".bind(&w_id)) - .and_where_eq("v2_job.workspace_id", "?".bind(&w_id)); - } - - if let Some(p) = &lq.schedule_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - sqlb.and_where_eq("trigger_kind", "'schedule'"); - } - - if let Some(ps) = &lq.script_path_start { - sqlb.and_where_like_left("runnable_path", ps); - } - if let Some(p) = &lq.script_path_exact { - sqlb.and_where_eq("runnable_path", "?".bind(p)); - } - if let Some(h) = &lq.script_hash { - sqlb.and_where_eq("runnable_id", "?".bind(h)); - } - if let Some(t) = &lq.tag { - if lq.allow_wildcards.unwrap_or(false) { - sqlb.and_where_like_left("v2_job.tag", t.replace("*", "%")); - } else { - sqlb.and_where_eq("v2_job.tag", "?".bind(t)); - } - } - - if let Some(cb) = &lq.created_by { - sqlb.and_where_eq("created_by", "?".bind(cb)); - } - if let Some(r) = &lq.success { - if *r { - sqlb.and_where_eq("status", "'success'") - .or_where_eq("status", "'skipped'"); - } else { - sqlb.and_where_eq("status", "'failure'") - .or_where_eq("status", "'canceled'"); - } - } - if let Some(pj) = &lq.parent_job { - sqlb.and_where_eq("parent_job", "?".bind(pj)); - } - if let Some(dt) = &lq.started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.started_after { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_before { - sqlb.and_where_le("started_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_or_started_after { - let ts = dt.to_rfc3339(); - sqlb.and_where(format!( - "(created_at >= '{}' OR started_at >= '{}')", - ts.replace("'", "''"), - ts.replace("'", "''") - )); - } - - if let Some(dt) = &lq.created_before { - sqlb.and_where_le("created_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.created_after { - sqlb.and_where_ge("created_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.created_or_started_after_completed_jobs { - sqlb.and_where_ge("started_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(dt) = &lq.completed_after { - sqlb.and_where_ge("completed_at", "?".bind(&dt.to_rfc3339())); - } - if let Some(dt) = &lq.completed_before { - sqlb.and_where_le("completed_at", "?".bind(&dt.to_rfc3339())); - } - - if let Some(sk) = &lq.is_skipped { - if *sk { - sqlb.and_where_eq("status", "'skipped'"); - } else { - sqlb.and_where_ne("status", "'skipped'"); - } - } - if let Some(fs) = &lq.is_flow_step { - if *fs { - sqlb.and_where_is_not_null("flow_step_id"); - } else { - sqlb.and_where_is_null("flow_step_id"); - } - } - if let Some(fs) = &lq.has_null_parent { - if *fs { - sqlb.and_where_is_null("parent_job"); - } - } - if let Some(jk) = &lq.job_kinds { - sqlb.and_where_in( - "kind", - &jk.split(',').into_iter().map(quote).collect::>(), - ); - } - - if let Some(args) = &lq.args { - sqlb.and_where("args @> ?".bind(&args.replace("'", "''"))); - } - - if let Some(result) = &lq.result { - sqlb.and_where("result @> ?".bind(&result.replace("'", "''"))); - } - - if lq.is_not_schedule.unwrap_or(false) { - sqlb.and_where("trigger_kind IS DISTINCT FROM 'schedule'"); - } - - if let Some(tk) = &lq.trigger_kind { - sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); - } - - if let Some(tp) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(tp)); - } - - sqlb -} - -pub fn list_completed_jobs_query( - w_id: &str, - per_page: Option, - offset: usize, - lq: &ListCompletedQuery, - fields: &[&str], - join_outstanding_wait_times: bool, - tags: Option>, -) -> SqlBuilder { - let mut sqlb = SqlBuilder::select_from("v2_job_completed") - .fields(fields) - .order_by( - if lq.completed_before.is_some() || lq.completed_after.is_some() { - "v2_job_completed.completed_at" - } else { - "v2_job.created_at" - }, - lq.order_desc.unwrap_or(true), - ) - .offset(offset) - .clone(); - if let Some(per_page) = per_page { - sqlb.limit(per_page); - } - - if let Some(tags) = tags { - sqlb.and_where_in( - "v2_job.tag", - &tags.iter().map(|x| quote(x)).collect::>(), - ); - } - - filter_list_completed_query(sqlb, lq, w_id, join_outstanding_wait_times) -} -#[derive(Deserialize, Clone)] -pub struct ListCompletedQuery { - pub script_path_start: Option, - pub script_path_exact: Option, - pub script_hash: Option, - pub created_by: Option, - pub started_before: Option>, - pub started_after: Option>, - pub created_before: Option>, - pub created_after: Option>, - pub created_or_started_before: Option>, - pub created_or_started_after: Option>, - pub created_or_started_after_completed_jobs: Option>, - pub created_before_queue: Option>, - pub created_after_queue: Option>, - pub completed_after: Option>, - pub completed_before: Option>, - pub success: Option, - pub running: Option, - pub parent_job: Option, - pub order_desc: Option, - pub job_kinds: Option, - pub is_skipped: Option, - pub is_flow_step: Option, - pub suspended: Option, - pub schedule_path: Option, - // filter by matching a subset of the args using base64 encoded json subset - pub args: Option, - // filter by matching a subset of the result using base64 encoded json subset - pub result: Option, - pub tag: Option, - pub scheduled_for_before_now: Option, - pub all_workspaces: Option, - pub has_null_parent: Option, - pub label: Option, - pub is_not_schedule: Option, - pub concurrency_key: Option, - pub worker: Option, - pub allow_wildcards: Option, - pub trigger_kind: Option, - pub trigger_path: Option, - pub include_args: Option, -} - async fn list_completed_jobs( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/jobs_export.rs b/backend/windmill-api/src/jobs_export.rs index f1b83cba05..bb7e62cfa9 100644 --- a/backend/windmill-api/src/jobs_export.rs +++ b/backend/windmill-api/src/jobs_export.rs @@ -1,827 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use axum::{ - extract::{Path, Query}, - Extension, Json, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use uuid::Uuid; -use windmill_common::{ - db::UserDB, - error, - jobs::{JobKind, JobStatus, JobTriggerKind}, - scripts::ScriptLang, - utils::{paginate, paginate_without_limits, require_admin, Pagination}, -}; - -use crate::db::ApiAuthed; - -#[derive(Serialize, Deserialize)] -pub struct ExportableCompletedJob { - pub id: Uuid, - pub raw_code: Option, - pub raw_lock: Option, - pub raw_flow: Option>>, - pub tag: String, - pub created_at: chrono::DateTime, - pub created_by: String, - pub permissioned_as: String, - pub permissioned_as_email: String, - pub kind: JobKind, - pub runnable_id: Option, - pub runnable_path: Option, - pub parent_job: Option, - pub root_job: Option, - pub script_lang: Option, - pub script_entrypoint_override: Option, - pub flow_step: Option, - pub flow_step_id: Option, - pub flow_innermost_root_job: Option, - pub trigger: Option, - pub trigger_kind: Option, - pub same_worker: bool, - pub visible_to_owner: bool, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub cache_ttl: Option, - pub timeout: Option, - pub priority: Option, - pub preprocessed: Option, - pub args: Option>>, - pub labels: Option>, - pub pre_run_error: Option, - pub duration_ms: i64, - pub result: Option>>, - pub deleted: bool, - pub canceled_by: Option, - pub canceled_reason: Option, - pub flow_status: Option>>, - pub started_at: Option>, - pub memory_peak: Option, - pub status: JobStatus, - pub completed_at: chrono::DateTime, - pub worker: Option, - pub workflow_as_code_status: Option>>, - pub result_columns: Option>, - pub retries: Option>, - pub extras: Option>>, - pub logs: Option, - pub log_offset: Option, - pub log_file_index: Option>, -} - -#[derive(Serialize, Deserialize)] -pub struct ExportableQueuedJob { - // v2_job columns (excluding workspace_id) - pub id: Uuid, - pub raw_code: Option, - pub raw_lock: Option, - pub raw_flow: Option>>, - pub tag: String, - pub created_at: chrono::DateTime, - pub created_by: String, - pub permissioned_as: String, - pub permissioned_as_email: String, - pub kind: JobKind, - pub runnable_id: Option, - pub runnable_path: Option, - pub parent_job: Option, - pub root_job: Option, - pub script_lang: Option, - pub script_entrypoint_override: Option, - pub flow_step: Option, - pub flow_step_id: Option, - pub flow_innermost_root_job: Option, - pub trigger: Option, - pub trigger_kind: Option, - pub same_worker: bool, - pub visible_to_owner: bool, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub cache_ttl: Option, - pub timeout: Option, - pub priority: Option, - pub preprocessed: Option, - pub args: Option>>, - pub labels: Option>, - pub pre_run_error: Option, - - // v2_job_queue columns (excluding workspace_id and id/created_at/tag/priority) - pub started_at: Option>, - pub scheduled_for: chrono::DateTime, - pub running: bool, - pub canceled_by: Option, - pub canceled_reason: Option, - pub suspend: Option, - pub suspend_until: Option>, - pub worker: Option, - pub extras: Option>>, - - // v2_job_runtime columns (excluding id) - pub ping: Option>, - pub memory_peak: Option, - - // v2_job_status columns (excluding id) - pub flow_status: Option>>, - pub flow_leaf_jobs: Option>>, - pub workflow_as_code_status: Option>>, - - // concurrency_key table - pub concurrency_key: Option, -} - -pub async fn export_completed_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> error::JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let (per_page, offset) = paginate(pagination); - let per_page = per_page as i64; - let offset = offset as i64; - - let mut tx = user_db.begin(&authed).await?; - - let jobs = sqlx::query_as!( - ExportableCompletedJob, - r#" - SELECT - v2_job.id, - v2_job.raw_code, - v2_job.raw_lock, - v2_job.raw_flow as "raw_flow: _", - v2_job.tag, - v2_job.created_at, - v2_job.created_by, - v2_job.permissioned_as, - v2_job.permissioned_as_email, - v2_job.kind as "kind: _", - v2_job.runnable_id, - v2_job.runnable_path, - v2_job.parent_job, - v2_job.root_job, - v2_job.script_lang as "script_lang: _", - v2_job.script_entrypoint_override, - v2_job.flow_step, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job.trigger, - v2_job.trigger_kind as "trigger_kind: _", - v2_job.same_worker, - v2_job.visible_to_owner, - v2_job.concurrent_limit, - v2_job.concurrency_time_window_s, - v2_job.cache_ttl, - v2_job.timeout, - v2_job.priority, - v2_job.preprocessed, - v2_job.args as "args: _", - v2_job.labels, - v2_job.pre_run_error, - v2_job_completed.duration_ms, - v2_job_completed.result as "result: _", - v2_job_completed.deleted, - v2_job_completed.canceled_by, - v2_job_completed.canceled_reason, - v2_job_completed.flow_status as "flow_status: _", - v2_job_completed.started_at, - v2_job_completed.memory_peak, - v2_job_completed.status as "status: _", - v2_job_completed.completed_at, - v2_job_completed.worker, - v2_job_completed.workflow_as_code_status as "workflow_as_code_status: _", - v2_job_completed.result_columns, - v2_job_completed.retries, - v2_job_completed.extras as "extras: _", - job_logs.logs AS logs, - job_logs.log_offset, - job_logs.log_file_index - FROM v2_job_completed - INNER JOIN v2_job ON v2_job.id = v2_job_completed.id - LEFT JOIN v2_job_status ON v2_job_completed.id = v2_job_status.id - LEFT JOIN job_logs ON job_logs.job_id = v2_job.id - WHERE v2_job_completed.workspace_id = $1 - ORDER BY v2_job.created_at DESC - LIMIT $2 - OFFSET $3 - "#, - w_id, - per_page, - offset - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(jobs)) -} - -pub async fn export_queued_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(pagination): Query, -) -> error::JsonResult> { - require_admin(authed.is_admin, &authed.username)?; - - let (per_page, offset) = paginate_without_limits(pagination); - let per_page = per_page as i64; - let offset = offset as i64; - - let mut tx = user_db.begin(&authed).await?; - - let jobs = sqlx::query_as!( - ExportableQueuedJob, - r#" - SELECT - v2_job.id, - v2_job.raw_code, - v2_job.raw_lock, - v2_job.raw_flow as "raw_flow: _", - v2_job.tag, - v2_job.created_at, - v2_job.created_by, - v2_job.permissioned_as, - v2_job.permissioned_as_email, - v2_job.kind as "kind: _", - v2_job.runnable_id, - v2_job.runnable_path, - v2_job.parent_job, - v2_job.root_job, - v2_job.script_lang as "script_lang: _", - v2_job.script_entrypoint_override, - v2_job.flow_step, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job."trigger", - v2_job.trigger_kind as "trigger_kind: _", - v2_job.same_worker, - v2_job.visible_to_owner, - v2_job.concurrent_limit, - v2_job.concurrency_time_window_s, - v2_job.cache_ttl, - v2_job.timeout, - v2_job.priority, - v2_job.preprocessed, - v2_job.args as "args: _", - v2_job.labels, - v2_job.pre_run_error, - - v2_job_queue.started_at, - v2_job_queue.scheduled_for, - v2_job_queue.running, - v2_job_queue.canceled_by, - v2_job_queue.canceled_reason, - v2_job_queue.suspend, - v2_job_queue.suspend_until, - v2_job_queue.worker, - v2_job_queue.extras as "extras: _", - - v2_job_runtime.ping, - v2_job_runtime.memory_peak, - - v2_job_status.flow_status as "flow_status: _", - v2_job_status.flow_leaf_jobs as "flow_leaf_jobs: _", - v2_job_status.workflow_as_code_status as "workflow_as_code_status: _", - - concurrency_key.key as "concurrency_key?" - FROM v2_job_queue - INNER JOIN v2_job ON v2_job.id = v2_job_queue.id - LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id - LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id - LEFT JOIN concurrency_key ON concurrency_key.job_id = v2_job_queue.id - WHERE v2_job_queue.workspace_id = $1 - AND v2_job_queue.running = false - AND v2_job.parent_job IS NULL - AND v2_job.trigger_kind IS DISTINCT FROM 'schedule' - ORDER BY v2_job.created_at DESC - LIMIT $2 - OFFSET $3 - "#, - w_id, - per_page, - offset - ) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(jobs)) -} - -pub async fn import_completed_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(jobs): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - for job in jobs { - // Insert into v2_job table - sqlx::query!( - r#" - INSERT INTO v2_job ( - id, - raw_code, - raw_lock, - raw_flow, - tag, - workspace_id, - created_at, - created_by, - permissioned_as, - permissioned_as_email, - kind, - runnable_id, - runnable_path, - parent_job, - root_job, - script_lang, - script_entrypoint_override, - flow_step, - flow_step_id, - flow_innermost_root_job, - trigger, - trigger_kind, - same_worker, - visible_to_owner, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - timeout, - priority, - preprocessed, - args, - labels, - pre_run_error - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.raw_code, - job.raw_lock, - job.raw_flow as _, - &job.tag, - &w_id, - job.created_at, - &job.created_by, - &job.permissioned_as, - job.permissioned_as_email, - job.kind as _, - job.runnable_id, - job.runnable_path, - job.parent_job, - job.root_job, - job.script_lang as _, - job.script_entrypoint_override, - job.flow_step, - job.flow_step_id, - job.flow_innermost_root_job, - job.trigger, - job.trigger_kind as _, - job.same_worker, - job.visible_to_owner, - job.concurrent_limit, - job.concurrency_time_window_s, - job.cache_ttl, - job.timeout, - job.priority, - job.preprocessed, - job.args as _, - job.labels as _, - job.pre_run_error - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_completed table - sqlx::query!( - r#" - INSERT INTO v2_job_completed ( - id, - workspace_id, - started_at, - completed_at, - duration_ms, - result, - deleted, - canceled_by, - canceled_reason, - flow_status, - memory_peak, - status, - worker, - workflow_as_code_status, - result_columns, - retries, - extras - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - &w_id, - job.started_at, - job.completed_at, - job.duration_ms, - job.result as _, - job.deleted, - job.canceled_by, - job.canceled_reason, - job.flow_status as _, - job.memory_peak, - job.status as _, - job.worker, - job.workflow_as_code_status as _, - job.result_columns as _, - job.retries as _, - job.extras as _ - ) - .execute(&mut *tx) - .await?; - - // Insert into job_logs table - if let Some(logs) = &job.logs { - sqlx::query!( - r#" - INSERT INTO job_logs ( - job_id, - workspace_id, - logs, - log_offset, - log_file_index - ) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (job_id) DO NOTHING - "#, - job.id, - &w_id, - logs, - job.log_offset, - job.log_file_index as _ - ) - .execute(&mut *tx) - .await?; - } - } - - tx.commit().await?; - Ok(format!("Successfully imported jobs")) -} - -pub async fn import_queued_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(jobs): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - for job in jobs { - // Insert into v2_job table - // Insert into v2_job table, fields and order adapted to match ExportableQueuedJob - sqlx::query!( - r#" - INSERT INTO v2_job ( - id, - raw_code, - raw_lock, - raw_flow, - tag, - workspace_id, - created_at, - created_by, - permissioned_as, - permissioned_as_email, - kind, - runnable_id, - runnable_path, - parent_job, - root_job, - script_lang, - script_entrypoint_override, - flow_step, - flow_step_id, - flow_innermost_root_job, - trigger, - trigger_kind, - same_worker, - visible_to_owner, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - timeout, - priority, - preprocessed, - args, - labels, - pre_run_error - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, - $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, - $31, $32, $33 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.raw_code, - job.raw_lock, - job.raw_flow as _, - &job.tag, - &w_id, - job.created_at, - &job.created_by, - &job.permissioned_as, - &job.permissioned_as_email, - job.kind as _, - job.runnable_id, - job.runnable_path, - job.parent_job, - job.root_job, - job.script_lang as _, - job.script_entrypoint_override, - job.flow_step, - job.flow_step_id, - job.flow_innermost_root_job, - job.trigger, - job.trigger_kind as _, - job.same_worker, - job.visible_to_owner, - job.concurrent_limit, - job.concurrency_time_window_s, - job.cache_ttl, - job.timeout, - job.priority, - job.preprocessed, - job.args as _, - job.labels as _, - job.pre_run_error, - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_queue table - sqlx::query!( - r#" - INSERT INTO v2_job_queue ( - id, - workspace_id, - started_at, - scheduled_for, - running, - canceled_by, - canceled_reason, - suspend, - suspend_until, - worker, - extras, - tag, - priority - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - &w_id, - job.started_at, - job.scheduled_for, - job.running, - job.canceled_by, - job.canceled_reason, - job.suspend, - job.suspend_until, - job.worker, - job.extras as _, - &job.tag, - job.priority - ) - .execute(&mut *tx) - .await?; - - // Insert into v2_job_runtime if any present - if job.ping.is_some() || job.memory_peak.is_some() { - sqlx::query!( - r#" - INSERT INTO v2_job_runtime ( - id, ping, memory_peak - ) VALUES ( - $1, $2, $3 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.ping, - job.memory_peak - ) - .execute(&mut *tx) - .await?; - } - - // Insert into v2_job_status if present - if job.flow_status.is_some() - || job.flow_leaf_jobs.is_some() - || job.workflow_as_code_status.is_some() - { - sqlx::query!( - r#" - INSERT INTO v2_job_status ( - id, flow_status, flow_leaf_jobs, workflow_as_code_status - ) VALUES ( - $1, $2, $3, $4 - ) - ON CONFLICT (id) DO NOTHING - "#, - job.id, - job.flow_status as _, - job.flow_leaf_jobs as _, - job.workflow_as_code_status as _ - ) - .execute(&mut *tx) - .await?; - } - - // Insert concurrency_key if present - if let Some(ref concurrency_key) = job.concurrency_key { - sqlx::query!( - r#" - WITH inserted_concurrency_counter AS ( - INSERT INTO concurrency_counter (concurrency_id, job_uuids) - VALUES ($1, '{}'::jsonb) - ON CONFLICT DO NOTHING - ) - INSERT INTO concurrency_key(key, job_id) - VALUES ($1, $2) - ON CONFLICT (job_id) DO NOTHING - "#, - concurrency_key, - job.id - ) - .execute(&mut *tx) - .await?; - } - } - - tx.commit().await?; - Ok(format!("Successfully imported jobs")) -} - -pub async fn delete_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(job_ids): Json>, -) -> error::Result { - require_admin(authed.is_admin, &authed.username)?; - - if job_ids.is_empty() { - return Ok("No jobs to delete".to_string()); - } - - let mut tx = user_db.begin(&authed).await?; - - // Delete from all job-related tables - // Order matters due to foreign key constraints - - // 1. Delete job logs - let logs_deleted = sqlx::query!( - "DELETE FROM job_logs WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 2. Delete job permissions - let perms_deleted = sqlx::query!( - "DELETE FROM job_perms WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 3. Delete job stats - let stats_deleted = sqlx::query!( - "DELETE FROM job_stats WHERE workspace_id = $1 AND job_id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 4. Delete resume jobs - let resume_deleted = sqlx::query!("DELETE FROM resume_job WHERE job = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 5. Delete from v2_job_runtime - let runtime_deleted = sqlx::query!("DELETE FROM v2_job_runtime WHERE id = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 6. Delete from v2_job_status - let status_deleted = sqlx::query!("DELETE FROM v2_job_status WHERE id = ANY($1)", &job_ids) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 7. Delete from concurrency_key - let concurrency_key_deleted = sqlx::query!( - "DELETE FROM concurrency_key WHERE job_id = ANY($1)", - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 8. Delete from v2_job_queue (for queued jobs) - let queue_deleted = sqlx::query!( - "DELETE FROM v2_job_queue WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 9. Delete from v2_job_completed (for completed jobs) - let completed_deleted = sqlx::query!( - "DELETE FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 10. Delete zombie job counter - let zombie_deleted = sqlx::query!( - "DELETE FROM zombie_job_counter WHERE job_id = ANY($1)", - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - // 11. Finally, delete from v2_job (main table) - let jobs_deleted = sqlx::query!( - "DELETE FROM v2_job WHERE workspace_id = $1 AND id = ANY($2)", - &w_id, - &job_ids - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - tx.commit().await?; - - let total_rows_deleted = logs_deleted - + perms_deleted - + stats_deleted - + resume_deleted - + runtime_deleted - + status_deleted - + concurrency_key_deleted - + queue_deleted - + completed_deleted - + zombie_deleted - + jobs_deleted; - - tracing::info!( - "Successfully deleted {} jobs ({} total rows across all tables) from workspace {}", - job_ids.len(), - total_rows_deleted, - w_id - ); - - Ok(format!( - "Successfully deleted {} jobs ({} total rows across all tables)", - jobs_deleted, total_rows_deleted - )) -} +pub use windmill_api_jobs::jobs_export::*; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a78b6fa2a3..e82763197c 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -7,16 +7,14 @@ */ use crate::db::ApiAuthed; -#[cfg(feature = "enterprise")] -use crate::ee_oss::ExternalJwks; #[cfg(feature = "embedding")] use crate::embeddings::load_embeddings_db; #[cfg(feature = "oauth2")] -use crate::oauth2_oss::AllClients; -#[cfg(feature = "oauth2")] use crate::oauth2_oss::SlackVerifier; #[cfg(feature = "smtp")] use crate::smtp_server_oss::SmtpServer; +#[cfg(feature = "enterprise")] +use windmill_api_auth::ee_oss::ExternalJwks; #[cfg(feature = "mcp")] use crate::mcp::{extract_and_store_workspace_id, setup_mcp_server}; @@ -41,8 +39,6 @@ use axum::response::Response; use axum::{middleware::from_extractor, routing::get, routing::post, Extension, Json, Router}; use db::DB; use reqwest::Client; -#[cfg(feature = "oauth2")] -use std::collections::HashMap; use tokio::task::JoinHandle; use windmill_common::global_settings::load_value_from_global_settings; use windmill_common::global_settings::EMAIL_DOMAIN_SETTING; @@ -150,7 +146,6 @@ mod schedule; #[cfg(feature = "private")] pub mod scim_ee; mod scim_oss; -mod scopes; mod scripts; mod secret_backend_ext; mod service_logs; @@ -187,7 +182,6 @@ mod users; pub mod users_ee; mod users_oss; mod utils; -pub mod var_resource_cache; mod variables; pub mod webhook_util; mod workers; @@ -230,15 +224,11 @@ lazy_static::lazy_static! { } +#[cfg(feature = "oauth2")] +pub use windmill_oauth::OAUTH_CLIENTS; + #[cfg(feature = "oauth2")] lazy_static::lazy_static! { - pub static ref OAUTH_CLIENTS: Arc> = Arc::new(RwLock::new(AllClients { - logins: HashMap::new(), - connects: HashMap::new(), - slack: None - })); - - pub static ref SLACK_SIGNING_SECRET: Option = std::env::var("SLACK_SIGNING_SECRET") .ok() .map(|x| SlackVerifier::new(x).unwrap()); diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index f96d815b00..e7fa3eb13f 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -67,4 +67,3 @@ async fn fix_flow_versioning_migration( } Ok(()) } - diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 570b4c0de7..90419aed4d 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -11,7 +11,7 @@ use windmill_mcp::common::transform::apply_key_transformation; use windmill_mcp::common::types::{ FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, }; -use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpAuth, McpBackend}; +use windmill_mcp::server::{BackendResult, EndpointTool, ErrorData, McpBackend}; use crate::db::ApiAuthed; use crate::jobs::{ @@ -39,36 +39,7 @@ use axum::{ }; use windmill_common::error::JsonResult; -/// Implement McpAuth for ApiAuthed -impl McpAuth for ApiAuthed { - fn username(&self) -> &str { - &self.username - } - - fn email(&self) -> &str { - &self.email - } - - fn is_admin(&self) -> bool { - self.is_admin - } - - fn is_operator(&self) -> bool { - self.is_operator - } - - fn groups(&self) -> &[String] { - &self.groups - } - - fn folders(&self) -> &[(String, bool, bool)] { - &self.folders - } - - fn scopes(&self) -> Option<&[String]> { - self.scopes.as_deref() - } -} +// McpAuth impl for ApiAuthed is in windmill-api-auth (same crate as the type) /// Windmill's MCP backend implementation #[derive(Clone)] diff --git a/backend/windmill-api/src/native_triggers/mod.rs b/backend/windmill-api/src/native_triggers/mod.rs index 8c0b829478..95b4535217 100644 --- a/backend/windmill-api/src/native_triggers/mod.rs +++ b/backend/windmill-api/src/native_triggers/mod.rs @@ -1,1053 +1 @@ -//! Native Triggers Module -//! -//! This module provides integration with external services (like Nextcloud) that can -//! trigger Windmill scripts/flows via webhooks. -//! -//! ## Adding a New Native Trigger Service -//! -//! When adding a new service (e.g., "NewService"), you need to update the following locations: -//! -//! ### 1. This file (mod.rs): -//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block -//! - Add `NewService` variant to `ServiceName` enum -//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` -//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` -//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) -//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) -//! - Update `ServiceName::fmt()` (Display impl) - add match arm -//! -//! ### 2. sync.rs: -//! - Add `sync_service!()` macro call in `sync_all_triggers()` -//! -//! ### 3. handler.rs: -//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` -//! -//! ### 4. Database migration: -//! - Add `'newservice'` to the `native_trigger_service` enum type -//! -//! ### 5. windmill_common (if needed): -//! - Add `NewService` variant to `TriggerKind` enum -//! - Add `'newservice'` to `job_trigger_kind` enum type in migration -//! -//! The generic code (trait definitions, route handlers, database operations) does NOT -//! need modification when adding new services. - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use http::StatusCode; -use itertools::Itertools; -use reqwest::{Client, Method}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::json; -use serde_json::value::RawValue; -use sqlx::{FromRow, PgConnection, Postgres}; -use std::{collections::HashMap, fmt::Debug}; -use strum::{EnumIter, IntoEnumIterator}; -use tokio::task; -use windmill_common::{ - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - variables::{build_crypt, decrypt, encrypt}, - DB, -}; -use windmill_queue::PushArgsOwned; - -#[cfg(feature = "native_trigger")] -use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; - -use crate::db::ApiAuthed; -pub mod handler; -pub mod sync; -pub mod workspace_integrations; - -// Service modules - add new services here: -#[cfg(feature = "native_trigger")] -pub mod nextcloud; -// #[cfg(feature = "native_trigger")] -// pub mod newservice; - -/// Enum of all supported native trigger services. -/// When adding a new service, add a variant here (e.g., `NewService`). -#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ServiceName { - Nextcloud, - // Add new services here: - // NewService, -} - -impl TryFrom for ServiceName { - type Error = Error; - fn try_from(value: String) -> std::result::Result { - // Add new service match arms here: - let service = match value.as_str() { - "nextcloud" => ServiceName::Nextcloud, - // "newservice" => ServiceName::NewService, - _ => { - return Err(anyhow::anyhow!( - "Unknown service, currently supported services are: [{}]", - ServiceName::iter().join(",") - ) - .into()) - } - }; - - Ok(service) - } -} - -impl ServiceName { - /// Returns the lowercase string identifier for this service. - /// Add new service match arms here. - pub fn as_str(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "nextcloud", - // ServiceName::NewService => "newservice", - } - } - - /// Returns the corresponding TriggerKind for this service. - /// Requires adding the variant to TriggerKind in windmill_common. - pub fn as_trigger_kind(&self) -> TriggerKind { - match self { - ServiceName::Nextcloud => TriggerKind::Nextcloud, - // ServiceName::NewService => TriggerKind::NewService, - } - } - - /// Returns the corresponding JobTriggerKind for this service. - /// Requires adding the variant to JobTriggerKind in windmill_common. - pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { - match self { - ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, - // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, - } - } - - /// Returns the OAuth token endpoint path for this service. - /// Used for building OAuth clients dynamically. - pub fn token_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", - // ServiceName::NewService => "/oauth/token", - } - } - - /// Returns the OAuth authorization endpoint path for this service. - /// Used for building OAuth authorization URLs. - pub fn auth_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/authorize", - // ServiceName::NewService => "/oauth/authorize", - } - } -} - -impl std::fmt::Display for ServiceName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct NativeTrigger { - pub external_id: String, - pub workspace_id: String, - pub service_name: ServiceName, - pub script_path: String, - pub is_flow: bool, - pub webhook_token_prefix: String, - pub service_config: Option, - pub error: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NativeTriggerConfig { - pub script_path: String, - pub is_flow: bool, - pub webhook_token: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct NativeTriggerData { - pub script_path: String, - pub is_flow: bool, - pub service_config: C, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WorkspaceIntegration { - pub workspace_id: String, - pub service_name: ServiceName, - pub oauth_data: serde_json::Value, - pub created_at: DateTime, - pub updated_at: DateTime, - pub created_by: String, -} - -#[async_trait] -pub trait External: Send + Sync + 'static { - type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - type TriggerData: Debug + Serialize + Send + Sync; - type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; - type CreateResponse: DeserializeOwned + Send + Sync; - - const SUPPORT_WEBHOOK: bool; - const SERVICE_NAME: ServiceName; - const DISPLAY_NAME: &'static str; - const TOKEN_ENDPOINT: &'static str; - const REFRESH_ENDPOINT: &'static str; - - async fn create( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn update( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - async fn get( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn delete( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - #[allow(unused)] - async fn exists( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn list_all( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - db: &DB, - tx: &mut PgConnection, - ) -> Result>; - - async fn prepare_webhook( - &self, - _db: &DB, - _w_id: &str, - _header: HashMap, - _body: String, - _script_path: &str, - _is_flow: bool, - ) -> Result { - Ok(PushArgsOwned { extra: None, args: HashMap::new() }) - } - - fn external_id_and_metadata_from_response( - &self, - resp: &Self::CreateResponse, - ) -> (String, Option); - - fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; - - /// Extracts the service-specific config from trigger data (from external service). - /// Used for comparison during sync to detect config drift. - /// Default implementation converts the trigger data to a JSON value - /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. - fn extract_service_config_from_trigger_data( - &self, - data: &Self::TriggerData, - ) -> Result { - serde_json::to_value(data).map_err(|e| { - Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) - }) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn http_client_request( - &self, - url: &str, - method: Method, - workspace_id: &str, - tx: &mut PgConnection, - db: &DB, - headers: Option>, - body: Option<&B>, - ) -> Result { - let oauth_config: OAuthConfig = - decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; - - let result = make_http_request( - url, - method.clone(), - headers.clone(), - body.as_ref(), - &oauth_config.access_token, - ) - .await; - - match result { - Ok(response) => Ok(response), - Err(err) - if err.status() == Some(StatusCode::UNAUTHORIZED) - || err.status() == Some(StatusCode::FORBIDDEN) => - { - tracing::info!( - "HTTP auth error ({}), attempting token refresh", - err.status().unwrap() - ); - - let refreshed_oauth_config = - refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; - - task::spawn({ - let db_clone = db.clone(); - let workspace_id_clone = workspace_id.to_string(); - let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); - async move { - update_workspace_integration_tokens_helper( - db_clone, - workspace_id_clone, - Self::SERVICE_NAME, - refreshed_json, - ) - .await; - } - }); - - let response = make_http_request( - url, - method, - headers, - body.as_ref(), - &refreshed_oauth_config.access_token, - ) - .await - .map_err(to_anyhow)?; - Ok(response) - } - Err(e) => Err(to_anyhow(e).into()), - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct OAuthConfig { - pub base_url: String, - pub access_token: String, - pub refresh_token: Option, - pub client_id: String, - pub client_secret: String, -} - -pub async fn make_http_request( - url: &str, - method: Method, - headers: Option>, - body: Option<&B>, - access_token: &str, -) -> std::result::Result { - let client = Client::new(); - let mut request = client.request(method, url); - - request = request - .header("Accept", "application/json") - .header("Authorization", format!("Bearer {}", access_token)); - - if body.is_some() { - request = request.header("Content-Type", "application/json"); - } - - if let Some(custom_headers) = headers { - for (key, value) in custom_headers { - request = request.header(key, value); - } - } - - if let Some(body_content) = body { - request = request.json(body_content); - } - - let response = request.send().await?.error_for_status()?; - - let response_json = response.json().await?; - - Ok(response_json) -} - -pub async fn decrypt_oauth_data< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, - T: DeserializeOwned, ->( - tx: E, - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = get_workspace_integration(tx, workspace_id, service_name).await?; - - let mc = build_crypt(db, workspace_id).await?; - let mut oauth_data: serde_json::Value = integration.oauth_data; - - if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { - let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; - oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); - } - - if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) - { - let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; - oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); - } - - serde_json::from_value(oauth_data) - .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) -} - -#[allow(unused)] -pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { - let base_url = oauth_data - .get("base_url") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? - .to_string(); - - let access_token = oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? - .to_string(); - - let refresh_token = oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let client_id = oauth_data - .get("client_id") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? - .to_string(); - - let client_secret = oauth_data - .get("client_secret") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? - .to_string(); - - Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) -} - -#[inline] -pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { - let mut json = json!({ - "base_url": config.base_url, - "access_token": config.access_token, - "client_id": config.client_id, - "client_secret": config.client_secret, - }); - - if let Some(refresh_token) = &config.refresh_token { - json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); - } - - json -} - -/// Token refresh response -#[cfg(feature = "native_trigger")] -#[derive(Debug, Deserialize)] -struct RefreshTokenResponse { - access_token: String, - refresh_token: Option, -} - -/// Refresh OAuth tokens using windmill-oauth. -#[cfg(feature = "native_trigger")] -pub async fn refresh_oauth_tokens( - oauth_config: &OAuthConfig, - refresh_endpoint: &str, -) -> Result { - let refresh_token_str = oauth_config - .refresh_token - .as_ref() - .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; - - // Build OAuth client for token refresh - // Auth URL is not used for refresh, but required by the client constructor - let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; - - let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); - client.set_client_secret(oauth_config.client_secret.clone()); - - let token_response: RefreshTokenResponse = client - .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) - .with_client(&*OAUTH_HTTP_CLIENT) - .execute() - .await - .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; - - Ok(OAuthConfig { - base_url: oauth_config.base_url.clone(), - access_token: token_response.access_token, - refresh_token: token_response - .refresh_token - .or_else(|| oauth_config.refresh_token.clone()), - client_id: oauth_config.client_id.clone(), - client_secret: oauth_config.client_secret.clone(), - }) -} - -/// Fallback refresh without native_triggers feature -#[cfg(not(feature = "native_trigger"))] -pub async fn refresh_oauth_tokens( - _oauth_config: &OAuthConfig, - _refresh_endpoint: &str, -) -> Result { - Err(Error::InternalErr( - "Native triggers feature is not enabled".to_string(), - )) -} - -async fn update_workspace_integration_tokens_helper( - db: DB, - workspace_id: String, - service_name: ServiceName, - oauth_data: serde_json::Value, -) { - let result = async { - let mut tx = db.begin().await?; - let mc = build_crypt(&db, &workspace_id).await?; - let mut encrypted_oauth_data = oauth_data; - - if let Some(access_token) = encrypted_oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - { - let encrypted_access_token = encrypt(&mc, access_token); - encrypted_oauth_data["access_token"] = - serde_json::Value::String(encrypted_access_token); - } - - if let Some(refresh_token) = encrypted_oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - { - let encrypted_refresh_token = encrypt(&mc, refresh_token); - encrypted_oauth_data["refresh_token"] = - serde_json::Value::String(encrypted_refresh_token); - } - - sqlx::query!( - r#" - UPDATE workspace_integrations - SET oauth_data = $1, updated_at = now() - WHERE workspace_id = $2 AND service_name = $3 - "#, - encrypted_oauth_data, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok::<(), Error>(()) - } - .await; - - if let Err(e) = result { - tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", - service_name, workspace_id, e); - } -} - -/// Look up the full token from the token table using its prefix -pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result> { - let token = sqlx::query_scalar!( - r#" - SELECT token - FROM token - WHERE token LIKE concat($1::text, '%') - LIMIT 1 - "#, - token_prefix - ) - .fetch_optional(db) - .await?; - - Ok(token) -} - -/// Delete a token from the token table using its prefix -pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM token - WHERE token LIKE concat($1::text, '%') - "#, - token_prefix - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: C, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - INSERT INTO native_trigger ( - external_id, - workspace_id, - service_name, - script_path, - is_flow, - webhook_token_prefix, - service_config - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7 - ) - ON CONFLICT (external_id, workspace_id, service_name) - DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() - "#, - external_id, - workspace_id, - service_name as ServiceName, - config.script_path, - config.is_flow, - webhook_token_prefix, - sqlx::types::Json(service_config) as _, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: Option<&RawValue>, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - UPDATE native_trigger - SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() - WHERE - workspace_id = $5 - AND service_name = $6 - AND external_id = $7 - "#, - config.script_path, - config.is_flow, - webhook_token_prefix, - service_config.map(sqlx::types::Json) as _, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} -pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - script_path: &str, - is_flow: bool, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND script_path = $3 - AND is_flow = $4 - LIMIT 1 - "#, - workspace_id, - service_name as ServiceName, - script_path, - is_flow - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - page: Option, - per_page: Option, - path: Option<&str>, - is_flow: Option, -) -> Result> { - let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; - let limit = per_page.unwrap_or(100) as i64; - - let triggers = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - nt.external_id, - nt.workspace_id, - nt.service_name AS "service_name!: ServiceName", - nt.script_path, - nt.is_flow, - nt.webhook_token_prefix, - nt.service_config, - nt.error, - nt.created_at, - nt.updated_at - FROM - native_trigger nt - WHERE - nt.workspace_id = $1 AND - nt.service_name = $2 AND - ($5::text IS NULL OR nt.script_path = $5) AND - ($6::bool IS NULL OR nt.is_flow = $6) AND - ( - (nt.is_flow = false AND EXISTS ( - SELECT 1 FROM script s - WHERE s.workspace_id = nt.workspace_id - AND s.path = nt.script_path - )) - OR - (nt.is_flow = true AND EXISTS ( - SELECT 1 FROM flow f - WHERE f.workspace_id = nt.workspace_id - AND f.path = nt.script_path - )) - ) - LIMIT $3 - OFFSET $4 - "#, - workspace_id, - service_name as ServiceName, - limit, - offset, - path, - is_flow - ) - .fetch_all(db) - .await?; - - Ok(triggers) -} - -pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - error: Option<&str>, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET error = $1 - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - error, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger_service_config< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, ->( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - service_config: &serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET service_config = $1, updated_at = NOW() - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - service_config, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn store_workspace_integration( - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - service_name: ServiceName, - oauth_data: serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - INSERT INTO workspace_integrations ( - workspace_id, - service_name, - oauth_data, - created_by, - created_at, - updated_at - ) VALUES ( - $1, $2, $3, $4, now(), now() - ) - ON CONFLICT (workspace_id, service_name) - DO UPDATE SET - oauth_data = $3, - updated_at = now() - "#, - workspace_id, - service_name as ServiceName, - oauth_data, - authed.username, - ) - .execute(&mut *tx) - .await?; - - Ok(()) -} - -pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = sqlx::query_as!( - WorkspaceIntegration, - r#" - SELECT - workspace_id, - service_name AS "service_name!: ServiceName", - oauth_data, - created_at, - updated_at, - created_by - FROM - workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .fetch_one(db) - .await?; - - Ok(integration) -} - -pub async fn delete_workspace_integration( - tx: &mut PgConnection, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -/// Generates the webhook URL that external services will call. -/// -/// `external_id` is optional because during CREATE we don't have it yet -/// (it's returned by the external service). During UPDATE, we have it. -pub fn generate_webhook_service_url( - base_url: &str, - w_id: &str, - script_path: &str, - is_flow: bool, - external_id: Option<&str>, - service_name: ServiceName, - webhook_token: &str, -) -> String { - let runnable_prefix = if is_flow { "f" } else { "p" }; - - let mut url = format!( - "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", - base_url, - w_id, - runnable_prefix, - script_path, - &webhook_token, - service_name.as_str(), - ); - - if let Some(id) = external_id { - url.push_str(&format!("&trigger_external_id={}", id)); - } - - url -} +pub use windmill_native_triggers::*; diff --git a/backend/windmill-api/src/npm_proxy.rs b/backend/windmill-api/src/npm_proxy.rs index 8bd7f851f7..02f8a3dbb8 100644 --- a/backend/windmill-api/src/npm_proxy.rs +++ b/backend/windmill-api/src/npm_proxy.rs @@ -39,7 +39,9 @@ fn parse_package_and_version(path: &str) -> Result<(String, String)> { // Scoped package: @scope/name/version let parts: Vec<&str> = path.splitn(3, '/').collect(); if parts.len() < 3 { - return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string())); + return Err(Error::BadRequest( + "Invalid scoped package path, expected @scope/name/version".to_string(), + )); } let package = format!("{}/{}", parts[0], parts[1]); let version = parts[2].to_string(); @@ -48,7 +50,9 @@ fn parse_package_and_version(path: &str) -> Result<(String, String)> { // Regular package: name/version let parts: Vec<&str> = path.splitn(2, '/').collect(); if parts.len() < 2 { - return Err(Error::BadRequest("Invalid package path, expected name/version".to_string())); + return Err(Error::BadRequest( + "Invalid package path, expected name/version".to_string(), + )); } Ok((parts[0].to_string(), parts[1].to_string())) } @@ -62,7 +66,10 @@ fn parse_package_version_and_file(path: &str) -> Result<(String, String, String) // Scoped package: @scope/name/version/filepath let parts: Vec<&str> = path.splitn(4, '/').collect(); if parts.len() < 4 { - return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string())); + return Err(Error::BadRequest( + "Invalid scoped package file path, expected @scope/name/version/filepath" + .to_string(), + )); } let package = format!("{}/{}", parts[0], parts[1]); let version = parts[2].to_string(); @@ -72,9 +79,15 @@ fn parse_package_version_and_file(path: &str) -> Result<(String, String, String) // Regular package: name/version/filepath let parts: Vec<&str> = path.splitn(3, '/').collect(); if parts.len() < 3 { - return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string())); + return Err(Error::BadRequest( + "Invalid package file path, expected name/version/filepath".to_string(), + )); } - Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string())) + Ok(( + parts[0].to_string(), + parts[1].to_string(), + parts[2].to_string(), + )) } } @@ -313,10 +326,7 @@ async fn get_package_filetree( .unwrap_or("index.js") .to_string(); - Ok(Json(PackageFiletree { - default: main, - files, - })) + Ok(Json(PackageFiletree { default: main, files })) } /// Get a specific file from a package version @@ -419,7 +429,10 @@ fn format_registry_url( match (version, file) { (Some(v), Some(f)) => { - format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f) + format!( + "{}/{}/{}/-/{}-{}/{}", + registry_base, package_path, v, package, v, f + ) } (Some(v), None) => { format!("{}/{}/{}", registry_base, package_path, v) @@ -453,9 +466,7 @@ fn extract_tarball_files(tarball_bytes: &[u8]) -> Result> { // Remove the package/ prefix that npm tarballs have let path_str = path.to_string_lossy().to_string(); if let Some(stripped) = path_str.strip_prefix("package/") { - files.push(FileEntry { - name: format!("/{}", stripped), - }); + files.push(FileEntry { name: format!("/{}", stripped) }); } } @@ -490,13 +501,16 @@ fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result< if let Some(stripped) = path_str.strip_prefix("package/") { if stripped == target { let mut content = String::new(); - entry - .read_to_string(&mut content) - .map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?; + entry.read_to_string(&mut content).map_err(|e| { + Error::InternalErr(format!("Failed to read file content: {}", e)) + })?; return Ok(content); } } } - Err(Error::NotFound(format!("File {} not found in tarball", target_file))) + Err(Error::NotFound(format!( + "File {} not found in tarball", + target_file + ))) } diff --git a/backend/windmill-api/src/oauth2_oss.rs b/backend/windmill-api/src/oauth2_oss.rs index fba65a11bd..2b47c66bdf 100644 --- a/backend/windmill-api/src/oauth2_oss.rs +++ b/backend/windmill-api/src/oauth2_oss.rs @@ -20,14 +20,14 @@ use hmac::Mac; #[cfg(all(feature = "oauth2", not(feature = "private")))] use itertools::Itertools; -#[cfg(all(feature = "oauth2", not(feature = "private")))] -use windmill_oauth::{OClient, AccessToken, RefreshToken, Scope, helpers}; #[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[cfg(not(feature = "private"))] use sqlx::{Postgres, Transaction}; #[cfg(all(feature = "oauth2", not(feature = "private")))] use windmill_common::more_serde::maybe_number_opt; +#[cfg(all(feature = "oauth2", not(feature = "private")))] +use windmill_oauth::{helpers, AccessToken, RefreshToken, Scope}; #[cfg(all(feature = "oauth2", not(feature = "private")))] use crate::OAUTH_CLIENTS; @@ -54,47 +54,10 @@ pub fn workspaced_service() -> Router { } #[cfg(all(feature = "oauth2", not(feature = "private")))] -#[derive(Debug, Clone)] -pub struct ClientWithScopes { - _client: OClient, - _scopes: Vec, - _extra_params: Option>, - _extra_params_callback: Option>, - _allowed_domains: Option>, - _userinfo_url: Option, -} -#[cfg(all(feature = "oauth2", not(feature = "private")))] -pub type BasicClientsMap = HashMap; +pub use windmill_oauth::{AllClients, BasicClientsMap, ClientWithScopes}; #[cfg(not(feature = "private"))] -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OAuthConfig { - auth_url: String, - token_url: String, - userinfo_url: Option, - scopes: Option>, - extra_params: Option>, - extra_params_callback: Option>, - req_body_auth: Option, -} - -#[cfg(not(feature = "private"))] -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct OAuthClient { - id: String, - secret: String, - allowed_domains: Option>, - connect_config: Option, - login_config: Option, -} - -#[cfg(all(feature = "oauth2", not(feature = "private")))] -#[derive(Debug)] -pub struct AllClients { - pub logins: BasicClientsMap, - pub connects: BasicClientsMap, - pub slack: Option, -} +pub use windmill_oauth::{OAuthClient, OAuthConfig}; #[cfg(all(feature = "oauth2", not(feature = "private")))] pub async fn build_oauth_clients( diff --git a/backend/windmill-api/src/openapi.rs b/backend/windmill-api/src/openapi.rs index 04d17a6127..a178c38163 100644 --- a/backend/windmill-api/src/openapi.rs +++ b/backend/windmill-api/src/openapi.rs @@ -25,8 +25,7 @@ use crate::{ db::ApiAuthed, resources::try_get_resource_from_db_as, triggers::http::{ - http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, - AuthenticationMethod, RequestType, + http_trigger_auth::ApiKeyAuthentication, AuthenticationMethod, HttpMethod, RequestType, }, }; diff --git a/backend/windmill-api/src/public_app_rate_limit.rs b/backend/windmill-api/src/public_app_rate_limit.rs index bff89d694b..5d49c17a71 100644 --- a/backend/windmill-api/src/public_app_rate_limit.rs +++ b/backend/windmill-api/src/public_app_rate_limit.rs @@ -17,8 +17,7 @@ struct RateLimitEntry { minute_bucket: i64, } -static RATE_LIMIT_COUNTER: LazyLock> = - LazyLock::new(DashMap::new); +static RATE_LIMIT_COUNTER: LazyLock> = LazyLock::new(DashMap::new); pub fn check_and_increment(workspace_id: &str, limit: i32) -> Result<()> { let current_minute = Utc::now().timestamp() / 60; diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 575e09421c..47655d4149 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -1,1821 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use std::collections::HashMap; - -use crate::{ - db::{ApiAuthed, DB}, - secret_backend_ext::rename_vault_secret, - users::{maybe_refresh_folders, require_owner_of_path, Tokened}, - utils::{check_scopes, require_super_admin, BulkDeleteRequest}, - var_resource_cache::{cache_resource, get_cached_resource}, - variables::get_value_internal, - webhook_util::{WebhookMessage, WebhookShared}, -}; -use axum::{ - body::Body, - extract::{Extension, Path, Query}, - response::Response, - routing::{delete, get, post}, - Json, Router, -}; -use futures::future::try_join_all; -use hyper::{header, StatusCode}; -use serde::{Deserialize, Serialize}; -use serde_json::{value::RawValue, Value}; -use sql_builder::{bind::Bind, quote, SqlBuilder}; -use sqlx::{Acquire, FromRow, Postgres, Transaction}; -use std::process::Stdio; -use tokio::process::Command; -use uuid::Uuid; -use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; -use windmill_audit::ActionKind; -use windmill_common::{ - db::{DbWithOptAuthed, UserDB}, - error::{self, Error, JsonResult, Result}, - get_database_url, - utils::{ - get_custom_pg_instance_password, not_found_if_none, paginate, require_admin, Pagination, - StripPath, - }, - variables, - worker::{CLOUD_HOSTED, TMP_DIR}, - PgDatabase, -}; - -pub fn workspaced_service() -> Router { - let router = Router::new() - .route("/list", get(list_resources)) - .route("/list_search", get(list_search_resources)) - .route("/list_names/:type", get(list_names)) - .route("/get/*path", get(get_resource)) - .route("/exists/*path", get(exists_resource)) - .route("/get_value/*path", get(get_resource_value)) - .route( - "/get_value_interpolated/*path", - get(get_resource_value_interpolated), - ) - .route("/update/*path", post(update_resource)) - .route("/update_value/*path", post(update_resource_value)) - .route("/delete/*path", delete(delete_resource)) - .route("/delete_bulk", delete(delete_resources_bulk)) - .route("/create", post(create_resource)) - .route("/git_commit_hash/*path", get(get_git_commit_hash)) - .route("/type/list", get(list_resource_types)) - .route("/type/listnames", get(list_resource_types_names)) - .route("/type/get/:name", get(get_resource_type)) - .route("/type/exists/:name", get(exists_resource_type)) - .route("/type/update/:name", post(update_resource_type)) - .route("/type/delete/:name", delete(delete_resource_type)) - .route( - "/file_resource_type_to_file_ext_map", - get(file_resource_ext_to_resource_type), - ) - .route("/type/create", post(create_resource_type)); - - #[cfg(feature = "mcp")] - let router = router.route("/mcp_tools/*path", get(get_mcp_tools)); - - router -} - -pub fn public_service() -> Router { - Router::new().route("/custom_component/:name", get(custom_component)) -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct ResourceType { - pub workspace_id: String, - pub name: String, - pub schema: Option, - pub description: Option, - pub created_by: Option, - pub edited_at: Option>, - pub format_extension: Option, -} - -#[derive(Deserialize)] -pub struct CreateResourceType { - pub name: String, - pub schema: Option, - pub description: Option, - pub format_extension: Option, -} - -#[derive(Deserialize)] -pub struct EditResourceType { - pub schema: Option, - pub description: Option, -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct Resource { - pub workspace_id: String, - pub path: String, - pub value: Option, - pub description: Option, - pub resource_type: String, - pub extra_perms: serde_json::Value, - pub created_by: Option, - pub edited_at: Option>, -} - -#[derive(FromRow, Serialize, Deserialize)] -pub struct ListableResource { - pub workspace_id: String, - pub path: String, - pub value: Option, - pub description: Option, - pub resource_type: String, - pub extra_perms: serde_json::Value, - pub created_by: Option, - pub edited_at: Option>, - pub is_linked: Option, - pub is_refreshed: Option, - pub is_oauth: Option, - pub is_expired: Option, - pub refresh_error: Option, - pub account: Option, -} - -#[derive(Deserialize)] -pub struct CreateResource { - pub path: String, - pub value: Option>, - pub description: Option, - pub resource_type: String, -} -#[derive(Deserialize)] -struct EditResource { - path: Option, - description: Option, - value: Option>, -} - -#[derive(Deserialize)] -pub struct ListResourceQuery { - resource_type: Option, - resource_type_exclude: Option, - path_start: Option, -} - -#[derive(Serialize, FromRow)] -pub struct NamePath { - name: String, - path: String, -} -async fn list_names( - authed: ApiAuthed, - Path((w_id, rt)): Path<(String, String)>, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query!( - "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2", - rt, - &w_id - ) - .fetch_all(&mut *tx) - .await? - .into_iter() - .filter_map(|x| x.name.map(|name| NamePath { name, path: x.path })) - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Serialize, FromRow)] -pub struct SearchResource { - path: String, - value: serde_json::Value, -} -async fn list_search_resources( - authed: ApiAuthed, - Path(w_id): Path, - Extension(user_db): Extension, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - #[cfg(feature = "enterprise")] - let n = 1000; - - #[cfg(not(feature = "enterprise"))] - let n = 3; - - let rows = sqlx::query_as!( - SearchResource, - "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", - &w_id, - n - ) - .fetch_all(&mut *tx) - .await? - .into_iter() - .collect::>(); - tx.commit().await?; - Ok(Json(rows)) -} - -async fn list_resources( - authed: ApiAuthed, - Query(lq): Query, - Query(pagination): Query, - Extension(user_db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let mut sqlb = SqlBuilder::select_from("resource") - .fields(&[ - "resource.workspace_id", - "resource.path", - "null::JSONB as value", - "resource.description", - "resource_type", - "resource.extra_perms", - "(now() > account.expires_at) as is_expired", - "variable.path IS NOT NULL as is_linked", - "account.refresh_token != '' as is_refreshed", - "variable.is_oauth", - "variable.account", - "account.refresh_error", - "resource.created_by", - "resource.edited_at", - ]) - .left() - .join("variable") - .on("variable.path = resource.path AND variable.workspace_id = resource.workspace_id") - .left() - .join("account") - .on("variable.account = account.id AND account.workspace_id = variable.workspace_id") - .order_by("path", true) - .and_where("resource.workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page) - .clone(); - - if let Some(rt) = &lq.resource_type { - let resource_type_filters = rt.split(',').collect::>(); - if resource_type_filters.len() == 1 { - sqlb.and_where_eq("resource_type", "?".bind(rt)); - } else { - let mut list = Vec::new(); - for rt in resource_type_filters { - let quoted_value = quote(rt); - list.push(quoted_value); - } - sqlb.and_where_in("resource_type", list.as_slice()); - } - } - if let Some(rt) = &lq.resource_type_exclude { - for rt in rt.split(',') { - sqlb.and_where_ne("resource_type", "?".bind(&rt)); - } - } - - if let Some(path_start) = &lq.path_start { - sqlb.and_where_like_left("resource.path", path_start); - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - let rows = sqlx::query_as::<_, ListableResource>(&sql) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(Json(rows)) -} - -async fn get_resource( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let resource_o = sqlx::query_as!( - ListableResource, - "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed, - account.refresh_error, - variable.path IS NOT NULL as is_linked, - variable.is_oauth as \"is_oauth?\", - variable.account - FROM resource - LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2 - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 - WHERE resource.path = $1 AND resource.workspace_id = $2", - path.to_owned(), - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - if resource_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; - } - let resource = not_found_if_none(resource_o, "Resource", path)?; - Ok(Json(resource)) -} - -async fn exists_resource( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn get_resource_value( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - - let value_o = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path.to_owned(), - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - if value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; - } - - let value = not_found_if_none(value_o, "Resource", path)?; - - Ok(Json(value)) -} - -async fn explain_resource_perm_error( - path: &str, - w_id: &str, - db: &sqlx::Pool, - authed: &ApiAuthed, -) -> windmill_common::error::Result<()> { - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Resource {} not found", path)))?; - if path.starts_with("f/") { - let folder = path.split("/").nth(1).ok_or_else(|| { - Error::BadRequest(format!( - "path {} should have at least 2 components separated by /", - path - )) - })?; - let folder_extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(db) - .await?; - return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() - ))); - } else { - return Err(Error::NotAuthorized(format!( - "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default() - ))); - } -} - -async fn custom_component( - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - let cc_o = sqlx::query_scalar!( - "SELECT value->>'js' FROM resource - WHERE path = $1 AND workspace_id = $2", - format!("f/app_custom/{name}"), - &w_id - ) - .fetch_optional(&db) - .await? - .flatten(); - - let cc = not_found_if_none(cc_o, "Custom Component", name)?; - let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript"); - - Ok(res.body(Body::from(cc)).unwrap()) -} - -#[derive(Deserialize)] -struct JobInfo { - job_id: Option, - allow_cache: Option, -} - -async fn get_resource_value_interpolated( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Tokened { token }: Tokened, - Path((w_id, path)): Path<(String, StripPath)>, - Query(job_info): Query, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let db_with_opt_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - return get_resource_value_interpolated_internal( - &db_with_opt_authed, - w_id.as_str(), - path, - job_info.job_id, - Some(token.as_str()), - job_info.allow_cache.unwrap_or(false), - ) - .await - .map(|success| Json(success)); -} - -use async_recursion::async_recursion; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -pub async fn get_resource_value_interpolated_internal<'a>( - db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, - workspace: &str, - path: &str, - job_id: Option, - token_for_context: Option<&str>, - allow_cache: bool, -) -> Result> { - // This is a special syntax to help debugging custom instance databases - if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { - let db = db_with_opt_authed.db(); - require_super_admin(db_with_opt_authed.db(), &db_with_opt_authed.email()).await?; - let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; - pg_creds.dbname = dbname.to_string(); - pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); - pg_creds.user = Some("custom_instance_user".to_string()); - let pg_creds = serde_json::to_value(&pg_creds) - .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?; - return Ok(Some(pg_creds)); - } - - if allow_cache { - if let Some(cached_value) = get_cached_resource(&workspace, &path) { - return Ok(Some(cached_value)); - } - } - use sqlx::Acquire; - let mut tx = db_with_opt_authed.begin().await?; - - let value_o = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path, - workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - if value_o.is_none() { - if let Some(authed) = db_with_opt_authed.authed() { - let db = db_with_opt_authed.db(); - explain_resource_perm_error(path, workspace, db, authed).await?; - } - } - - let value = not_found_if_none(value_o, "Resource", path)?; - if let Some(value) = value { - let r = transform_json_value( - &db_with_opt_authed, - workspace, - value, - &job_id, - token_for_context, - ) - .await?; - if allow_cache { - cache_resource(&workspace, &path, r.clone()); - } - Ok(Some(r)) - } else { - Ok(None) - } -} - -#[async_recursion] -pub async fn transform_json_value( - db_with_opt_authed: &DbWithOptAuthed, - workspace: &str, - v: Value, - job_id: &Option, - token: Option<&str>, -) -> Result { - match v { - Value::String(y) if y.starts_with("$var:") => { - let path = y.strip_prefix("$var:").unwrap(); - - let v = - crate::variables::get_value_internal(&db_with_opt_authed, workspace, path, false) - .await?; - Ok(Value::String(v)) - } - Value::String(y) if y.starts_with("$res:") => { - let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 { - return Err(Error::internal_err(format!( - "Invalid resource path: {path}" - ))); - } - let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; - let v = sqlx::query_scalar!( - "SELECT value from resource WHERE path = $1 AND workspace_id = $2", - path, - &workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - let v = not_found_if_none(v, "Resource", path)?; - if let Some(v) = v { - transform_json_value(db_with_opt_authed, workspace, v, job_id, token).await - } else { - Ok(Value::Null) - } - } - Value::String(y) if y.starts_with("$") && job_id.is_some() => { - let mut tx = db_with_opt_authed.begin().await?; - let job_id = job_id.unwrap(); - let job = sqlx::query!( - "SELECT - v2_job.permissioned_as_email, - v2_job.created_by, - v2_job.parent_job, - v2_job.permissioned_as, - v2_job.runnable_path, - CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path, - v2_job.flow_step_id, - v2_job.flow_innermost_root_job, - v2_job.root_job, - v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\" - FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id - WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", - job_id, - workspace - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let job = not_found_if_none(job, "Job", job_id.to_string())?; - - let flow_path = if let Some(uuid) = job.parent_job { - let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; - let p = sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) - .fetch_optional(&mut *tx) - .await? - .flatten(); - tx.commit().await?; - p - } else { - None - }; - - let variables = variables::get_reserved_variables( - &db_with_opt_authed.db().into(), - workspace, - token.unwrap_or_else(|| "no_token_available"), - &job.permissioned_as_email, - &job.created_by, - &job_id.to_string(), - &job.permissioned_as, - job.runnable_path.clone(), - job.parent_job.map(|x| x.to_string()), - flow_path, - job.schedule_path.clone(), - job.flow_step_id.clone(), - job.flow_innermost_root_job.map(|x| x.to_string()), - job.root_job.map(|x| x.to_string()), - Some(job.scheduled_for.clone()), - None, - None, - ) - .await; - - let name = y.strip_prefix("$").unwrap(); - - let value = variables - .iter() - .find(|x| x.name == name) - .map(|x| x.value.clone()) - .unwrap_or_else(|| y); - Ok(serde_json::json!(value)) - } - Value::Object(mut m) => { - for (a, b) in m.clone().into_iter() { - let v = - transform_json_value(db_with_opt_authed, workspace, b, job_id, token).await?; - m.insert(a.clone(), v); - } - Ok(Value::Object(m)) - } - a @ _ => Ok(a), - } -} - -// async fn authed_transaction_or_default<'c>( -// db_with_opt_authed: &'c DbWithOptAuthed, -// ) -> sqlx::error::Result> { -// if let Some(user_db) = user_db { -// user_db.begin(authed).await -// } else { -// db.clone().begin().await -// } -// } - -async fn check_path_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - path: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Resource {} already exists", - path - ))); - } - return Ok(()); -} - -#[derive(Deserialize)] -struct CreateResourceQuery { - update_if_exists: Option, -} -async fn create_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Query(q): Query, - Json(resource): Json, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || format!("resources:write:{}", resource.path))?; - if *CLOUD_HOSTED { - let nb_resources = sqlx::query_scalar!( - "SELECT COUNT(*) FROM resource WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - if nb_resources.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of resources (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await; - - let mut tx = user_db.begin(&authed).await?; - - let update_if_exists = q.update_if_exists.unwrap_or(false); - if !update_if_exists { - check_path_conflict(&mut tx, &w_id, &resource.path).await?; - } - - let res_value = resource.value.unwrap_or_default(); - let raw_json = sqlx::types::Json(res_value.as_ref()); - - if resource.path.starts_with("f/app_themes/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } else if resource.path.starts_with("f/app_custom/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } else if resource.path.starts_with("f/app_groups/") { - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - w_id, - authed.username, - ) - .execute(&db) - .await?; - } - sqlx::query!( - "INSERT INTO resource - (workspace_id, path, value, description, resource_type, created_by, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", - w_id, - resource.path, - raw_json as sqlx::types::Json<&RawValue>, - resource.description, - resource.resource_type, - authed.username - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.create", - ActionKind::Create, - &w_id, - Some(&resource.path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: resource.path.clone(), parent_path: None }, - Some(format!("Resource '{}' created", resource.path.clone())), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("resource {} created", resource.path), - )) -} - -async fn delete_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - - check_scopes(&authed, || format!("resources:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let deleted_path = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(deleted_path, "Resource", &path)?; - sqlx::query!( - "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Resource '{}' deleted", path)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("resource {} deleted", path)) -} - -async fn delete_resources_bulk( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(request): Json, -) -> JsonResult> { - for path in &request.paths { - check_scopes(&authed, || format!("resources:write:{}", path))?; - } - - let mut tx = user_db.begin(&authed).await?; - - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id - ) - .fetch_all(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "resources.delete_bulk", - ActionKind::Delete, - &w_id, - Some(&deleted_paths.join(", ")), - None, - ) - .await?; - - tx.commit().await?; - - try_join_all(deleted_paths.iter().map(|path| { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Resource '{}' deleted", path)), - true, - None, - ) - })) - .await?; - - for path in &deleted_paths { - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResource { workspace: w_id.clone(), path: path.to_owned() }, - ); - } - - Ok(Json(deleted_paths)) -} - -async fn update_resource( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let path = path.to_path(); - check_scopes(&authed, || format!("resources:write:{}", path))?; - - let mut sqlb = SqlBuilder::update_table("resource"); - sqlb.and_where_eq("path", "?".bind(&path)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - if let Some(npath) = &ns.path { - sqlb.set_str("path", npath); - } - if let Some(nvalue) = ns.value { - sqlb.set_str("value", nvalue.to_string()); - } - if let Some(ndesc) = ns.description { - sqlb.set_str("description", ndesc); - } - - sqlb.set_str("edited_at", "now()"); - - sqlb.returning("path"); - let authed = maybe_refresh_folders(path, &w_id, authed, &db).await; - - let mut tx = user_db.begin(&authed).await?; - - if let Some(npath) = ns.path.clone() { - if npath != path { - check_path_conflict(&mut tx, &w_id, &npath).await?; - - require_owner_of_path(&authed, path)?; - - // Handle Vault secret rename if the linked variable is a Vault-stored secret - let linked_var = sqlx::query!( - "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(var) = linked_var { - if var.is_secret { - // Check if this is a Vault-stored secret and rename it - if let Some(new_value) = - rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? - { - // Update the variable's value to point to the new Vault path - sqlx::query!( - "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", - new_value, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - } - - sqlx::query!( - "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", - npath, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; - - let npath = not_found_if_none(npath_o, "Resource", path)?; - - audit_log( - &mut *tx, - &authed, - "resources.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - // Detect if this was a rename operation - let old_path_if_renamed = if npath != path { Some(path) } else { None }; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: npath.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Resource '{}' updated", npath)), - true, - old_path_if_renamed, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResource { - workspace: w_id, - old_path: path.to_owned(), - new_path: npath.clone(), - }, - ); - - Ok(format!("resource {} updated (npath: {:?})", path, npath)) -} - -#[derive(FromRow, Serialize, Deserialize)] -struct UpdateResource { - value: Option, -} - -async fn update_resource_value( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(nv): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "UPDATE resource SET value = $1, edited_at = now() WHERE path = $2 AND workspace_id = $3", - nv.value, - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "resources.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResource { - workspace: w_id, - old_path: path.to_owned(), - new_path: path.to_owned(), - }, - ); - - Ok(format!("value of resource {} updated", path)) -} - -async fn file_resource_ext_to_resource_type( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - #[derive(Serialize, sqlx::FromRow)] - struct LocalFileResourceExtension { - name: String, - format_extension: Option, - } - - let r = sqlx::query_as!(LocalFileResourceExtension, " - SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) - .fetch_all(&db) - .await?; - - let hashmap: HashMap = r - .into_iter() - .filter_map(|entry| { - if let Some(format_extension) = entry.format_extension { - Some((entry.name, format_extension)) - } else { - None - } - }) - .collect(); - - Ok(Json(hashmap)) -} - -async fn list_resource_types( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let rows = sqlx::query_as!( - ResourceType, - "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ - BY name", - &w_id - ) - .fetch_all(&db) - .await?; - - Ok(Json(rows)) -} - -async fn list_resource_types_names( - Extension(db): Extension, - Path(w_id): Path, -) -> JsonResult> { - let rows = sqlx::query_scalar!( - "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') \ - ORDER BY name", - &w_id - ) - .fetch_all(&db) - .await?; - - Ok(Json(rows)) -} - -async fn get_resource_type( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - - let resource_type_o = sqlx::query_as!( - ResourceType, - "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", - &name, - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?; - Ok(Json(resource_type)) -} - -async fn exists_resource_type( - Extension(db): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> JsonResult { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins'))", - name, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn create_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(resource_type): Json, -) -> Result<(StatusCode, String)> { - let mut tx = user_db.begin(&authed).await?; - - check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?; - - sqlx::query!( - "INSERT INTO resource_type - (workspace_id, name, schema, description, created_by, format_extension, edited_at) - VALUES ($1, $2, $3, $4, $5, $6, now())", - w_id, - resource_type.name, - resource_type.schema, - resource_type.description, - authed.username, - resource_type.format_extension, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "resource_types.create", - ActionKind::Create, - &w_id, - Some(&resource_type.name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: resource_type.name.clone() }, - Some(format!( - "Resource Type '{}' created", - resource_type.name.clone() - )), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateResourceType { name: resource_type.name.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("resource_type {} created", resource_type.name), - )) -} - -async fn check_rt_path_conflict<'c>( - tx: &mut Transaction<'c, Postgres>, - w_id: &str, - name: &str, -) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)", - name, - w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Resource type {} already exists", - name - ))); - } - return Ok(()); -} - -async fn delete_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, -) -> Result { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - let deleted_name = sqlx::query_scalar!( - "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", - name, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - not_found_if_none(deleted_name, "ResourceType", &name)?; - - audit_log( - &mut *tx, - &authed, - "resource_types.delete", - ActionKind::Delete, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: name.clone() }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteResourceType { name: name.clone() }, - ); - - Ok(format!("resource_type {} deleted", name)) -} - -async fn update_resource_type( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, name)): Path<(String, String)>, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let mut sqlb = SqlBuilder::update_table("resource_type"); - sqlb.and_where_eq("name", "?".bind(&name)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - if let Some(nschema) = ns.schema { - sqlb.set_str("schema", nschema); - } - if let Some(ndesc) = ns.description { - sqlb.set_str("description", ndesc); - } - sqlb.set_str("edited_at", "now()"); - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - let mut tx = user_db.begin(&authed).await?; - - sqlx::query(&sql).execute(&mut *tx).await?; - audit_log( - &mut *tx, - &authed, - "resource_types.update", - ActionKind::Update, - &w_id, - Some(&name), - None, - ) - .await?; - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::ResourceType { path: name.clone() }, - None, - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateResourceType { name: name.clone() }, - ); - - Ok(format!("resource_type {} updated", name)) -} - -#[cfg(any( - feature = "http_trigger", - feature = "postgres_trigger", - feature = "mqtt_trigger", - all( - feature = "enterprise", - any( - feature = "sqs_trigger", - feature = "gcp_trigger", - feature = "kafka", - feature = "nats" - ) - ) -))] -pub async fn try_get_resource_from_db_as( - authed: &ApiAuthed, - user_db: Option, - db: &DB, - resource_path: &str, - w_id: &str, -) -> Result -where - T: serde::de::DeserializeOwned, -{ - let resource = get_resource_value_interpolated_internal( - &DbWithOptAuthed::from_authed(authed, db.clone(), user_db), - &w_id, - &resource_path, - None, - None, - false, - ) - .await?; - - let resource = match resource { - Some(resource) => serde_json::from_value::(resource) - .map_err(|e| Error::SerdeJson { error: e, location: "resources.rs".to_string() })?, - None => { - return { - Err(Error::NotFound(format!( - "resource at path :{} do not exist", - &resource_path - ))) - } - } - }; - - Ok(resource) -} - -/// Get list of tools from an MCP resource -#[cfg(feature = "mcp")] -async fn get_mcp_tools( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult> { - let path = path.to_path(); - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let mut tx = user_db.clone().begin(&authed).await?; - - // Fetch the MCP resource from database - let resource_value_o = sqlx::query_scalar!( - "SELECT value as \"value: sqlx::types::Json>\" FROM resource WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - - if resource_value_o.is_none() { - explain_resource_perm_error(&path, &w_id, &db, &authed).await?; - } - - let resource_value = not_found_if_none(resource_value_o, "Resource", path)? - .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - - // Parse MCP resource - let mcp_resource = serde_json::from_str::(resource_value.0.get()) - .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; - - // Check if token needs refresh before creating MCP client - #[cfg(feature = "oauth2")] - { - tracing::info!("Checking if token needs refresh before creating MCP client"); - if let Some(ref token_path) = mcp_resource.token { - let token_var_path = token_path.trim_start_matches("$var:"); - - // Query to check if token is expired - let token_info = sqlx::query!( - r#" - SELECT - variable.account as account_id, - (now() > account.expires_at) as "is_expired: bool" - FROM variable - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 - WHERE variable.path = $1 AND variable.workspace_id = $2 - "#, - token_var_path, - &w_id - ) - .fetch_optional(&db) - .await?; - - if let Some(info) = token_info { - if let (Some(account_id), Some(true)) = (info.account_id, info.is_expired) { - let refresh_tx = user_db.begin(&authed).await?; - if let Err(e) = crate::oauth2_oss::_refresh_token( - refresh_tx, - token_var_path, - &w_id, - account_id, - &db, - ) - .await - { - tracing::warn!( - "Failed to refresh token for MCP resource: {}. Proceeding with possibly expired token.", - e - ); - } - } - } - } - } - - // Create MCP client connection - let client = windmill_mcp::McpClient::from_resource(mcp_resource, &db, &w_id) - .await - .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e)))?; - - // Get raw MCP tools and convert to JSON - let tools: Vec = client - .available_tools() - .iter() - .map(|tool| { - serde_json::to_value(tool) - .map_err(|e| Error::ExecutionErr(format!("Failed to serialize MCP tool: {}", e))) - }) - .collect::>>()?; - - // Gracefully shutdown the client - if let Err(e) = client.shutdown().await { - tracing::warn!("Failed to shutdown MCP client: {}", e); - } - - Ok(Json(tools)) -} - -#[derive(Deserialize, Serialize)] -struct GitRepositoryResource { - url: String, - #[serde(skip_serializing_if = "Option::is_none")] - branch: Option, -} - -/// Validates a git URL to prevent git option injection attacks. -/// Git URLs starting with '-' could be interpreted as command-line options. -fn validate_git_url(url: &str) -> Result<()> { - let url = url.trim(); - if url.is_empty() { - return Err(Error::BadRequest("Git URL cannot be empty".to_string())); - } - if url.starts_with('-') { - return Err(Error::BadRequest( - "Git URL cannot start with '-' (potential option injection)".to_string(), - )); - } - // Block other potentially dangerous patterns - if url.contains('\0') || url.contains('\n') || url.contains('\r') { - return Err(Error::BadRequest( - "Git URL contains invalid characters".to_string(), - )); - } - Ok(()) -} - -/// Validates a git branch/ref name to prevent injection attacks. -fn validate_git_ref(ref_name: &str) -> Result<()> { - let ref_name = ref_name.trim(); - if ref_name.is_empty() { - return Err(Error::BadRequest("Git ref cannot be empty".to_string())); - } - if ref_name.starts_with('-') { - return Err(Error::BadRequest( - "Git ref cannot start with '-' (potential option injection)".to_string(), - )); - } - // Git ref names have specific rules - block dangerous characters - if ref_name.contains('\0') - || ref_name.contains('\n') - || ref_name.contains('\r') - || ref_name.contains("..") - || ref_name.contains("@{") - || ref_name.ends_with('.') - || ref_name.ends_with('/') - || ref_name.contains("//") - { - return Err(Error::BadRequest( - "Git ref contains invalid characters or patterns".to_string(), - )); - } - Ok(()) -} - -#[derive(Serialize)] -struct GitCommitHashResponse { - commit_hash: String, -} - -#[derive(Deserialize)] -struct GitCommitHashQuery { - git_ssh_identity: Option, -} - -async fn get_git_commit_hash( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> JsonResult { - let path = path.to_path(); - - check_scopes(&authed, || format!("resources:read:{}", path))?; - - let db_with_opt_authed = - DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - let git_repo_resource_value = get_resource_value_interpolated_internal( - &db_with_opt_authed, - &w_id, - path, - None, - None, - false, - ) - .await - .map_err(|e| Error::NotFound(format!("Access to resource {} denied: ({e})", path)))?; - - let git_resource: GitRepositoryResource = match git_repo_resource_value { - Some(value) => serde_json::from_value(value).map_err(|e| { - Error::BadRequest(format!("Invalid git repository resource format: {}", e)) - })?, - None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), - }; - - let identities: Vec = query - .git_ssh_identity - .map(|s| { - s.split(",") - .filter_map(|s| { - if !s.is_empty() { - Some(s.to_string()) - } else { - None - } - }) - .collect() - }) - .unwrap_or(vec![]); - - let (git_ssh_cmd, filenames) = - get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; - - let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; - - delete_paths(&filenames).await; - - Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) -} - -async fn write_ssh_file( - authed: &ApiAuthed, - user_db: &UserDB, - db: &DB, - w_id: &str, - var_path: &str, -) -> std::result::Result { - let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); - let loc = std::path::Path::new(TMP_DIR) - .join("ssh_ids") - .join(id_file_name); - - let userdb_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); - let mut content = get_value_internal(&userdb_authed, &w_id, &var_path, false) - .await - .map_err(|e| { - ( - error::Error::NotFound(format!( - "Variable {var_path} not found for git ssh identity: {e:#}" - )), - loc.clone(), - ) - })?; - content.push_str("\n"); - - if let Some(p) = &loc.parent() { - tokio::fs::create_dir_all(p) - .await - .map_err(|e| (e.into(), loc.clone()))?; - } - tokio::fs::write(&loc, content) - .await - .map_err(|e| (e.into(), loc.clone()))?; - - #[cfg(unix)] - { - let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); - tokio::fs::set_permissions(&loc, perm) - .await - .map_err(|e| (e.into(), loc.clone()))?; - } - - return Ok(loc); -} - -async fn delete_paths(paths: &Vec) { - for path in paths { - let _ = tokio::fs::remove_file(&path).await; - } -} - -async fn get_git_ssh_cmd( - authed: &ApiAuthed, - user_db: &UserDB, - db: &DB, - w_id: &str, - git_ssh_identity: Vec, -) -> error::Result<(Option, Vec)> { - if git_ssh_identity.len() > 5 { - return Err(error::Error::BadRequest( - "Too many ssh identities, try using at most 1".to_string(), - )); - } - if git_ssh_identity.len() == 0 { - return Ok((None, vec![])); - } - - let mut ssh_id_files = vec![]; - let mut file_paths = vec![]; - for var_path in git_ssh_identity.iter() { - match write_ssh_file(authed, user_db, db, w_id, &var_path).await { - Ok(loc) => { - ssh_id_files.push(format!( - " -i '{}'", - loc.to_string_lossy().replace('\'', r"'\''") - )); - file_paths.push(loc); - } - Err((e, loc)) => { - file_paths.push(loc); - delete_paths(&file_paths).await; - return Err(e); - } - } - } - - let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); - Ok((Some(git_ssh_cmd), file_paths)) -} - -async fn get_repo_latest_commit_hash( - git_resource: &GitRepositoryResource, - git_ssh_command: Option, -) -> Result { - // Validate URL and branch to prevent option injection attacks - validate_git_url(&git_resource.url)?; - - let ref_spec = git_resource - .branch - .as_deref() - .filter(|s| !s.is_empty()) - .unwrap_or("HEAD"); - - // Validate ref_spec if it's not the default HEAD - if ref_spec != "HEAD" { - validate_git_ref(ref_spec)?; - } - - let mut git_cmd = Command::new("git"); - git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); - if let Some(git_ssh_command) = git_ssh_command { - git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); - } - git_cmd.stderr(Stdio::piped()); - - let output = git_cmd - .output() - .await - .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; - - if !output.status.success() { - let stderr = String::from_utf8(output.stderr) - .unwrap_or_else(|_| "Failed to decode stderr".to_string()); - return Err(Error::BadRequest(format!( - "Error getting git repo commit hash: {}", - stderr - ))); - } - - let stdout = String::from_utf8(output.stdout) - .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; - - let lines: Vec<&str> = stdout.lines().collect(); - - if lines.is_empty() { - return Err(Error::BadRequest(format!( - "No commits found for reference '{}' in repository '{}'", - ref_spec, git_resource.url - ))); - } - - let commit_hash = lines - .first() - .and_then(|line| line.split_whitespace().next()) - .map(|s| s.to_string()) - .ok_or_else(|| { - Error::BadRequest("Unexpected output format for git ls-remote".to_string()) - })?; - - Ok(commit_hash) -} +pub use windmill_store::resources::*; diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index ace6833bd2..a3200ca95a 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -1001,8 +1001,8 @@ async fn set_default_error_handler( DeployedObject::Schedule { path: updated_schedule_path }, None, true, - None, - ) + None, + ) .await?; } } diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs index 41d2267a94..0212df6da6 100644 --- a/backend/windmill-api/src/secret_backend_ext.rs +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -14,15 +14,20 @@ //! Note: HashiCorp Vault integration requires Enterprise Edition. //! The OSS version only supports the database backend. +#[cfg(all(feature = "private", feature = "enterprise"))] use std::sync::Arc; use windmill_common::{ db::DB, - error::{Error, Result}, - secret_backend::{database::DatabaseBackend, SecretBackend}, - variables::{build_crypt, decrypt, encrypt}, + error::Result, }; +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::error::Error; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::secret_backend::{database::DatabaseBackend, SecretBackend}; + #[cfg(all(feature = "private", feature = "enterprise"))] use windmill_common::{ global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, @@ -45,17 +50,9 @@ lazy_static::lazy_static! { static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); } -/// Get the current secret backend based on global settings -/// -/// OSS: Always returns DatabaseBackend -/// EE: Returns configured backend (Database or Vault) -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn get_secret_backend(db: &DB) -> Result> { - Ok(Arc::new(DatabaseBackend::new(db.clone()))) -} - +/// Get the current secret backend based on global settings (EE only) #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn get_secret_backend(db: &DB) -> Result> { +async fn get_secret_backend(db: &DB) -> Result> { let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { Some(value) => serde_json::from_value::(value).unwrap_or_default(), None => SecretBackendConfig::default(), @@ -109,25 +106,14 @@ async fn get_or_create_vault_backend( }; // Cache it - *cache = Some(CachedVaultBackend { - backend: backend.clone(), - settings, - }); + *cache = Some(CachedVaultBackend { backend: backend.clone(), settings }); Ok(backend) } -/// Check if a Vault backend is currently configured -/// -/// OSS: Always returns false -/// EE: Checks global settings -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn is_vault_backend_configured(_db: &DB) -> Result { - Ok(false) -} - +/// Check if a Vault backend is currently configured (EE only) #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn is_vault_backend_configured(db: &DB) -> Result { +async fn is_vault_backend_configured(db: &DB) -> Result { let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { Some(value) => serde_json::from_value::(value).unwrap_or_default(), None => SecretBackendConfig::default(), @@ -136,186 +122,10 @@ pub async fn is_vault_backend_configured(db: &DB) -> Result { Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) } -/// Get a secret value using the configured backend -/// -/// For database backend: decrypts using workspace key -/// For vault backend (EE only): fetches from Vault directly -pub async fn get_secret_value( - db: &DB, - workspace_id: &str, - path: &str, - encrypted_value: &str, -) -> Result { - let backend = get_secret_backend(db).await?; - - match backend.backend_name() { - "database" => { - // Use existing database decryption - let mc = build_crypt(db, workspace_id).await?; - decrypt(&mc, encrypted_value.to_string()).map_err(|e| { - Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) - }) - } - "hashicorp_vault" => { - // Fetch from Vault directly - backend.get_secret(workspace_id, path).await - } - _ => Err(Error::internal_err(format!( - "Unknown backend: {}", - backend.backend_name() - ))), - } -} - -/// Store a secret value using the configured backend -/// -/// For database backend: encrypts using workspace key and returns encrypted value -/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage -pub async fn store_secret_value( - db: &DB, - workspace_id: &str, - path: &str, - plain_value: &str, -) -> Result { - let backend = get_secret_backend(db).await?; - - match backend.backend_name() { - "database" => { - // Use existing database encryption - let mc = build_crypt(db, workspace_id).await?; - Ok(encrypt(&mc, plain_value)) - } - "hashicorp_vault" => { - // Store in Vault and return a marker for DB - backend.set_secret(workspace_id, path, plain_value).await?; - // Return a marker indicating the value is stored in Vault - // The actual value in the DB will be this marker - Ok(format!("$vault:{}", path)) - } - _ => Err(Error::internal_err(format!( - "Unknown backend: {}", - backend.backend_name() - ))), - } -} - -/// Delete a secret from the configured backend (if using Vault) -/// -/// For database backend: no-op (DB delete is handled separately) -/// For vault backend (EE only): deletes from Vault -pub async fn delete_secret_from_backend( - db: &DB, - workspace_id: &str, - path: &str, -) -> Result<()> { - if is_vault_backend_configured(db).await? { - let backend = get_secret_backend(db).await?; - // Ignore NotFound errors during deletion (secret might not exist in Vault) - match backend.delete_secret(workspace_id, path).await { - Ok(()) => Ok(()), - Err(Error::NotFound(_)) => Ok(()), - Err(e) => Err(e), - } - } else { - Ok(()) - } -} - /// Check if a value is stored in Vault (indicated by the $vault: prefix) -pub fn is_vault_stored_value(value: &str) -> bool { - value.starts_with("$vault:") -} - -/// Rename a secret in Vault when a variable path changes (EE only) -/// -/// This function: -/// 1. Reads the secret value from the old path -/// 2. Writes it to the new path -/// 3. Deletes from the old path -/// 4. Returns the new marker value ($vault:new_path) -/// -/// If the value is not a Vault-stored value, returns None (no action needed). -/// If Vault is not configured, returns None. -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn rename_vault_secret( - _db: &DB, - _workspace_id: &str, - _old_path: &str, - new_path: &str, - current_value: &str, -) -> Result> { - // OSS: If value has $vault: prefix, just update the reference - // (This handles edge case where EE was used before downgrading to OSS) - if is_vault_stored_value(current_value) { - tracing::warn!( - "Variable has $vault: prefix but Vault requires Enterprise Edition. \ - Updating DB reference to {}", - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - Ok(None) -} - #[cfg(all(feature = "private", feature = "enterprise"))] -pub async fn rename_vault_secret( - db: &DB, - workspace_id: &str, - old_path: &str, - new_path: &str, - current_value: &str, -) -> Result> { - // Only handle Vault-stored values - if !is_vault_stored_value(current_value) { - return Ok(None); - } - - // Check if Vault backend is configured - if !is_vault_backend_configured(db).await? { - // Vault not configured but value has $vault: prefix - this is an inconsistent state - // Log warning and return new marker to at least update the DB reference - tracing::warn!( - "Variable value has $vault: prefix but Vault is not configured. \ - Updating DB reference from {} to {}", - old_path, - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - - let backend = get_secret_backend(db).await?; - - // Read from old path - let secret_value = match backend.get_secret(workspace_id, old_path).await { - Ok(value) => value, - Err(Error::NotFound(_)) => { - // Secret doesn't exist in Vault - just update the DB reference - tracing::warn!( - "Secret not found in Vault at path {} during rename to {}", - old_path, - new_path - ); - return Ok(Some(format!("$vault:{}", new_path))); - } - Err(e) => return Err(e), - }; - - // Write to new path - backend - .set_secret(workspace_id, new_path, &secret_value) - .await?; - - // Delete from old path (ignore errors - new path is already written) - if let Err(e) = backend.delete_secret(workspace_id, old_path).await { - tracing::warn!( - "Failed to delete old secret at {} after rename to {}: {}", - old_path, - new_path, - e - ); - } - - Ok(Some(format!("$vault:{}", new_path))) +fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") } /// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) @@ -383,7 +193,10 @@ pub async fn rename_vault_secrets_with_prefix( }; // Write to new path - if let Err(e) = backend.set_secret(workspace_id, &new_path, &secret_value).await { + if let Err(e) = backend + .set_secret(workspace_id, &new_path, &secret_value) + .await + { tracing::error!( "Failed to write secret to {} during bulk rename: {}", new_path, diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 82c1b68d86..ed6368dada 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -11,13 +11,16 @@ use std::collections::HashMap; use windmill_common::error::Error; use windmill_common::variables::get_secret_value_as_admin; -use crate::{approvals::{ - extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, - MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, - ResumeFormField, ResumeSchema, -}, auth::OptTokened}; use crate::db::{ApiAuthed, DB}; use crate::jobs::{QueryApprover, ResumeUrls}; +use crate::{ + approvals::{ + extract_w_id_from_resume_url, handle_resume_action, ApprovalFormDetails, FieldType, + MessageFormat, QueryDefaultArgsJson, QueryDynamicEnumJson, QueryFlowStepId, QueryMessage, + ResumeFormField, ResumeSchema, + }, + auth::OptTokened, +}; #[derive(Deserialize, Debug)] pub struct SlackFormData { @@ -125,8 +128,12 @@ pub async fn slack_app_callback_handler( tracing::debug!("Payload: {:#?}", payload); match payload.r#type { - PayloadType::ViewSubmission => handle_submission(authed, opt_tokened, db, &payload, "resume").await?, - PayloadType::ViewClosed => handle_submission(authed, opt_tokened, db, &payload, "cancel").await?, + PayloadType::ViewSubmission => { + handle_submission(authed, opt_tokened, db, &payload, "resume").await? + } + PayloadType::ViewClosed => { + handle_submission(authed, opt_tokened, db, &payload, "cancel").await? + } _ => { if let Some(actions) = &payload.actions { if let Some(action) = actions.first() { @@ -296,7 +303,15 @@ async fn handle_submission( } // Use the common handler to process the resume/cancel action - handle_resume_action(authed, opt_tokened, db.clone(), &resume_url, state_json, action).await?; + handle_resume_action( + authed, + opt_tokened, + db.clone(), + &resume_url, + state_json, + action, + ) + .await?; let w_id = extract_w_id_from_resume_url(&resume_url)?; let slack_token = get_slack_token(&db, &resource_path, w_id).await?; diff --git a/backend/windmill-api/src/teams_oss.rs b/backend/windmill-api/src/teams_oss.rs index 037e556c99..0d4fa7f17a 100644 --- a/backend/windmill-api/src/teams_oss.rs +++ b/backend/windmill-api/src/teams_oss.rs @@ -44,7 +44,6 @@ pub async fn run_teams_message_test_job() -> Result { )); } - #[cfg(all(feature = "enterprise", not(feature = "private")))] pub fn teams_service() -> Router { Router::new() diff --git a/backend/windmill-api/src/triggers/email/mod.rs b/backend/windmill-api/src/triggers/email/mod.rs index 836af77fa0..1210a56460 100644 --- a/backend/windmill-api/src/triggers/email/mod.rs +++ b/backend/windmill-api/src/triggers/email/mod.rs @@ -1,11 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct EmailTrigger; +pub use windmill_trigger_email::*; diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs index dd2f2a65df..c12f93724d 100644 --- a/backend/windmill-api/src/triggers/gcp/mod.rs +++ b/backend/windmill-api/src/triggers/gcp/mod.rs @@ -1,16 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Clone, Copy)] -pub struct GcpTrigger; +pub use windmill_trigger_gcp::*; diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs index 4ca7e2e15e..660ecf6051 100644 --- a/backend/windmill-api/src/triggers/handler.rs +++ b/backend/windmill-api/src/triggers/handler.rs @@ -1,685 +1,11 @@ -use crate::{ - db::ApiAuthed, - triggers::{StandardTriggerQuery, TriggerData, TriggerMode}, -}; -use async_trait::async_trait; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::{FromRow, PgConnection}; -use std::fmt::Debug; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::{paginate, Pagination, StripPath}, - worker::CLOUD_HOSTED, - DB, -}; -use windmill_git_sync::DeployedObject; - -use axum::{ - extract::{Path, Query}, - http::StatusCode, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use std::sync::Arc; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_git_sync::handle_deployment_metadata; - -use crate::utils::check_scopes; - -#[async_trait] -pub trait TriggerCrud: Send + Sync + 'static { - type Trigger: Serialize - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Send - + Sync - + Unpin; - - type TriggerConfig: Debug - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Serialize - + Send - + Sync - + Unpin; - - type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; - type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - - const TABLE_NAME: &'static str; - const TRIGGER_TYPE: &'static str; - const SUPPORTS_SERVER_STATE: bool; - const SUPPORTS_TEST_CONNECTION: bool; - const ROUTE_PREFIX: &'static str; - const DEPLOYMENT_NAME: &'static str; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; - const IS_ALLOWED_ON_CLOUD: bool; - - fn get_deployed_object(path: String) -> DeployedObject; - - async fn validate_new( - &self, - db: &DB, - workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - self.validate_config(db, new, workspace_id).await - } - - async fn validate_edit( - &self, - db: &DB, - workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - self.validate_config(db, edit, workspace_id).await - } - - async fn validate_config( - &self, - _db: &DB, - _config: &Self::TriggerConfigRequest, - _workspace_id: &str, - ) -> Result<()> { - Ok(()) - } - - fn scope_domain_name() -> &'static str { - &Self::ROUTE_PREFIX[1..] - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn test_connection( - &self, - _db: &DB, - _authed: &ApiAuthed, - _user_db: &UserDB, - _workspace_id: &str, - _config: Self::TestConnectionConfig, - ) -> Result<()> { - Err( - anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) - .into(), - ) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn get_trigger_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND - path = $2 - "#, - fields.join(", "), - Self::TABLE_NAME - ); - - sqlx::query_as(&sql) - .bind(workspace_id) - .bind(path) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) - } - - async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { - let exists = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .fetch_one(db) - .await?; - - Ok(exists) - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) - } - - async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { - Ok(()) - } - - async fn set_trigger_mode( - &self, - authed: &ApiAuthed, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - mode: &TriggerMode, - ) -> Result { - let updated = if Self::SUPPORTS_SERVER_STATE { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - } else { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now() - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - }; - - self.set_trigger_mode_extra_action(&mut *tx).await?; - - Ok(updated > 0) - } - - #[allow(unused)] - async fn trigger_count( - &self, - tx: &mut PgConnection, - workspace_id: &str, - is_flow: bool, - script_path: &str, - ) -> i64 { - let count = sqlx::query_scalar(&format!( - r#" - SELECT - COUNT(*) - FROM - {} - WHERE - workspace_id = $1 AND - is_flow = $2 AND - script_path = $3 - "#, - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(is_flow) - .bind(script_path) - .fetch_one(&mut *tx) - .await - .unwrap_or(0); - - count - } - - async fn list_triggers( - &self, - tx: &mut PgConnection, - workspace_id: &str, - query: Option<&StandardTriggerQuery>, - ) -> Result> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&workspace_id)); - - if let Some(query) = query { - let (per_page, offset) = - paginate(Pagination { per_page: query.per_page, page: query.page }); - if let Some(path) = &query.path { - sqlb.and_where_eq("script_path", "?".bind(path)); - } - - if let Some(is_flow) = query.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - - if let Some(path_start) = &query.path_start { - sqlb.and_where_like_left("path", path_start); - } - - sqlb.offset(offset).limit(per_page); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; - - Ok(triggers) - } -} - -pub fn trigger_routes() -> Router { - let mut router = Router::new() - .route("/create", post(create_trigger::)) - .route("/list", get(list_triggers::)) - .route("/get/*path", get(get_trigger::)) - .route("/update/*path", post(update_trigger::)) - .route("/delete/*path", delete(delete_trigger::)) - .route("/exists/*path", get(exists_trigger::)) - .route("/setmode/*path", post(set_trigger_mode::)); - - if T::SUPPORTS_TEST_CONNECTION { - router = router.route("/test", post(test_connection::)); - } - - router -} - -async fn create_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(new_trigger): Json>, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &new_trigger.base.path - ) - })?; - - if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { - return Err(Error::BadRequest(format!( - "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", - T::TRIGGER_TYPE - ))); - } - - handler - .validate_new(&db, &workspace_id, &new_trigger.config) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = new_trigger.base.path.clone(); - - handler - .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.create", T::TRIGGER_TYPE), - ActionKind::Create, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok((StatusCode::CREATED, new_path)) -} - -async fn list_triggers( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path(workspace_id): Path, - Query(query): Query, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let triggers = handler - .list_triggers(&mut *tx, &workspace_id, Some(&query)) - .await?; - tx.commit().await?; - - Ok(Json(triggers)) -} - -async fn get_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let trigger = handler - .get_trigger_by_path(&mut *tx, &workspace_id, path) - .await?; - - tx.commit().await?; - - Ok(Json(trigger)) -} - -async fn update_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(edit_trigger): Json>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &edit_trigger.base.path - ) - })?; - - handler - .validate_edit(&db, &workspace_id, &edit_trigger.config, path) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = edit_trigger.base.path.to_string(); - - handler - .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.update", T::TRIGGER_TYPE), - ActionKind::Update, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' updated", path)) -} - -async fn delete_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:write:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let deleted = handler - .delete_by_path(&mut *tx, &workspace_id, path) - .await?; - - if !deleted { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.delete", T::TRIGGER_TYPE), - ActionKind::Delete, - &workspace_id, - Some(&path), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' deleted", path)) -} - -async fn exists_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), path) - })?; - let exists = handler.exists(&db, &workspace_id, path).await?; - - Ok(Json(exists)) -} - -#[derive(serde::Deserialize)] -struct SetTriggerModePayload { - mode: TriggerMode, -} - -async fn set_trigger_mode( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; - - let mut tx = user_db.begin(&authed).await?; - let updated = handler - .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) - .await?; - - if !updated { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(path.to_owned()), - Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), - true, - None, - ) - .await?; - - Ok(format!( - "Trigger '{}' {}", - path, - if payload.mode == TriggerMode::Enabled { - "enabled" - } else if payload.mode == TriggerMode::Disabled { - "disabled" - } else { - "suspended" - } - )) -} - -async fn test_connection( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(config): Json, -) -> Result<()> { - let connect_f = async move { - handler - .test_connection(&db, &authed, &user_db, &workspace_id, config) - .await - }; - - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) - })??; - Ok(()) -} - -#[allow(unused)] -pub fn complete_trigger_routes(handler: T) -> Router { - let standard_routes = trigger_routes::(); - - let additional_routes = handler.additional_routes(); - - standard_routes - .merge(additional_routes) - .layer(Extension(Arc::new(handler))) -} +use axum::{routing::post, Router}; +use serde::{Deserialize, Serialize}; +use windmill_common::{error::JsonResult, DB}; + +#[allow(unused_imports)] +use windmill_trigger::handler::complete_trigger_routes; +#[allow(unused_imports)] +use windmill_trigger::TriggerCrud; pub fn generate_trigger_routers() -> Router { #[allow(unused_mut)] @@ -687,7 +13,7 @@ pub fn generate_trigger_routers() -> Router { #[cfg(feature = "http_trigger")] { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; router = router.nest( HttpTrigger::ROUTE_PREFIX, @@ -776,7 +102,7 @@ pub fn generate_trigger_routers() -> Router { } { - use crate::triggers::global_handler::{ + use windmill_trigger::global_handler::{ cancel_suspended_trigger_jobs, resume_suspended_trigger_jobs, }; @@ -794,14 +120,9 @@ pub fn generate_trigger_routers() -> Router { router } -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggerPrimarySchedule { - schedule: String, -} - #[derive(Serialize, Deserialize, Debug)] pub struct TriggersCount { - primary_schedule: Option, + primary_schedule: Option, schedule_count: i64, http_routes_count: i64, webhook_count: i64, @@ -847,7 +168,7 @@ pub async fn get_triggers_count_internal( #[cfg(feature = "http_trigger")] let http_routes_count = { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; let count = HttpTrigger .trigger_count(&mut tx, w_id, is_flow, path) .await; @@ -984,8 +305,9 @@ pub async fn get_triggers_count_internal( .await? .unwrap_or(0); - Ok(Json(TriggersCount { - primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), + Ok(axum::Json(TriggersCount { + primary_schedule: primary_schedule + .map(|s| windmill_trigger::handler::TriggerPrimarySchedule { schedule: s }), schedule_count, http_routes_count, webhook_count, diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index a38532c46d..211861f17e 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -1,51 +1,36 @@ use super::{ - http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, RequestType, - TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, + http_trigger_args::RawHttpTriggerArgs, refresh_routers, AuthenticationMethod, HttpMethod, + RequestType, TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, }; use crate::{ auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, jobs::start_job_update_sse_stream, resources::try_get_resource_from_db_as, - triggers::{ - http::{ - refresh_routers, validate_authentication_method, HttpConfig, HttpConfigRequest, - RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, - }, - trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, - trigger_runnable_inner, RunnableId, - }, - Trigger, TriggerCrud, TriggerData, TriggerMode, + triggers::trigger_helpers::{ + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, + trigger_runnable_inner, RunnableId, }, users::fetch_api_authed, utils::{check_scopes, ExpiringCacheEntry}, }; use axum::{ - async_trait, extract::Path, response::{IntoResponse, Response}, - routing::{get, post}, - Extension, Json, Router, + routing::get, + Extension, Router, }; use futures::StreamExt; use http::{HeaderMap, StatusCode}; -use sqlx::PgConnection; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - sync::Arc, -}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; +use std::{collections::HashMap, sync::Arc}; use windmill_common::{ db::UserDB, error::{Error, Result}, jobs::JobTriggerKind, triggers::{TriggerKind, TriggerMetadata}, - utils::{not_found_if_none, require_admin, StripPath}, - worker::CLOUD_HOSTED, + utils::{not_found_if_none, StripPath}, }; -use windmill_git_sync::handle_deployment_metadata; +use windmill_trigger::TriggerMode; #[cfg(feature = "parquet")] use { @@ -53,572 +38,6 @@ use { windmill_common::s3_helpers::build_object_store_client, }; -use windmill_git_sync::DeployedObject; - -pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { - sqlx::query!("SELECT nextval('http_trigger_version_seq')") - .fetch_one(tx) - .await?; - Ok(()) -} - -pub fn generate_route_path_key(route_path: &str) -> String { - ROUTE_PATH_KEY_RE - .replace_all(route_path, "${1}${2}key") - .to_string() -} - -pub async fn route_path_key_exists( - route_path_key: &str, - http_method: &HttpMethod, - w_id: &str, - trigger_path: Option<&str>, - workspaced_route: Option, - db: &DB, -) -> Result { - let exists = if *CLOUD_HOSTED { - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - route_path_key = $1 - AND workspace_id = $2 - AND http_method = $3 - AND ($4::TEXT IS NULL OR path != $4) - ) - "#, - &route_path_key, - w_id, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - } else { - let route_path_key = match workspaced_route { - Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), - _ => Cow::Borrowed(route_path_key), - }; - - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) - OR (workspaced_route IS FALSE AND route_path_key = $1)) - AND http_method = $2 - AND ($3::TEXT IS NULL OR path != $3) - ) - "#, - &route_path_key, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; - - Ok(exists) -} - -pub async fn exists_route( - Extension(db): Extension, - Path(w_id): Path, - Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< - RouteExists, - >, -) -> Result> { - let route_path_key = generate_route_path_key(&route_path); - - let exists = route_path_key_exists( - &route_path_key, - &http_method, - &w_id, - trigger_path.as_deref(), - workspaced_route, - &db, - ) - .await?; - - Ok(Json(exists)) -} - -fn check_no_duplicates<'trigger>( - new_http_triggers: &[TriggerData], - route_path_key: &[String], -) -> Result<()> { - let mut seen = HashSet::with_capacity(new_http_triggers.len()); - - for (i, trigger) in new_http_triggers.iter().enumerate() { - if !seen.insert(( - &route_path_key[i], - trigger.config.http_method, - trigger.config.workspaced_route, - )) { - return Err(Error::BadRequest(format!( - "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", - &trigger.config.route_path - ))); - } - } - - Ok(()) -} - -pub async fn insert_new_trigger_into_db( - authed: &ApiAuthed, - tx: &mut PgConnection, - w_id: &str, - trigger: &TriggerData, - route_path_key: &str, -) -> Result<()> { - require_admin(authed.is_admin, &authed.username)?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - INSERT INTO http_trigger ( - workspace_id, - path, - route_path, - route_path_key, - workspaced_route, - authentication_resource_path, - wrap_body, - raw_string, - script_path, - summary, - description, - is_flow, - mode, - request_type, - authentication_method, - http_method, - static_asset_config, - edited_by, - email, - edited_at, - is_static_website, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 - ) - "#, - w_id, - trigger.base.path, - trigger.config.route_path, - route_path_key, - trigger.config.workspaced_route.unwrap_or(false), - trigger.config.authentication_resource_path, - trigger.config.wrap_body.unwrap_or(false), - trigger.config.raw_string.unwrap_or(false), - trigger.base.script_path, - trigger.config.summary, - trigger.config.description, - trigger.base.is_flow, - trigger.base.mode() as _, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(&mut *tx) - .await?; - Ok(()) -} - -pub async fn create_many_http_triggers( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_http_triggers): Json>>, -) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - let handler = HttpTrigger; - - let error_wrapper = |route_path: &str, error: Error| -> Error { - anyhow::anyhow!( - "Error occurred for HTTP route at route path: {}, error: {}", - route_path, - error - ) - .into() - }; - - let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); - - for new_http_trigger in new_http_triggers.iter() { - handler - .validate_new(&db, &w_id, &new_http_trigger.config) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - let route_path_key = - check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; - - route_path_keys.push(route_path_key.clone()); - } - - check_no_duplicates(&new_http_triggers, &route_path_keys)?; - - let mut tx = user_db.begin(&authed).await?; - - for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { - insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - audit_log( - &mut *tx, - &authed, - "http_trigger.create", - ActionKind::Create, - &w_id, - Some(&new_http_trigger.base.path), - None, - ) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - - increase_trigger_version(&mut tx) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - } - - tx.commit().await?; - - for http_trigger in new_http_triggers.into_iter() { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, - Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), - true, - None, - ) - .await - .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; - } - Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) -} - -async fn check_if_route_exist( - db: &DB, - config: &HttpConfigRequest, - workspace_id: &str, - trigger_path: Option<&str>, -) -> Result { - let route_path_key = generate_route_path_key(&config.route_path); - - let exists = route_path_key_exists( - &route_path_key, - &config.http_method, - workspace_id, - trigger_path, - config.workspaced_route, - db, - ) - .await?; - - if exists { - return Err(Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - Ok(route_path_key) -} - -pub struct HttpTrigger; - -#[async_trait] -impl TriggerCrud for HttpTrigger { - type TriggerConfig = HttpConfig; - type Trigger = Trigger; - type TriggerConfigRequest = HttpConfigRequest; - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = "http_trigger"; - const TRIGGER_TYPE: &'static str = "http"; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/http_triggers"; - const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; - const IS_ALLOWED_ON_CLOUD: bool = true; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "route_path", - "route_path_key", - "request_type", - "authentication_method", - "http_method", - "summary", - "description", - "static_asset_config", - "is_static_website", - "authentication_resource_path", - "workspaced_route", - "wrap_body", - "raw_string", - ]; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::HttpTrigger { path } - } - - fn additional_routes(&self) -> Router { - Router::new() - .route("/create_many", post(create_many_http_triggers)) - .route("/route_exists", post(exists_route)) - } - - async fn validate_new( - &self, - _db: &DB, - _workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method(new.authentication_method, new.raw_string)?; - - Ok(()) - } - - async fn validate_edit( - &self, - _db: &DB, - _workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - validate_authentication_method(edit.authentication_method, edit.raw_string)?; - - Ok(()) - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; - - insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - if authed.is_admin { - if trigger.config.route_path.is_empty() { - return Err(Error::BadRequest("route_path is required".to_string())); - }; - - let route_path = &trigger.config.route_path; - if !VALID_ROUTE_PATH_RE.is_match(route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - let route_path_key = - check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - route_path = $1, - route_path_key = $2, - workspaced_route = $3, - wrap_body = $4, - raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - email = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, - edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 - WHERE - workspace_id = $23 AND - path = $24 - "#, - route_path, - &route_path_key, - trigger.config.workspaced_route, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } else { - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - wrap_body = $1, - raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - email = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, - edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 - WHERE - workspace_id = $20 AND - path = $21 - "#, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { - increase_trigger_version(tx).await - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - increase_trigger_version(tx).await?; - - Ok(deleted > 0) - } -} - async fn conditional_cors_middleware( req: axum::extract::Request, next: axum::middleware::Next, @@ -757,10 +176,10 @@ async fn get_http_route_trigger( r#" SELECT EXISTS( SELECT 1 - FROM - http_trigger - WHERE - workspace_id = $1 AND + FROM + http_trigger + WHERE + workspace_id = $1 AND path = $2 ) "#, diff --git a/backend/windmill-api/src/triggers/http/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs index af829cec53..4866f9be34 100644 --- a/backend/windmill-api/src/triggers/http/http_trigger_args.rs +++ b/backend/windmill-api/src/triggers/http/http_trigger_args.rs @@ -4,7 +4,7 @@ use axum::{ extract::{FromRequest, Request}, response::Response, }; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use serde_json::value::RawValue; use windmill_common::{ error::Error, @@ -14,6 +14,7 @@ use windmill_common::{ }; use windmill_queue::PushArgsOwned; +use super::HttpMethod; use crate::{ args::{ build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs, @@ -24,31 +25,6 @@ use crate::{ pub struct RawHttpTriggerArgs(pub RawWebhookArgs); -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - #[axum::async_trait] impl FromRequest for RawHttpTriggerArgs where diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs index 291d9eadfe..f65a26c03e 100644 --- a/backend/windmill-api/src/triggers/http/mod.rs +++ b/backend/windmill-api/src/triggers/http/mod.rs @@ -1,410 +1,4 @@ -use std::collections::HashMap; - -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use tokio::sync::{RwLock, RwLockReadGuard}; -use windmill_common::{ - error::{Error, Result}, - flows::Retry, - s3_helpers::S3Object, - worker::CLOUD_HOSTED, - DB, -}; - -use crate::{db::ApiAuthed, triggers::TriggerMode, utils::ExpiringCacheEntry}; +pub use windmill_trigger_http::*; pub mod handler; pub mod http_trigger_args; -pub mod http_trigger_auth; - -lazy_static::lazy_static! { - static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); - static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); - - static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { - routers: HashMap::new(), - version: 0, - }); -} - -#[derive(Debug, Deserialize, Clone)] -pub struct TriggerRoute { - path: String, - script_path: String, - is_flow: bool, - route_path: String, - workspace_id: String, - request_type: RequestType, - authentication_method: AuthenticationMethod, - edited_by: String, - email: String, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: bool, - wrap_body: bool, - raw_string: bool, - error_handler_path: Option, - error_handler_args: Option>>, - retry: Option>, - mode: TriggerMode, -} - -pub struct RoutersCache { - routers: HashMap>, - version: i64, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] -#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum RequestType { - Sync, - Async, - SyncSse, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] -#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] -#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] -pub enum AuthenticationMethod { - None, - Windmill, - ApiKey, - BasicHttp, - CustomScript, - Signature, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct HttpConfig { - pub route_path: String, - pub route_path_key: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: bool, - pub wrap_body: bool, - pub raw_string: bool, -} - -#[derive(Debug, Clone, Serialize)] -pub struct HttpConfigRequest { - #[serde(default)] - pub route_path: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: Option, - pub wrap_body: Option, - pub raw_string: Option, -} - -#[derive(Deserialize)] -struct HttpConfigRequestHelper { - #[serde(default)] - route_path: String, - request_type: Option, - is_async: Option, - authentication_method: AuthenticationMethod, - http_method: HttpMethod, - summary: Option, - description: Option, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: Option, - wrap_body: Option, - raw_string: Option, -} - -impl<'de> Deserialize<'de> for HttpConfigRequest { - fn deserialize(deserializer: D) -> std::result::Result - where - D: serde::Deserializer<'de>, - { - let helper = HttpConfigRequestHelper::deserialize(deserializer)?; - - // Determine request_type with backward compatibility - let request_type = if let Some(mode) = helper.request_type { - mode - } else if let Some(is_async) = helper.is_async { - if is_async { - RequestType::Async - } else { - RequestType::Sync - } - } else { - RequestType::Sync - }; - - Ok(HttpConfigRequest { - route_path: helper.route_path, - request_type, - authentication_method: helper.authentication_method, - http_method: helper.http_method, - summary: helper.summary, - description: helper.description, - static_asset_config: helper.static_asset_config, - is_static_website: helper.is_static_website, - authentication_resource_path: helper.authentication_resource_path, - workspaced_route: helper.workspaced_route, - wrap_body: helper.wrap_body, - raw_string: helper.raw_string, - }) - } -} - -// Regex patterns for route validation -lazy_static::lazy_static! { - // Matches named params like :id or wildcards like :* or * - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); - static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); -} - -#[derive(Deserialize)] -pub struct RouteExists { - pub route_path: String, - pub http_method: HttpMethod, - pub trigger_path: Option, - pub workspaced_route: Option, -} - -pub fn validate_authentication_method( - authentication_method: AuthenticationMethod, - raw_string: Option, -) -> Result<()> { - match (authentication_method, raw_string) { - (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { - Err(Error::BadRequest( - "To use custom script authentication, please enable the raw body option." - .to_string(), - )) - } - _ => Ok(()), - } -} - -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { - let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) - .fetch_one(db) - .await?; - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { - drop(routers_cache); - let mut routers = HashMap::new(); - - for http_method in [ - HttpMethod::Get, - HttpMethod::Post, - HttpMethod::Put, - HttpMethod::Patch, - HttpMethod::Delete, - ] { - let triggers = sqlx::query_as!( - TriggerRoute, - r#" - SELECT - path, - script_path, - is_flow, - route_path, - authentication_resource_path, - workspace_id, - request_type AS "request_type: _", - authentication_method AS "authentication_method: _", - edited_by, - email, - static_asset_config AS "static_asset_config: _", - wrap_body, - raw_string, - workspaced_route, - is_static_website, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _", - mode as "mode: _" - FROM - http_trigger - WHERE - http_method = $1 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - "#, - &http_method as &HttpMethod - ) - .fetch_all(db) - .await?; - - let mut router = matchit::Router::new(); - - for trigger in triggers { - let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { - format!("/{}/{}", trigger.workspace_id, trigger.route_path) - } else { - format!("/{}", trigger.route_path) - }; - - if trigger.is_static_website { - router - .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider HTTP route {}/*wm_subpath: {:?}", - full_path, - e, - ); - }); - } - router - .insert(full_path.clone(), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); - }); - } - - routers.insert(http_method, router); - } - - let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; - - Ok((true, routers_cache.downgrade())) - } else { - tracing::debug!("No HTTP routers refresh needed"); - Ok((false, routers_cache)) - } -} - -pub async fn refresh_routers_loop( - db: &DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - match refresh_routers(db).await { - Ok(_) => { - tracing::info!("Loaded HTTP routers"); - } - Err(err) => { - tracing::error!("Error loading HTTP routers: {err:#}"); - } - }; - let db = db.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = killpill_rx.recv() => { - break; - } - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { - Ok((true, _)) => { - tracing::info!("Refreshed HTTP routers"); - } - Err(err) => { - tracing::error!("Error refreshing HTTP routers: {err:#}"); - } - _ => {} - } - } - } - } - }); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_request_type_backward_compatibility() { - // Test with new request_type field - let json_new = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - - // Test with legacy is_async = true - let json_legacy_async = r#"{ - "route_path": "/test", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); - assert_eq!(config.request_type, RequestType::Async); - - // Test with legacy is_async = false - let json_legacy_sync = r#"{ - "route_path": "/test", - "is_async": false, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test with neither field (default to sync) - let json_default = r#"{ - "route_path": "/test", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test that request_type takes precedence over is_async - let json_both = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - } -} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs index f09b2283ff..21e0c06b44 100644 --- a/backend/windmill-api/src/triggers/kafka/mod.rs +++ b/backend/windmill-api/src/triggers/kafka/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct KafkaTrigger; +pub use windmill_trigger_kafka::*; diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index ce4f651ba9..3e67f36109 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -1,846 +1,13 @@ -use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use windmill_common::DB; +#[allow(unused_imports)] +use windmill_trigger::listener::{listen_to_unlistened_events, listening}; +#[allow(unused_imports)] +use windmill_trigger::Listener; -use crate::{ - capture::insert_capture_payload, - db::ApiAuthed, - triggers::{ - handler::TriggerCrud, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - Trigger, TriggerErrorHandling, TriggerMode, - }, - users::fetch_api_authed, -}; -use async_trait::async_trait; -use itertools::Itertools; -use rand::seq::SliceRandom; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sql_builder::SqlBuilder; -use sqlx::{FromRow, Row}; -use tokio::sync::RwLock; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerKind, TriggerMetadata}, - utils::report_critical_error, - DB, INSTANCE_NAME, -}; - -#[allow(unused)] -#[async_trait] -pub trait Listener: TriggerCrud + TriggerJobArgs { - type Consumer: Send; - type Extra: Send + Sync; - type ExtraState: Send + Sync; - - const JOB_TRIGGER_KIND: JobTriggerKind; - const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; - const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; - - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result>; - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - extra: Option<&Self::ExtraState>, - ); - async fn fetch_enabled_unlistened_triggers( - &self, - db: &DB, - ) -> Result>> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - "error_handler_path", - "error_handler_args", - "retry", - ]; - - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - ); - - for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let triggers = triggers - .into_iter() - .map(|trigger| ListeningTrigger { - path: trigger.base.path, - workspace_id: trigger.base.workspace_id, - is_flow: trigger.base.is_flow, - username: trigger.base.edited_by, - email: trigger.base.email, - script_path: trigger.base.script_path, - trigger_config: trigger.config, - error_handling: Some(trigger.error_handling), - trigger_mode: true, - suspended_mode: trigger.base.mode == TriggerMode::Suspended, - }) - .collect_vec(); - - Ok(triggers) - } - - async fn fetch_unlistened_captures( - &self, - db: &DB, - ) -> Result>> { - let fields = vec![ - "path", - "is_flow", - "workspace_id", - "owner AS username", - "email", - "trigger_config", - ]; - - let mut sqlb = SqlBuilder::select_from("capture_config"); - sqlb.fields(&fields) - .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) - .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") - .and_where("trigger_config IS NOT NULL") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", - ); - - for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb.sql().expect("failed to build SQL"); - - let captures: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let captures = captures - .into_iter() - .map(|capture| ListeningTrigger { - username: capture.username, - path: capture.path, - workspace_id: capture.workspace_id, - script_path: "".to_string(), - email: capture.email, - trigger_config: capture.trigger_config, - trigger_mode: false, - is_flow: capture.is_flow, - error_handling: None, - suspended_mode: false, - }) - .collect_vec(); - - Ok(captures) - } - - async fn get_extra_state(&self) -> Option { - None - } - - async fn cleanup( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _extra: Option<&Self::ExtraState>, - ) -> Result<()> { - Ok(()) - } - - async fn loop_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - status: Arc>>, - error_message: Option, - ) { - update_rw_lock(status.clone(), error_message).await; - loop { - if let None = self - .update_ping(db, listening_trigger, status.read().await.as_deref()) - .await - { - return; - } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } - } - - async fn update_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - if listening_trigger.trigger_mode { - self.update_trigger_ping(db, listening_trigger, error).await - } else { - self.update_capture_ping(db, listening_trigger, error).await - } - } - - async fn update_ping_and_loop_ping_status( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - loop_ping_status: Arc>>, - error: Option, - ) -> Option<()> { - // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status - update_rw_lock(loop_ping_status.clone(), error.clone()).await; - if let None = self - .update_ping(db, listening_trigger, error.as_deref()) - .await - { - return None; - } - Some(()) - } - - async fn update_trigger_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar::<_, i32>(&format!( - r#" - UPDATE - {} - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - server_id = $4 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - RETURNING 1 - "#, - Self::TABLE_NAME - )) - .bind(error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .bind(&*INSTANCE_NAME) - .fetch_optional(db) - .await; - - self.handle_ping_result(updated, db, listening_trigger, "trigger") - .await - } - - async fn update_capture_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - server_id = $6 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind, - &*INSTANCE_NAME - ) - .fetch_optional(db) - .await - .map(|result| result.flatten()); - - self.handle_ping_result(updated, db, listening_trigger, "capture") - .await - } - - async fn handle_ping_result( - &self, - result: sqlx::Result>, - db: &DB, - listening_trigger: &ListeningTrigger, - entity_type: &str, - ) -> Option<()> { - match result { - Ok(updated) => { - if updated.is_none() { - self.reset_ping_for_restart(db, listening_trigger).await; - tracing::info!( - "{} {} {} changed, disabled, or deleted, stopping...", - Self::TRIGGER_KIND, - entity_type, - listening_trigger.path - ); - return None; - } - } - Err(error) => { - tracing::warn!( - "Error updating ping of {} {} {}: {:?}", - Self::TRIGGER_KIND, - entity_type, - &listening_trigger.path, - error - ); - } - } - - Some(()) - } - - async fn reset_ping_for_restart( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - ) { - if listening_trigger.trigger_mode { - let _ = sqlx::query(&format!( - r#" - UPDATE - {} - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - server_id IS NULL - "#, - Self::TABLE_NAME - )) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - } else { - let _ = sqlx::query!( - r#" - UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = $4 AND - server_id IS NULL - "#, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - } - } - - async fn disable_with_error( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: String, - ) { - if listening_trigger.trigger_mode { - let report_status = sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = 'disabled'::TRIGGER_MODE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - Self::TABLE_NAME - )) - .bind(&error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - - match report_status { - Ok(_) => { - report_critical_error( - format!( - "Disabling {} trigger {} because of error: {}", - Self::TRIGGER_KIND, - listening_trigger.path, - error - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ).await; - } - } - return; - } - - let report_status = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 - "#, - error, - listening_trigger.workspace_id, - listening_trigger.path, - listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - - if let Err(disable_err) = report_status { - tracing::error!( - "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", - Self::TRIGGER_KIND, - listening_trigger.path, - listening_trigger.workspace_id, - disable_err, - error - ) - } - } - - async fn handle_trigger( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - _extra: Option, - ) -> Result<()> { - let args = Self::build_job_args( - &listening_trigger.script_path, - listening_trigger.is_flow, - &listening_trigger.workspace_id, - db, - payload, - trigger_info, - ) - .await?; - - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let (retry, error_handler_path, error_handler_args) = - match listening_trigger.error_handling.as_ref() { - Some(error_handling) => ( - error_handling.retry.as_ref(), - error_handling.error_handler_path.as_deref(), - error_handling.error_handler_args.as_ref(), - ), - None => (None, None, None), - }; - - tracing::debug!( - "Triggering job from {} event {} with args {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - args - ); - - trigger_runnable( - db, - None, - authed, - &listening_trigger.workspace_id, - &listening_trigger.script_path, - listening_trigger.is_flow, - args, - retry, - error_handler_path.as_deref(), - error_handler_args, - format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), - None, - listening_trigger.suspended_mode, - TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), - ) - .await?; - - Ok(()) - } - - async fn handle_event( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - extra: Option, - ) -> Result<()> { - if listening_trigger.trigger_mode { - if let Err(err) = self - .handle_trigger(db, listening_trigger, payload, trigger_info, extra) - .await - { - report_critical_error( - format!( - "Failed to trigger job from {} event {}: {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - err - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - return Err(err); - }; - return Ok(()); - } - - let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); - if let Err(err) = insert_capture_payload( - db, - &listening_trigger.workspace_id, - &listening_trigger.path, - listening_trigger.is_flow, - &Self::TRIGGER_KIND, - main_args, - preprocessor_args, - &listening_trigger.username, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - return Err(err); - } - Ok(()) - } -} - -#[allow(unused)] -async fn listening( - db: DB, - listener: T, - listening_trigger: ListeningTrigger, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - let killpill_rx_consumer = killpill_rx.resubscribe(); - let killpill_rx_get_consumer = killpill_rx.resubscribe(); - - let loop_ping_status = Arc::new(RwLock::new(None)); - let extra_state = listener.get_extra_state().await; - let path = listening_trigger.path.clone(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - consumer = { - tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) - } => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { - tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = async { - match consumer { - Ok(Some(consumer)) => { - listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; - tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; - tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); - } - Err(error) => { - tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); - listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; - } - Ok(None) => { - tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); - } - } - } => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - } - } - } -} - -#[allow(unused)] -async fn listen_to_unlistened_events( - listener: T, - db: DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; - - match unlistend_enabled_triggers { - Ok(mut unlistend_enabled_triggers) => { - unlistend_enabled_triggers.shuffle(&mut rand::rng()); - for trigger in unlistend_enabled_triggers { - let has_lock = sqlx::query_scalar(&format!( - r#" - UPDATE - {} - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - T::TABLE_NAME, - )) - .bind(&*INSTANCE_NAME) - .bind(&trigger.workspace_id) - .bind(&trigger.path) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!( - "Spawning new task to listen for {} event", - T::TABLE_NAME - ); - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, trigger, killpill_rx).await } - }); - } else { - tracing::info!( - "{} trigger {} already being listened to", - T::TRIGGER_KIND, - trigger.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for {} trigger {}: {:?}", - T::TRIGGER_KIND, - trigger.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); - } - } - - let unlisted_captures = listener.fetch_unlistened_captures(&db).await; - - match unlisted_captures { - Ok(unlistened_captures) => { - for capture in unlistened_captures { - let has_lock = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - &capture.workspace_id, - &capture.path, - &capture.is_flow, - T::TRIGGER_KIND as TriggerKind - ) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, capture, killpill_rx).await } - }); - } else { - tracing::info!( - "{} capture {} already being listened to", - T::TRIGGER_KIND.to_string(), - capture.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture {} {}: {:?}", - T::TRIGGER_KIND, - capture.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!( - "Error fetching captures {} triggers: {:?}", - T::TRIGGER_KIND, - err - ); - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -struct Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - path: String, - is_flow: bool, - workspace_id: String, - username: String, - email: String, - #[serde(flatten)] - trigger_config: T, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let trigger_config_value = row.try_get("trigger_config")?; - let trigger_config: T = serde_json::from_value(trigger_config_value) - .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; - Ok(Capture { - path: row.try_get("path")?, - is_flow: row.try_get("is_flow")?, - workspace_id: row.try_get("workspace_id")?, - username: row.try_get("username")?, - email: row.try_get("email")?, - trigger_config, - }) - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ListeningTrigger { - pub path: String, - pub is_flow: bool, - pub workspace_id: String, - pub username: String, - pub email: String, - pub trigger_config: T, - pub script_path: String, - pub trigger_mode: bool, - pub error_handling: Option, - pub suspended_mode: bool, -} - -impl ListeningTrigger { - pub async fn authed(&self, db: &DB, username: &str) -> Result { - fetch_api_authed( - self.username.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("{}-{}", username, self.path)), - ) - .await - } -} - -#[allow(unused)] -pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { - let mut w = lock.write().await; - *w = value; -} +// Re-export for backward compat with concrete trigger modules that use +// `crate::triggers::listener::ListeningTrigger` +#[allow(unused_imports)] +pub use windmill_trigger::listener::ListeningTrigger; #[allow(unused)] fn listen_to( diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index ce8538de2d..94c5039775 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -1,16 +1,4 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use std::{collections::HashMap, fmt::Debug}; -use windmill_common::jobs::JobTriggerKind; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum HandlerAction { - Trigger { path: String, trigger_kind: JobTriggerKind }, - // Future variants can be added here (e.g., Script, Flow, etc.) -} - +// Concrete trigger submodules (feature-gated) #[cfg(all(feature = "smtp", feature = "private"))] pub mod email; #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] @@ -30,153 +18,33 @@ pub mod sqs; #[cfg(feature = "websocket")] pub mod websocket; -pub mod filter; -pub mod global_handler; +// Assembly modules that stay local (reference concrete trigger types) mod handler; mod listener; -pub mod trigger_helpers; +// Re-export everything from windmill-trigger base crate +pub use windmill_trigger::capture; +pub use windmill_trigger::filter; +pub use windmill_trigger::global_handler; +pub use windmill_trigger::trigger_helpers; +pub use windmill_trigger::types::*; + +// Re-export traits #[allow(unused)] -pub(crate) use handler::TriggerCrud; +pub(crate) use windmill_trigger::Listener; +#[allow(unused)] +pub(crate) use windmill_trigger::TriggerCrud; + +// Re-export windmill-trigger's handler module items needed by concrete triggers +pub use windmill_trigger::handler::{ + complete_trigger_routes, trigger_routes, TriggerPrimarySchedule, +}; + +// Re-export windmill-trigger's listener module items needed by concrete triggers +pub use windmill_trigger::listener::{ + listen_to_unlistened_events, listening, update_rw_lock, Capture, ListeningTrigger, +}; + +// Assembly functions (stay local because they reference concrete trigger types) pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; pub use listener::start_all_listeners; -#[allow(unused)] -pub(crate) use listener::Listener; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StandardTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - pub path_start: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct BaseTrigger { - pub workspace_id: String, - pub path: String, - pub script_path: String, - pub mode: TriggerMode, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: DateTime, - pub extra_perms: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct ServerState { - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct TriggerErrorHandling { - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - #[serde(flatten)] - pub base: BaseTrigger, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub server_state: Option, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let base = BaseTrigger::from_row(row)?; - - Ok(Trigger { - base, - config: T::from_row(row)?, - server_state: ServerState::from_row(row).ok(), - error_handling: TriggerErrorHandling::from_row(row)?, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BaseTriggerData { - pub path: String, - pub script_path: String, - pub is_flow: bool, - #[deprecated(note = "Use mode instead")] - enabled: Option, // Kept for backwards compatibility, use mode instead - mode: Option, -} - -impl BaseTriggerData { - pub fn mode(&self) -> &TriggerMode { - self.mode.as_ref().unwrap_or( - #[allow(deprecated)] - if self.enabled.unwrap_or(true) { - &TriggerMode::Enabled - } else { - &TriggerMode::Disabled - }, - ) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TriggerData { - #[serde(flatten)] - pub base: BaseTriggerData, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl StandardTriggerQuery { - pub fn offset(&self) -> i64 { - let page = self.page.unwrap_or(0); - let per_page = self.per_page.unwrap_or(100); - (page * per_page) as i64 - } - - pub fn limit(&self) -> i64 { - self.per_page.unwrap_or(100) as i64 - } -} - -impl Default for StandardTriggerQuery { - fn default() -> Self { - Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] -#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum TriggerMode { - Enabled, - Disabled, - Suspended, -} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs index cfc8e36900..0effa0e75a 100644 --- a/backend/windmill-api/src/triggers/mqtt/mod.rs +++ b/backend/windmill-api/src/triggers/mqtt/mod.rs @@ -1,346 +1 @@ -use base64::{engine, prelude::*}; -use itertools::Itertools; -use rumqttc::{ - v5::{ - mqttbytes::{ - v5::{ConnectProperties, Filter}, - QoS as V5QoS, - }, - AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, - }, - AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, - QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow, Type}; -use std::{collections::HashMap, time::Duration}; -use windmill_common::{ - error::{to_anyhow, Error}, - triggers::TriggerKind, - worker::to_raw_value, -}; - -use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; - -pub mod handler; -pub mod listener; - -#[derive(Clone, Copy)] -pub struct MqttTrigger; - -impl TriggerJobArgs for MqttTrigger { - type Payload = Vec; - const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; - - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { - let base64_payload = engine::general_purpose::STANDARD.encode(payload); - HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[serde(rename_all = "lowercase")] -pub enum QualityOfService { - Qos0, - Qos1, - Qos2, -} - -impl From for V3QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V3QoS::AtMostOnce, - QualityOfService::Qos1 => V3QoS::AtLeastOnce, - QualityOfService::Qos2 => V3QoS::ExactlyOnce, - } - } -} - -impl From for V5QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V5QoS::AtMostOnce, - QualityOfService::Qos1 => V5QoS::AtLeastOnce, - QualityOfService::Qos2 => V5QoS::ExactlyOnce, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[sqlx(type_name = "MQTT_CLIENT_VERSION")] -#[sqlx(rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum MqttClientVersion { - V3, - V5, -} - -#[derive(Debug, Deserialize)] -pub struct Tls { - enabled: bool, - ca_certificate: String, - pkcs12_client_certificate: Option, - pkcs12_certificate_password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Credentials { - username: Option, - password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct MqttResource { - broker: String, - port: u16, - credentials: Option, - tls: Option, -} - -#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] -pub struct SubscribeTopic { - qos: QualityOfService, - topic: String, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct MqttConfig { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MqttConfigRequest { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec, - pub v3_config: Option, - pub v5_config: Option, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestMqttConfig { - pub mqtt_resource_path: String, - pub client_version: Option, - pub v3_config: Option, - pub v5_config: Option, -} - -// Constants -pub const KEEP_ALIVE: u64 = 60; -pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; -pub const TIMEOUT_DURATION: u64 = 10; -pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -pub struct V3MqttHandler; -pub struct V5MqttHandler; - -pub enum MqttClientResult { - V3((V3MqttHandler, V3EventLoop)), - V5((V5MqttHandler, V5EventLoop)), -} - -#[derive(Debug, thiserror::Error)] -pub enum MqttError { - #[error("{0}")] - Common(#[from] Error), - #[error("{0}")] - V5RumqttClient(#[from] rumqttc::v5::ClientError), - #[error("{0}")] - V5ConnectionError(#[from] rumqttc::v5::ConnectionError), - #[error("{0}")] - V3RumqttClient(#[from] rumqttc::ClientError), - #[error("{0}")] - V3ConnectionError(#[from] rumqttc::ConnectionError), - #[error("{0}")] - Base64Decode(#[from] base64::DecodeError), -} - -pub struct MqttClientBuilder<'client> { - mqtt_resource: MqttResource, - client_id: &'client str, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, -} - -impl<'client> MqttClientBuilder<'client> { - pub fn new( - mqtt_resource: MqttResource, - client_id: Option<&'client str>, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, - ) -> Self { - Self { - mqtt_resource, - client_id: client_id.unwrap_or(""), - subscribe_topics, - v3_config, - v5_config, - mqtt_client_version, - } - } - - pub async fn build_client(&self) -> Result { - match self.mqtt_client_version { - Some(MqttClientVersion::V5) | None => self.build_v5_client().await, - Some(MqttClientVersion::V3) => self.build_v3_client().await, - } - } - - fn get_tls_configuration(&self) -> Result, Error> { - let transport = match self.mqtt_resource.tls { - Some(ref tls) if tls.enabled => { - let transport = match tls.ca_certificate.trim().is_empty() { - true => rumqttc::Transport::Tls(TlsConfiguration::Native), - false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { - ca: tls.ca_certificate.as_bytes().to_vec(), - client_auth: { - match tls.pkcs12_client_certificate.as_ref() { - Some(client_certificate) - if !client_certificate.trim().is_empty() => - { - let client_certificate = BASE64_STANDARD - .decode(client_certificate) - .map_err(to_anyhow)?; - let password = tls - .pkcs12_certificate_password - .clone() - .unwrap_or("".to_string()); - Some((client_certificate, password)) - } - _ => None, - } - }, - }), - }; - - Some(transport) - } - _ => None, - }; - - Ok(transport) - } - - async fn build_v5_client(&self) -> Result { - let mut mqtt_options = V5MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - - mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); - - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - - if let Some(v5_config) = self.v5_config { - mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); - mqtt_options.set_connect_properties(ConnectProperties { - session_expiry_interval: v5_config.session_expiry_interval, - receive_maximum: None, - max_packet_size: None, - topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), - request_response_info: None, - request_problem_info: None, - user_properties: vec![], - authentication_method: None, - authentication_data: None, - }); - } - - let (async_client, mut event_loop) = - V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) - } - - async fn build_v3_client(&self) -> Result { - let mut mqtt_options = V3MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - if let Some(v3_config) = self.v3_config { - mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); - } - - let (async_client, mut event_loop) = - V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) - } -} +pub use windmill_trigger_mqtt::*; diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs index 668df073c9..a07227cf53 100644 --- a/backend/windmill-api/src/triggers/nats/mod.rs +++ b/backend/windmill-api/src/triggers/nats/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct NatsTrigger; +pub use windmill_trigger_nats::*; diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs index ea267721a2..a13d931090 100644 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -1,553 +1 @@ -use std::collections::HashMap; - -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::trigger_helpers::TriggerJobArgs, -}; -use chrono::Utc; -use itertools::Itertools; -use native_tls::{Certificate, TlsConnector}; -use pg_escape::quote_identifier; -use rand::Rng; -use rust_postgres::{config::SslMode, Client, Config, NoTls}; -use rust_postgres_native_tls::MakeTlsConnector; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::value::RawValue; -use sqlx::FromRow; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - utils::empty_as_none, -}; - -mod bool; -mod converter; -pub mod handler; -mod hex; -pub mod listener; -mod mapper; -mod relation; -mod replication_message; - -#[derive(Clone, Copy)] -pub struct PostgresTrigger; - -impl TriggerJobArgs for PostgresTrigger { - type Payload = HashMap>; - const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; - fn v1_payload_fn(payload: &HashMap>) -> HashMap> { - payload.to_owned() - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct PostgresConfig { - pub postgres_resource_path: String, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub basic_mode: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PostgresConfigRequest { - postgres_resource_path: String, - #[serde(default)] - replication_slot_name: String, - #[serde(default)] - publication_name: String, - publication: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestPostgresConfig { - pub postgres_resource_path: String, -} - -fn check_if_valid_relation<'de, D>( - relations: D, -) -> std::result::Result>, D::Error> -where - D: Deserializer<'de>, -{ - let relations: Option> = Option::deserialize(relations)?; - let mut track_all_table_in_schema = false; - let mut track_specific_columns_in_table = false; - match relations { - Some(relations) => { - for relation in relations.iter() { - if relation.schema_name.is_empty() { - return Err(serde::de::Error::custom( - "Schema Name must not be empty".to_string(), - )); - } - - if !track_all_table_in_schema && relation.table_to_track.is_empty() { - track_all_table_in_schema = true; - continue; - } - - for table_to_track in relation.table_to_track.iter() { - if table_to_track.table_name.trim().is_empty() { - return Err(serde::de::Error::custom( - "Table name must not be empty".to_string(), - )); - } - - if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { - track_specific_columns_in_table = true; - } - } - - if track_all_table_in_schema && track_specific_columns_in_table { - return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); - } - } - - if !relations - .iter() - .map(|relation| relation.schema_name.as_str()) - .all_unique() - { - return Err(serde::de::Error::custom( - "You cannot choose a schema more than one time".to_string(), - )); - } - - Ok(Some(relations)) - } - None => Ok(None), - } -} - -fn check_if_valid_transaction_type<'de, D>( - transaction_type: D, -) -> std::result::Result, D::Error> -where - D: Deserializer<'de>, -{ - let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; - if transaction_type.len() > 3 { - return Err(serde::de::Error::custom( - "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" - .to_string(), - )); - } - transaction_type.sort_unstable(); - transaction_type.dedup(); - - for transaction in transaction_type.iter() { - match transaction.to_lowercase().as_ref() { - "insert" => {}, - "update" => {}, - "delete" => {}, - _ => { - return Err(serde::de::Error::custom( - "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" - .to_string(), - )) - } - } - } - - Ok(transaction_type) -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PublicationData { - #[serde(default, deserialize_with = "check_if_valid_relation")] - pub table_to_track: Option>, - #[serde(deserialize_with = "check_if_valid_transaction_type")] - pub transaction_to_track: Vec, -} - -impl PublicationData { - pub fn new( - table_to_track: Option>, - transaction_to_track: Vec, - ) -> PublicationData { - PublicationData { table_to_track, transaction_to_track } - } -} - -// Slot list struct -#[derive(FromRow, Debug, Serialize)] -pub struct SlotList { - pub slot_name: Option, - pub active: Option, -} - -// Slot struct -#[derive(Debug, Serialize, Deserialize)] -pub struct Slot { - pub name: String, -} - -// Template script struct -#[derive(Debug, Deserialize)] -pub struct TemplateScript { - pub postgres_resource_path: String, - #[serde(deserialize_with = "check_if_valid_relation")] - pub relations: Option>, - pub language: Language, -} - -// Language enum -#[derive(Deserialize, Debug)] -pub enum Language { - #[serde(rename = "typescript", alias = "Typescript")] - Typescript, -} - -// Test postgres struct -#[derive(Serialize, Deserialize)] -pub struct TestPostgres { - pub postgres_resource_path: String, -} - -// PostgreSQL publication replication struct -#[derive(Serialize, Deserialize)] -pub struct PostgresPublicationReplication { - pub publication_name: String, - pub replication_slot_name: String, -} - -impl PostgresPublicationReplication { - pub fn new( - publication_name: String, - replication_slot_name: String, - ) -> PostgresPublicationReplication { - PostgresPublicationReplication { publication_name, replication_slot_name } - } -} - -pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct TableToTrack { - pub table_name: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub where_clause: Option, - #[serde(default, deserialize_with = "empty_as_none")] - pub columns_name: Option>, -} - -impl TableToTrack { - pub fn new( - table_name: String, - where_clause: Option, - columns_name: Option>, - ) -> TableToTrack { - TableToTrack { table_name, where_clause, columns_name } - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct Relations { - pub schema_name: String, - pub table_to_track: Vec, -} - -impl Relations { - pub fn new(schema_name: String, table_to_track: Vec) -> Relations { - Relations { schema_name, table_to_track } - } - - pub fn add_new_table(&mut self, table_to_track: TableToTrack) { - self.table_to_track.push(table_to_track); - } -} - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = - Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new( - connector.build().map_err(to_anyhow)?, - ))) -} - -pub async fn get_raw_postgres_connection( - database: &Postgres, - logical_mode: bool, -) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - )) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode); - - if logical_mode { - config.replication_mode(rust_postgres::config::ReplicationMode::Logical); - } - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; - tokio::spawn(async move { - tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); - if let Err(e) = connection.await { - tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); - }; - tracing::info!("PostgreSQL trigger connection closed"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; - - Ok(client) -} - -pub async fn get_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - logical_mode: bool, -) -> Result { - let database = - try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) - .await?; - - Ok(get_raw_postgres_connection(&database, logical_mode).await?) -} - -pub async fn get_default_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, -) -> Result { - get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await -} - -pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { - tx.execute( - &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn check_if_valid_publication_for_postgres_version( - pg_connection: &Client, - table_to_track: Option<&[Relations]>, -) -> Result { - use crate::triggers::postgres::handler::get_postgres_version_internal; - - let postgres_version = get_postgres_version_internal(pg_connection).await?; - - let pg_14 = postgres_version.starts_with("14"); - if pg_14 { - let unsupported_publication = table_to_track - .and_then(|relations| { - relations.iter().find(|relation| { - let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { - table_to_track.where_clause.is_some() - || table_to_track.columns_name.is_some() - }); - - relation.table_to_track.is_empty() || invalid_relation.is_some() - }) - }) - .is_some(); - - if unsupported_publication { - return Err(Error::BadRequest( - "Your PostgreSQL database is running version 14, which does not support the following publication features: \ - - WHERE clause filtering, \ - - selective column tracking, and \ - - tracking all tables within a schema.\n\ - These features are only available in PostgreSQL 15 and above.".to_string(), - )); - } - } - Ok(pg_14) -} - -pub async fn create_pg_publication( - pg_connection: &Client, - publication_name: &str, - table_to_track: Option<&[Relations]>, - transaction_to_track: &[String], -) -> Result<()> { - let pg_14 = - check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; - let mut query = String::from("CREATE PUBLICATION "); - - query.push_str("e_identifier(publication_name)); - - match table_to_track { - Some(database_component) if !database_component.is_empty() => { - query.push_str(" FOR"); - let mut first = true; - for (i, schema) in database_component.iter().enumerate() { - if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - query.push_str("e_identifier(&schema.schema_name)); - } else { - if pg_14 && first { - query.push_str(" TABLE ONLY "); - first = false - } else if !pg_14 { - query.push_str(" TABLE ONLY "); - } - for (j, table) in schema.table_to_track.iter().enumerate() { - let table_name = quote_identifier(&table.table_name); - let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); - query.push_str(&full_name); - if let Some(columns) = table.columns_name.as_ref() { - query.push_str(" ("); - let columns = columns - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(")"); - } - - if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); - } - - if j + 1 != schema.table_to_track.len() { - query.push_str(", "); - } - } - } - if i < database_component.len() - 1 { - query.push_str(", "); - } - } - } - _ => { - query.push_str(" FOR ALL TABLES "); - } - }; - - if !transaction_to_track.is_empty() { - let transactions = || transaction_to_track.iter().join(", "); - query.push_str(" WITH (publish = '"); - query.push_str(&transactions()); - query.push_str("');"); - } - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { - let mut query = String::from("DROP PUBLICATION IF EXISTS "); - let quoted_publication_name = quote_identifier(publication_name); - query.push_str("ed_publication_name); - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub fn generate_random_string() -> String { - let timestamp = Utc::now().timestamp_millis().to_string(); - let mut rng = rand::rng(); - let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; - - let random_part = (0..10) - .map(|_| { - charset - .chars() - .nth(rng.random_range(0..charset.len())) - .unwrap() - }) - .collect::(); - - format!("{}_{}", timestamp, random_part) -} +pub use windmill_trigger_postgres::*; diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs index 8c7217725c..1c85eebbc3 100644 --- a/backend/windmill-api/src/triggers/sqs/mod.rs +++ b/backend/windmill-api/src/triggers/sqs/mod.rs @@ -1,15 +1 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct SqsTrigger; +pub use windmill_trigger_sqs::*; diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs index f20b6334f7..7833ee704b 100644 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -1,128 +1 @@ -use std::collections::HashMap; - -use crate::{ - db::ApiAuthed, - triggers::trigger_helpers::{ - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow}; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerMetadata, TriggerKind}, - worker::to_raw_value, - DB, -}; -use windmill_queue::PushArgsOwned; - -mod handler; -mod listener; - -#[derive(Copy, Clone)] -pub struct WebsocketTrigger; - -impl TriggerJobArgs for WebsocketTrigger { - type Payload = String; - const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("msg".to_string(), to_raw_value(&payload))]) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WebsocketConfig { - pub url: String, - #[serde(default)] - pub filters: Vec>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub initial_messages: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_runnable_args: Option>>, - #[serde(default)] - pub can_return_message: bool, - #[serde(default)] - pub can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebsocketConfigRequest { - url: String, - filters: Vec, - initial_messages: Option>, - url_runnable_args: Option, - can_return_message: bool, - can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestWebsocketConfig { - url: String, - url_runnable_args: Option, -} - -pub fn value_to_args_hashmap( - args: Option<&Box>, -) -> Result>> { - let args = if let Some(args) = args { - let args_map: Option> = serde_json::from_str(args.get()) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; - - args_map - .unwrap_or_else(HashMap::new) - .into_iter() - .map(|(k, v)| { - let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { - Error::BadRequest(format!("failed to convert to raw value: {}", e)) - })?; - Ok((k, raw_value)) - }) - .collect::>>>() - } else { - Ok(HashMap::new()) - }?; - Ok(args) -} - -pub async fn get_url_from_runnable_value( - path: &str, - is_flow: bool, - db: &DB, - authed: ApiAuthed, - args: Option<&Box>, - workspace_id: &str, -) -> Result { - tracing::info!( - "Running {} {} to get WebSocket URL", - if is_flow { "flow" } else { "script" }, - path - ); - - let args = value_to_args_hashmap(args)?; - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed, - workspace_id, - path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - Error::BadConfig(format!( - "{} {} did not return a string", - if is_flow { "Flow" } else { "Script" }, - path, - )) - }) -} +pub use windmill_trigger_websocket::*; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 2e28999779..b70a1b98f1 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -8,8 +8,7 @@ #![allow(non_snake_case)] -use quick_cache::sync::Cache; -use sqlx::{PgConnection, Postgres, Transaction}; +use sqlx::{Postgres, Transaction}; use std::sync::atomic::AtomicBool; use std::sync::Arc; @@ -24,14 +23,11 @@ use crate::utils::{ generate_instance_wide_unique_username, get_instance_username_or_create_pending, }; use crate::{ - auth::ExpiringAuthCache, db::DB, utils::require_super_admin, webhook_util::WebhookShared, - COOKIE_DOMAIN, IS_SECURE, + db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE, }; use argon2::{Argon2, PasswordHash, PasswordVerifier}; use axum::{ - async_trait, - extract::{Extension, FromRequestParts, Path, Query}, - http::request::Parts, + extract::{Extension, Path, Query}, response::{IntoResponse, Response}, routing::{delete, get, post}, Json, Router, @@ -47,11 +43,11 @@ use tracing::Instrument; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{fetch_authed_from_permissioned_as, TOKEN_PREFIX_LEN}; +use windmill_common::auth::TOKEN_PREFIX_LEN; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; +use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; -use windmill_common::users::{truncate_token, username_to_permissioned_as}; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::BASE_URL; @@ -132,135 +128,13 @@ pub fn make_unauthed_service() -> Router { .route("/is_smtp_configured", get(is_smtp_configured)) } -pub async fn maybe_refresh_folders( - path: &str, - w_id: &str, - authed: ApiAuthed, - db: &DB, -) -> ApiAuthed { - if authed.is_admin { - return authed; - } - let splitted = path.split('/').collect::>(); - if splitted.len() >= 2 - && splitted[0] == "f" - && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) - { - let name = &authed.username; - let groups = get_groups_for_user(w_id, name, &authed.email, db) - .await - .ok() - .unwrap_or_default(); +pub use windmill_api_auth::{ + create_token_internal, fetch_api_authed, get_scope_tags, maybe_refresh_folders, + require_is_writer, require_path_read_access_for_preview, NewToken, OptAuthed, +}; - let folders = get_folders_for_user(w_id, name, &groups, db) - .await - .ok() - .unwrap_or_default(); - ApiAuthed { folders, ..authed } - } else { - authed - } -} - -pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { - authed.scopes.as_ref()?.iter().find_map(|s| { - if s.starts_with("if_jobs:filter_tags:") { - Some( - s.trim_start_matches("if_jobs:filter_tags:") - .split(",") - .collect::>(), - ) - } else { - None - } - }) -} - -#[derive(Clone, Debug)] -pub struct OptAuthed(pub Option); - -#[async_trait] -impl FromRequestParts for OptAuthed -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - ApiAuthed::from_request_parts(parts, state) - .await - .map(|authed| Self(Some(authed))) - .or_else(|_| Ok(Self(None))) - } -} - -#[allow(unused)] -pub async fn fetch_api_authed( - username: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let permissioned_as = username_to_permissioned_as(username.as_str()); - fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await -} - -lazy_static::lazy_static! { - static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300); -} - -#[allow(unused)] -pub async fn fetch_api_authed_from_permissioned_as( - permissioned_as: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); - - let mut api_authed = match API_AUTHED_CACHE.get(&key) { - Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { - tracing::debug!("API authed cache hit for user {}", email); - expiring_authed.authed - } - _ => { - tracing::debug!("API authed cache miss for user {}", email); - let authed = - fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; - - let api_authed = ApiAuthed { - username: authed.username, - email, - is_admin: authed.is_admin, - is_operator: authed.is_operator, - groups: authed.groups, - folders: authed.folders, - scopes: authed.scopes, - username_override: None, - token_prefix: authed.token_prefix, - }; - - API_AUTHED_CACHE.insert( - key, - ExpiringAuthCache { - authed: api_authed.clone(), - expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - - api_authed - } - }; - - api_authed.username_override = username_override; - Ok(api_authed) -} +#[cfg(feature = "parquet")] +pub use windmill_api_auth::fetch_api_authed_from_permissioned_as; #[derive(FromRow, Serialize)] pub struct User { @@ -381,27 +255,7 @@ pub struct TruncatedToken { pub scopes: Option>, } -#[derive(Deserialize)] -pub struct NewToken { - pub label: Option, - pub expiration: Option>, - pub impersonate_email: Option, - pub scopes: Option>, - pub workspace_id: Option, -} - -#[cfg(feature = "native_trigger")] -impl NewToken { - pub fn new( - label: Option, - expiration: Option>, - impersonate_email: Option, - scopes: Option>, - workspace_id: Option, - ) -> NewToken { - NewToken { label, expiration, impersonate_email, scopes, workspace_id } - } -} +// NewToken is re-exported from windmill-api-auth above #[derive(Deserialize)] pub struct Login { @@ -914,182 +768,9 @@ pub async fn is_owner_of_path( } } -pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { - if authed.is_admin { - return Ok(()); - } - if !path.is_empty() { - let splitted = path.split("/").collect::>(); - if splitted[0] == "u" { - if splitted[1] == authed.username { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "only the owner {} is authorized to perform this operation", - splitted[1] - ))) - } - } else if splitted[0] == "f" { - crate::folders::require_is_owner(authed, splitted[1]) - } else { - Err(Error::BadRequest(format!( - "Not recognized path kind: {}", - path - ))) - } - } else { - Err(Error::BadRequest(format!( - "Cannot be owner of an empty path" - ))) - } -} +pub use windmill_api_auth::require_owner_of_path; -/// Checks that a user has at least read access to the path for preview jobs. -/// This prevents privilege escalation where a user could run preview code -/// under a path they don't have access to. -pub fn require_path_read_access_for_preview( - authed: &ApiAuthed, - path: &Option, -) -> Result<()> { - let Some(path) = path else { - return Ok(()); - }; - - if authed.is_admin { - return Ok(()); - } - - if path.is_empty() { - return Ok(()); - } - - let splitted: Vec<&str> = path.split('/').collect(); - if splitted.len() < 2 { - return Err(Error::BadRequest(format!( - "Invalid path format for preview job: {}", - path - ))); - } - - match splitted[0] { - "u" => { - if splitted[1] == authed.username { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "You can only run preview jobs in your own namespace (u/{}) or in folders you have read access to", - authed.username - ))) - } - } - "f" => { - let folder = splitted[1]; - if authed.folders.iter().any(|(f, _, _)| f == folder) { - Ok(()) - } else { - Err(Error::BadRequest(format!( - "You do not have read access to folder '{}'. Preview jobs require at least read access to the target folder.", - folder - ))) - } - } - "hub" => Ok(()), - _ => Err(Error::BadRequest(format!( - "Invalid path format for preview job: {}. Path must start with 'u/' or 'f/'", - path - ))), - } -} - -pub fn get_perm_in_extra_perms_for_authed( - v: serde_json::Value, - authed: &ApiAuthed, -) -> Option { - match v { - serde_json::Value::Object(obj) => { - let mut keys = vec![format!("u/{}", authed.username)]; - for g in authed.groups.iter() { - keys.push(format!("g/{}", g)); - } - let mut res = None; - for k in keys { - if let Some(v) = obj.get(&k) { - if let Some(v) = v.as_bool() { - if v { - return Some(true); - } - res = Some(v); - } - } - } - res - } - _ => None, - } -} - -pub async fn require_is_writer( - authed: &ApiAuthed, - path: &str, - w_id: &str, - db: DB, - query: &str, - kind: &str, -) -> Result<()> { - if authed.is_admin { - return Ok(()); - } - if !path.is_empty() { - if require_owner_of_path(authed, path).is_ok() { - return Ok(()); - } - if path.starts_with("f/") && path.split('/').count() >= 2 { - let folder = path.split('/').nth(1).unwrap(); - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(&db) - .await?; - if let Some(perms) = extra_perms { - let is_folder_writer = - get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); - if is_folder_writer { - return Ok(()); - } - } - } - let extra_perms = sqlx::query_scalar(query) - .bind(path) - .bind(w_id) - .fetch_optional(&db) - .await?; - if let Some(perms) = extra_perms { - let perm = get_perm_in_extra_perms_for_authed(perms, authed); - match perm { - Some(true) => Ok(()), - Some(false) => Err(Error::BadRequest(format!( - "User {} is not a writer of {kind} path {path}", - authed.username - ))), - None => Err(Error::BadRequest(format!( - "User {} has neither read or write permission on {kind} {path}", - authed.username - ))), - } - } else { - Err(Error::BadRequest(format!( - "{path} does not exist yet and user {} is not an owner of the parent folder", - authed.username - ))) - } - } else { - Err(Error::BadRequest(format!( - "Cannot be writer of an empty path" - ))) - } -} +// get_perm_in_extra_perms_for_authed and require_is_writer are re-exported from windmill-api-auth above async fn whois( Extension(db): Extension, Path((w_id, username)): Path<(String, String)>, @@ -1963,6 +1644,7 @@ async fn set_login_type( )) } +#[allow(unreachable_code, unused_variables)] async fn login( cookies: Cookies, Extension(db): Extension, @@ -2169,62 +1851,7 @@ pub async fn create_session_token<'c>( Ok(token) } -pub async fn create_token_internal( - tx: &mut PgConnection, - db: &DB, - authed: &ApiAuthed, - token_config: NewToken, -) -> Result { - let token = rd_string(32); - - let is_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - authed.email - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(false); - if *CLOUD_HOSTED { - let nb_tokens = - sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email) - .fetch_one(db) - .await?; - if nb_tokens.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - sqlx::query!( - "INSERT INTO token - (token, email, label, expiration, super_admin, scopes, workspace_id) - VALUES ($1, $2, $3, $4, $5, $6, $7)", - token, - authed.email, - token_config.label, - token_config.expiration, - is_super_admin, - token_config.scopes.as_ref().map(|x| x.as_slice()), - token_config.workspace_id, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - authed, - "users.token.create", - ActionKind::Create, - &"global", - Some(&token[0..10]), - None, - ) - .instrument(tracing::info_span!("token", email = &authed.email)) - .await?; - - Ok(token) -} +// create_token_internal is re-exported from windmill-api-auth above async fn create_token( Extension(db): Extension, diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index ea31585cba..f03f004411 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -13,12 +13,11 @@ use sqlx::{Postgres, Transaction}; #[cfg(feature = "enterprise")] use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ - auth::{is_devops_email, is_super_admin_email}, error::{self, Error}, DB, }; -use crate::{db::ApiAuthed, scopes::ScopeDefinition}; +pub use windmill_api_auth::{check_scopes, require_devops_role, require_super_admin}; #[cfg(feature = "enterprise")] use windmill_common::error::JsonResult; @@ -37,60 +36,6 @@ pub struct BulkDeleteRequest { pub paths: Vec, } -pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { - let is_admin = is_super_admin_email(db, email).await?; - - if !is_admin { - Err(Error::NotAuthorized( - "This endpoint requires the caller to be a super admin".to_owned(), - )) - } else { - Ok(()) - } -} - -pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> -where - F: FnOnce() -> String, -{ - if let Some(scopes) = authed.scopes.as_ref() { - let mut is_scoped_token = false; - let required_scope = ScopeDefinition::from_scope_string(&required())?; - for scope in scopes { - if !scope.starts_with("if_jobs:filter_tags:") { - if !is_scoped_token { - is_scoped_token = true; - } - - match ScopeDefinition::from_scope_string(scope) { - Ok(scope) if scope.includes(&required_scope) => return Ok(()), - _ => {} - } - } - } - - if is_scoped_token { - return Err(Error::NotAuthorized(format!( - "Required scope: {}", - required_scope.as_string() - ))); - } - } - Ok(()) -} - -pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { - let is_devops = is_devops_email(db, email).await?; - - if is_devops { - Ok(()) - } else { - Err(Error::NotAuthorized( - "This endpoint requires the caller to have the `devops` role".to_string(), - )) - } -} - lazy_static::lazy_static! { pub static ref INVALID_USERNAME_CHARS: Regex = Regex::new(r"[^A-Za-z0-9_]").unwrap(); } diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 6fd97247d4..10e03b2146 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -1,920 +1 @@ -/* - * Author: Ruben Fiszel - * Copyright: Windmill Labs, Inc 2022 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use crate::{ - db::{ApiAuthed, DB}, - secret_backend_ext::{ - delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, - store_secret_value, - }, - users::{maybe_refresh_folders, require_owner_of_path}, - utils::{check_scopes, BulkDeleteRequest}, - webhook_util::{WebhookMessage, WebhookShared}, -}; - -use axum::{ - extract::{Extension, Path, Query}, - routing::{delete, get, post}, - Json, Router, -}; -use futures::future::try_join_all; -use hyper::StatusCode; -use serde_json::Value; - -use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; -use windmill_audit::ActionKind; -use windmill_common::{ - db::{DbWithOptAuthed, UserDB}, - error::{Error, JsonResult, Result}, - scripts::ScriptHash, - utils::{not_found_if_none, paginate, Pagination, StripPath, WarnAfterExt}, - variables::{ - build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, - }, - worker::CLOUD_HOSTED, -}; - -use crate::var_resource_cache::{cache_variable, get_cached_variable}; -use lazy_static::lazy_static; -use serde::Deserialize; -use sqlx::{Acquire, Postgres, Transaction}; -use windmill_common::variables::encrypt; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; - -lazy_static! { - pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); -} - -pub fn workspaced_service() -> Router { - Router::new() - .route("/list", get(list_variables)) - .route("/list_contextual", get(list_contextual_variables)) - .route("/get/*path", get(get_variable)) - .route("/get_value/*path", get(get_value)) - .route("/exists/*path", get(exists_variable)) - .route("/update/*path", post(update_variable)) - .route("/delete/*path", delete(delete_variable)) - .route("/delete_bulk", delete(delete_variables_bulk)) - .route("/create", post(create_variable)) - .route("/encrypt", post(encrypt_value)) -} - -async fn list_contextual_variables( - Path(w_id): Path, - ApiAuthed { username, email, .. }: ApiAuthed, - Extension(db): Extension, -) -> JsonResult> { - Ok(Json( - get_reserved_variables( - &db.into(), - &w_id, - "q1A0qcPuO00yxioll7iph76N9CJDqn", - &email, - &username, - "017e0ad5-f499-73b6-5488-92a61c5196dd", - format!("u/{username}").as_str(), - Some("u/user/script_path".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some("u/user/encapsulating_flow_path".to_string()), - Some("u/user/triggering_flow_path".to_string()), - Some("c".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), - Some(chrono::offset::Utc::now()), - Some(ScriptHash(1234567890)), - None, - ) - .await - .to_vec(), - )) -} - -#[derive(Deserialize)] -struct ListVariableQuery { - path_start: Option, -} - -async fn list_variables( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lq): Query, - Query(pagination): Query, -) -> JsonResult> { - let (per_page, offset) = paginate(pagination); - - let mut tx = user_db.begin(&authed).await?; - - let rows = sqlx::query_as::<_, ListableVariable>( - "SELECT variable.workspace_id, variable.path, CASE WHEN is_secret IS TRUE THEN null ELSE variable.value::text END as value, - is_secret, variable.description, variable.extra_perms, account, is_oauth, (now() > account.expires_at) as is_expired, - account.refresh_error, - resource.path IS NOT NULL as is_linked, - account.refresh_token != '' as is_refreshed, - variable.expires_at - from variable - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1 - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1 - WHERE variable.workspace_id = $1 AND variable.path NOT LIKE 'u/' || $2 || '/secret_arg/%' - AND variable.path LIKE $3 || '%' - ORDER BY path - LIMIT $4 OFFSET $5 -", - ) - .bind(&w_id) - .bind(&authed.username) - .bind(&lq.path_start.unwrap_or_default()) - .bind(per_page as i32) - .bind(offset as i32) - .fetch_all(&mut *tx) - .await?; - - tx.commit().await?; - Ok(Json(rows)) -} - -#[derive(Deserialize)] -struct GetVariableQuery { - decrypt_secret: Option, - include_encrypted: Option, -} - -async fn get_variable( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Query(q): Query, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("variables:read:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - - let variable_o = sqlx::query_as::<_, ListableVariable>( - "SELECT variable.*, (now() > account.expires_at) as is_expired, account.refresh_error, - resource.path IS NOT NULL as is_linked, - account.refresh_token != '' as is_refreshed - from variable - LEFT JOIN account ON variable.account = account.id - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2 - WHERE variable.path = $1 AND variable.workspace_id = $2 - LIMIT 1", - ) - .bind(&path) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; - - let variable = if let Some(variable) = variable_o { - variable - } else { - explain_variable_perm_error(&path, &w_id, &db).await?; - unreachable!() - }; - - let decrypt_secret = q.decrypt_secret.unwrap_or(true); - - let r = if variable.is_secret { - if decrypt_secret { - audit_log( - &mut *tx, - &authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - } - - let value = variable.value.unwrap_or_else(|| "".to_string()); - ListableVariable { - value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() { - #[cfg(feature = "oauth2")] - { - Some( - crate::oauth2_oss::_refresh_token( - tx, - &variable.path, - &w_id, - variable.account.unwrap(), - &db, - ) - .await?, - ) - } - #[cfg(not(feature = "oauth2"))] - return Err(Error::internal_err("Require oauth2 feature".to_string())); - } else if !value.is_empty() && decrypt_secret { - let _ = tx.commit().await; - // Use secret backend for decryption (supports both DB and Vault) - Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) - } else if q.include_encrypted.unwrap_or(false) { - Some(value) - } else { - None - }, - ..variable - } - } else { - variable - }; - - Ok(Json(r)) -} - -#[derive(Deserialize)] -struct GetValueQuery { - allow_cache: Option, -} -async fn get_value( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(q): Query, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("variables:read:{}", path))?; - let userdb_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); - - return get_value_internal(&userdb_authed, &w_id, &path, q.allow_cache.unwrap_or(false)) - .warn_after_seconds(10) - .await - .map(Json); -} - -async fn explain_variable_perm_error( - path: &str, - w_id: &str, - db: &sqlx::Pool, -) -> windmill_common::error::Result<()> { - let extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Variable {} not found", path)))?; - if path.starts_with("f/") { - let folder = path.split("/").nth(1).ok_or_else(|| { - Error::BadRequest(format!( - "path {} should have at least 2 components separated by /", - path - )) - })?; - let folder_extra_perms = sqlx::query_scalar!( - "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", - folder, - w_id - ) - .fetch_optional(db) - .await?; - return Err(Error::NotAuthorized(format!( - "Variable exists but you don't have access to it:\nvariable perms: {}\nfolder perms: {}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() - ))); - } else { - return Err(Error::NotAuthorized(format!( - "Variable exists but you don't have access to it:\nvariable perms: {}", - serde_json::to_string_pretty(&extra_perms).unwrap_or_default() - ))); - } -} - -async fn exists_variable( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", - path, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "Variable {} already exists", - path - ))); - } - return Ok(()); -} - -async fn create_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Query(AlreadyEncrypted { already_encrypted }): Query, - Json(variable): Json, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || format!("variables:write:{}", variable.path))?; - if *CLOUD_HOSTED { - let nb_variables = sqlx::query_scalar!( - "SELECT COUNT(*) FROM variable WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - if nb_variables.unwrap_or(0) >= 10000 { - return Err(Error::BadRequest( - "You have reached the maximum number of variables (10000) on cloud. Contact support@windmill.dev to increase the limit" - .to_string(), - )); - } - } - let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await; - - check_path_conflict(&db, &w_id, &variable.path).await?; - let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { - // Use secret backend for encryption (supports both DB and Vault) - store_secret_value(&db, &w_id, &variable.path, &variable.value).await? - } else { - variable.value - }; - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "INSERT INTO variable - (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - &w_id, - variable.path, - value, - variable.is_secret, - variable.description, - variable.account, - variable.is_oauth.unwrap_or(false), - variable.expires_at - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "variables.create", - ActionKind::Create, - &w_id, - Some(&variable.path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: variable.path.clone(), parent_path: None }, - Some(format!("Variable '{}' created", variable.path.clone())), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::CreateVariable { workspace: w_id, path: variable.path.clone() }, - ); - - Ok(( - StatusCode::CREATED, - format!("variable {} created", variable.path), - )) -} - -async fn encrypt_value( - Extension(db): Extension, - Path(w_id): Path, - Json(variable): Json, -) -> Result { - let mc = build_crypt(&db, &w_id).await?; - let value = encrypt(&mc, &variable); - - Ok(value) -} - -async fn delete_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - - check_scopes(&authed, || format!("variables:write:{}", path))?; - - // Check if variable is a secret before deleting (for Vault cleanup) - let is_secret = sqlx::query_scalar!( - "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false); - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - sqlx::query!( - "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .execute(&mut *tx) - .await?; - audit_log( - &mut *tx, - &authed, - "variables.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - tx.commit().await?; - - // If variable was a secret, also delete from Vault backend (if configured) - if is_secret { - delete_secret_from_backend(&db, &w_id, path).await?; - } - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: path.to_string(), parent_path: Some(path.to_string()) }, - Some(format!("Variable '{}' deleted", path)), - true, - None, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() }, - ); - - Ok(format!("variable {} deleted", path)) -} - -async fn delete_variables_bulk( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path(w_id): Path, - Json(request): Json, -) -> JsonResult> { - for path in &request.paths { - check_scopes(&authed, || format!("variables:write:{}", path))?; - } - - // Query which paths are secrets before deletion (for Vault cleanup) - let secret_paths: Vec = sqlx::query_scalar!( - "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", - &request.paths, - &w_id - ) - .fetch_all(&db) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let deleted_paths = sqlx::query_scalar!( - "DELETE FROM variable WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", - &request.paths, - w_id - ) - .fetch_all(&mut *tx) - .await?; - sqlx::query!( - "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2", - &deleted_paths, - w_id - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "variables.delete_bulk", - ActionKind::Delete, - &w_id, - Some(&deleted_paths.join(", ")), - None, - ) - .await?; - - tx.commit().await?; - - // Delete secrets from Vault backend (if configured) - for path in &secret_paths { - if deleted_paths.contains(path) { - delete_secret_from_backend(&db, &w_id, path).await?; - } - } - - try_join_all(deleted_paths.iter().map(|path| { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { - path: path.to_string(), - parent_path: Some(path.to_string()), - }, - Some(format!("Variable '{}' deleted", path)), - true, - None, - ) - })) - .await?; - - for path in &deleted_paths { - webhook.send_message( - w_id.clone(), - WebhookMessage::DeleteVariable { workspace: w_id.clone(), path: path.to_owned() }, - ); - } - - Ok(Json(deleted_paths)) -} - -#[derive(Deserialize)] -struct EditVariable { - path: Option, - value: Option, - is_secret: Option, - description: Option, - account: Option, -} - -#[derive(Deserialize)] -struct AlreadyEncrypted { - already_encrypted: Option, -} - -async fn update_variable( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Extension(webhook): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(AlreadyEncrypted { already_encrypted }): Query, - Json(ns): Json, -) -> Result { - use sql_builder::prelude::*; - - let path = path.to_path(); - check_scopes(&authed, || format!("variables:write:{}", path))?; - let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; - - let mut sqlb = SqlBuilder::update_table("variable"); - sqlb.and_where_eq("path", "?".bind(&path)); - sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); - - if let Some(npath) = &ns.path { - sqlb.set_str("path", npath); - } - let ns_value_is_none = ns.value.is_none(); - // Determine the target path for storing secrets (use new path if provided) - let target_path = ns.path.as_deref().unwrap_or(path); - if let Some(nvalue) = ns.value.clone() { - let is_secret = if ns.is_secret.is_some() { - ns.is_secret.unwrap() - } else { - sqlx::query_scalar!( - "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false) - }; - - let value = if is_secret && !already_encrypted.unwrap_or(false) { - // Use secret backend for encryption (supports both DB and Vault) - // Store at target_path (new path if renaming, otherwise current path) - store_secret_value(&db, &w_id, target_path, &nvalue).await? - } else { - nvalue - }; - sqlb.set_str("value", &value); - } - - if let Some(desc) = ns.description { - sqlb.set_str("description", &desc); - } - - if let Some(account_id) = ns.account { - sqlb.set_str("account", account_id); - } - - if let Some(nbool) = ns.is_secret { - let old_secret = sqlx::query_scalar!( - "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .unwrap_or(false); - if old_secret != nbool && ns_value_is_none { - return Err(Error::BadRequest( - "cannot change is_secret without updating value too".to_string(), - )); - } - sqlb.set_str("is_secret", nbool); - } - sqlb.returning("path"); - - // Get old account_id if we're updating the account field - let old_account_id = if ns.account.is_some() { - sqlx::query_scalar!( - "SELECT account FROM variable WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .flatten() - } else { - None - }; - - let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; - - if let Some(npath) = ns.path.clone() { - if npath != path { - check_path_conflict(&db, &w_id, &npath).await?; - require_owner_of_path(&authed, path)?; - - // Handle Vault secret rename if the variable is a secret stored in Vault - let current_var = sqlx::query!( - "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await?; - - if let Some(var) = current_var { - if var.is_secret && is_vault_stored_value(&var.value) { - if ns.value.is_some() { - // New value was provided and already stored at new path - // Just delete the old secret from Vault - delete_secret_from_backend(&db, &w_id, path).await?; - } else { - // No new value - rename the secret in Vault - if let Some(new_value) = - rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? - { - // Update the variable's value to point to the new Vault path - sqlb.set_str("value", &new_value); - } - } - } - } - - let mut v = sqlx::query_scalar!( - "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", - path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - if let Some(old_v) = v { - v = Some(replace_path( - old_v, - &format!("$var:{path}"), - &format!("$var:{npath}"), - )) - } - - sqlx::query!( - "UPDATE resource SET path = $1, value = $2, edited_at = now() WHERE path = $3 AND workspace_id = $4", - npath, - v, - path, - w_id - ) - .execute(&mut *tx) - .await?; - } - } - - let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; - - let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; - - let npath = not_found_if_none(npath_o, "Variable", path)?; - - audit_log( - &mut *tx, - &authed, - "variables.update", - ActionKind::Update, - &w_id, - Some(path), - None, - ) - .await?; - - // Clean up old account if it's no longer referenced and different from new account - if let Some(old_acc_id) = old_account_id { - if ns.account.is_some() && ns.account != Some(old_acc_id) { - // Check if old account is still referenced by other variables or resources - let account_still_used = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM variable WHERE account = $1 AND workspace_id = $2)", - old_acc_id, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(true); - - if !account_still_used { - // Delete the orphaned account - sqlx::query!( - "DELETE FROM account WHERE id = $1 AND workspace_id = $2", - old_acc_id, - &w_id - ) - .execute(&mut *tx) - .await?; - } - } - } - - tx.commit().await?; - - // Detect if this was a rename operation - let old_path_if_renamed = if npath != path { Some(path) } else { None }; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::Variable { path: npath.clone(), parent_path: Some(path.to_string()) }, - None, - true, - old_path_if_renamed, - ) - .await?; - - webhook.send_message( - w_id.clone(), - WebhookMessage::UpdateVariable { - workspace: w_id, - old_path: path.to_owned(), - new_path: npath.clone(), - }, - ); - - Ok(format!("variable {} updated (npath: {:?})", path, npath)) -} - -fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { - match v { - Value::Object(v) => Value::Object( - v.into_iter() - .map(|(k, v)| (k, replace_path(v, path, npath))) - .collect(), - ), - Value::Array(arr) => Value::Array( - arr.into_iter() - .map(|v| replace_path(v, path, npath)) - .collect(), - ), - Value::String(s) if s == path => Value::String(npath.to_owned()), - _ => v, - } -} - -pub async fn get_value_internal<'a>( - db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, - w_id: &str, - path: &str, - allow_cache: bool, -) -> Result { - if allow_cache { - if let Some(cached_variable) = get_cached_variable(&w_id, &path) { - return Ok(cached_variable); - } - } - - let mut tx = db_with_opt_authed.begin().await?; - let variable_o = sqlx::query!( - "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable - LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id - ) - .fetch_optional(&mut *tx) - .warn_after_seconds(5) - .await?; - drop(tx); - - let variable = if let Some(variable) = variable_o { - variable - } else { - explain_variable_perm_error(path, w_id, &db_with_opt_authed.db()).await?; - unreachable!() - }; - - let r = if variable.is_secret { - // let audit_author = - let mut tx = db_with_opt_authed.db().begin().await?; - audit_log( - &mut *tx, - db_with_opt_authed, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - tx.commit().await?; - - let value = variable.value; - if variable.is_expired.unwrap_or(false) && variable.account.is_some() { - #[cfg(feature = "oauth2")] - { - let db = db_with_opt_authed.db(); - let tx = db.begin().await?; - crate::oauth2_oss::_refresh_token( - tx, - &variable.path, - &w_id, - variable.account.unwrap(), - db, - ) - .await? - } - #[cfg(not(feature = "oauth2"))] - return Err(Error::internal_err("Require oauth2 feature".to_string())); - } else if !value.is_empty() { - // Use secret backend for decryption (supports both DB and Vault) - get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? - } else { - "".to_string() - } - } else { - variable.value - }; - - // Cache the result when explicitly allowed and caching appropriate - if allow_cache { - cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); - } - - Ok(r) -} +pub use windmill_store::variables::*; diff --git a/backend/windmill-api/src/webhook_util.rs b/backend/windmill-api/src/webhook_util.rs index 362893a7a1..ec9916a396 100644 --- a/backend/windmill-api/src/webhook_util.rs +++ b/backend/windmill-api/src/webhook_util.rs @@ -73,12 +73,14 @@ impl WebhookShared { pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { let (tx, mut rx) = mpsc::unbounded_channel::(); let _process = tokio::spawn(async move { - let client = configure_client(reqwest::Client::builder() - .connect_timeout(Duration::from_secs(5)) - // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5))) - .build() - .unwrap(); + let client = configure_client( + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + // TODO: investigate pool timeouts and such if TCP load is high + .timeout(Duration::from_secs(5)), + ) + .build() + .unwrap(); loop { select! { diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 667131bb79..dc2480ce15 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -148,11 +148,7 @@ async fn exists_workers_with_tags( Query(tags_query): Query, ) -> JsonResult> { // Create a list of requested tags - let mut tags: Vec = tags_query - .tags - .split(',') - .map(|s| s.to_string()) - .collect(); + let mut tags: Vec = tags_query.tags.split(',').map(|s| s.to_string()).collect(); // When TAGS_ARE_SENSITIVE is enabled, filter tags based on workspace visibility if *TAGS_ARE_SENSITIVE { diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 2631e22624..2206efd3ed 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -651,7 +651,7 @@ pub(crate) async fn tarball_workspace( if include_triggers.unwrap_or(false) { #[cfg(feature = "http_trigger")] { - use crate::triggers::http::handler::HttpTrigger; + use crate::triggers::http::HttpTrigger; let handler = HttpTrigger; let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; @@ -810,7 +810,8 @@ pub(crate) async fn tarball_workspace( for service_name in ServiceName::iter() { let native_triggers = - list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None).await?; + list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None) + .await?; for trigger in native_triggers { let trigger_str = &to_string_without_metadata( @@ -927,7 +928,7 @@ pub(crate) async fn tarball_workspace( if include_settings.unwrap_or(false) { let row = sqlx::query_as::<_, SettingsRow>( - r#"SELECT + r#"SELECT auto_invite, webhook, deploy_to, @@ -945,10 +946,10 @@ pub(crate) async fn tarball_workspace( FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, - ) - .bind(&w_id) - .fetch_one(&mut *tx) - .await?; + ) + .bind(&w_id) + .fetch_one(&mut *tx) + .await?; // Use v2 format only if explicitly requested, otherwise use v1 (legacy) for backward compatibility let settings_str = if settings_version.as_deref() == Some("v2") { @@ -977,11 +978,18 @@ pub(crate) async fn tarball_workspace( let (auto_invite_enabled, auto_invite_as, auto_invite_mode) = if let Some(ref ai) = row.auto_invite { let enabled = ai.get("enabled").and_then(|v| v.as_bool()).unwrap_or(false); - let operator = ai.get("operator").and_then(|v| v.as_bool()).unwrap_or(false); + let operator = ai + .get("operator") + .and_then(|v| v.as_bool()) + .unwrap_or(false); let mode = ai.get("mode").and_then(|v| v.as_str()).unwrap_or("invite"); ( enabled, - if operator { "operator".to_string() } else { "developer".to_string() }, + if operator { + "operator".to_string() + } else { + "developer".to_string() + }, mode.to_string(), ) } else { @@ -992,7 +1000,10 @@ pub(crate) async fn tarball_workspace( if let Some(ref eh) = row.error_handler { let path = eh.get("path").and_then(|v| v.as_str()).map(String::from); let extra_args = eh.get("extra_args").cloned(); - let muted_on_cancel = eh.get("muted_on_cancel").and_then(|v| v.as_bool()).unwrap_or(false); + let muted_on_cancel = eh + .get("muted_on_cancel") + .and_then(|v| v.as_bool()) + .unwrap_or(false); (path, extra_args, muted_on_cancel) } else { (None, None, false) diff --git a/backend/windmill-common/src/agent_workers.rs b/backend/windmill-common/src/agent_workers.rs index b501e78fd9..30c84f5807 100644 --- a/backend/windmill-common/src/agent_workers.rs +++ b/backend/windmill-common/src/agent_workers.rs @@ -22,8 +22,8 @@ pub struct AgentConfig { impl AgentConfig { pub fn from_env() -> Result { - let agent_token = std::env::var("AGENT_TOKEN") - .map_err(|_| AgentConfigError::MissingAgentToken)?; + let agent_token = + std::env::var("AGENT_TOKEN").map_err(|_| AgentConfigError::MissingAgentToken)?; let base_internal_url = std::env::var("BASE_INTERNAL_URL") .map_err(|_| AgentConfigError::MissingBaseInternalUrl)?; Ok(Self { agent_token, base_internal_url }) @@ -44,10 +44,16 @@ impl std::fmt::Display for AgentConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { AgentConfigError::MissingAgentToken => { - write!(f, "AGENT_TOKEN environment variable is not set but required for agent mode") + write!( + f, + "AGENT_TOKEN environment variable is not set but required for agent mode" + ) } AgentConfigError::MissingBaseInternalUrl => { - write!(f, "BASE_INTERNAL_URL environment variable is not set but required for agent mode") + write!( + f, + "BASE_INTERNAL_URL environment variable is not set but required for agent mode" + ) } } } diff --git a/backend/windmill-common/src/ai_bedrock.rs b/backend/windmill-common/src/ai_bedrock.rs index 5a1ba8837c..ea817a1e41 100644 --- a/backend/windmill-common/src/ai_bedrock.rs +++ b/backend/windmill-common/src/ai_bedrock.rs @@ -20,8 +20,7 @@ use serde::{Deserialize, Serialize}; use crate::error::Error; use crate::ai_types::{ - ContentPart, OpenAIContent, OpenAIFunction, OpenAIMessage, - OpenAIToolCall, ToolDef, + ContentPart, OpenAIContent, OpenAIFunction, OpenAIMessage, OpenAIToolCall, ToolDef, }; // ============================================================================ @@ -432,7 +431,10 @@ fn convert_message(msg: &OpenAIMessage) -> Result { "user" => ConversationRole::User, "assistant" => ConversationRole::Assistant, _ => { - return Err(Error::internal_err(format!("Unsupported role: {}", msg.role))); + return Err(Error::internal_err(format!( + "Unsupported role: {}", + msg.role + ))); } }; diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 2e657fd0ac..323e10a128 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -8,7 +8,9 @@ use serde::{Deserialize, Deserializer, Serialize}; /// Deserializes an Option where empty strings become None. /// Use with `#[serde(default, deserialize_with = "empty_string_as_none")]` -pub fn empty_string_as_none<'de, D>(deserializer: D) -> std::result::Result, D::Error> +pub fn empty_string_as_none<'de, D>( + deserializer: D, +) -> std::result::Result, D::Error> where D: Deserializer<'de>, { @@ -47,11 +49,7 @@ pub enum AIProvider { impl AIProvider { /// Get the base URL for the AI provider - pub async fn get_base_url( - &self, - resource_base_url: Option, - db: &DB, - ) -> Result { + pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { if let Some(base_url) = resource_base_url { return Ok(base_url); } diff --git a/backend/windmill-common/src/flow_conversations.rs b/backend/windmill-common/src/flow_conversations.rs index 472c9c264c..4ba43e86a6 100644 --- a/backend/windmill-common/src/flow_conversations.rs +++ b/backend/windmill-common/src/flow_conversations.rs @@ -1,5 +1,6 @@ +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx; +use sqlx::{self, FromRow}; use uuid::Uuid; use crate::error::Result; @@ -14,6 +15,66 @@ pub enum MessageType { Tool, } +#[derive(Serialize, FromRow, Debug)] +pub struct FlowConversation { + pub id: Uuid, + pub workspace_id: String, + pub flow_path: String, + pub title: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +pub async fn get_or_create_conversation_with_id( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + w_id: &str, + flow_path: &str, + username: &str, + title: &str, + conversation_id: Uuid, +) -> Result { + // Check if conversation already exists + let existing_conversation = sqlx::query_as!( + FlowConversation, + "SELECT id, workspace_id, flow_path, title, created_at, updated_at, created_by + FROM flow_conversation + WHERE id = $1 AND workspace_id = $2", + conversation_id, + w_id + ) + .fetch_optional(&mut **tx) + .await?; + + if let Some(existing) = existing_conversation { + return Ok(existing); + } + + // Truncate title to 25 char characters max + let title = if title.len() > 25 { + format!("{}...", &title[..25]) + } else { + title.to_string() + }; + + // Create new conversation with provided ID + let conversation = sqlx::query_as!( + FlowConversation, + "INSERT INTO flow_conversation (id, workspace_id, flow_path, created_by, title) + VALUES ($1, $2, $3, $4, $5) + RETURNING id, workspace_id, flow_path, title, created_at, updated_at, created_by", + conversation_id, + w_id, + flow_path, + username, + title + ) + .fetch_one(&mut **tx) + .await?; + + Ok(conversation) +} + /// Add a message to a conversation using an existing transaction /// If the conversation doesn't exist, logs a warning and returns Ok (no error thrown) /// This allows memory_id to be used for agent memory without requiring a conversation diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 3fd4de438a..bde4892ad0 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -27,15 +27,14 @@ use scripts::ScriptLang; use sqlx::{Acquire, Postgres}; pub mod agent_workers; -pub mod ai_providers; -pub mod ai_types; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_providers; +pub mod ai_types; pub mod apps; -pub mod audit; pub mod assets; +pub mod audit; pub mod auth; -pub mod runtime_assets; #[cfg(feature = "benchmark")] pub mod bench; pub mod cache; @@ -59,10 +58,11 @@ pub mod indexer; pub mod job_metrics; #[cfg(all(feature = "parquet", feature = "private"))] pub mod job_s3_helpers_ee; -pub mod min_version; -pub mod notify_events; #[cfg(feature = "parquet")] pub mod job_s3_helpers_oss; +pub mod min_version; +pub mod notify_events; +pub mod runtime_assets; pub mod workspace_dependencies; #[cfg(feature = "private")] @@ -84,9 +84,9 @@ pub mod result_stream; pub mod runnable_settings; pub mod s3_helpers; pub mod schedule; -pub mod secret_backend; pub mod schema; pub mod scripts; +pub mod secret_backend; pub mod server; #[cfg(feature = "private")] pub mod stats_ee; @@ -100,6 +100,7 @@ pub mod triggers; pub mod users; pub mod utils; pub mod variables; +pub mod webhook; pub mod worker; pub mod worker_group_job_stats; pub mod workspaces; @@ -360,16 +361,22 @@ pub struct PgDatabase { // Wrapper enum to hold either Tls or NoTls connection pub enum TokioPgConnection { - Tls(tokio_postgres::Connection>), + Tls( + tokio_postgres::Connection< + tokio_postgres::Socket, + postgres_native_tls::TlsStream, + >, + ), NoTls(tokio_postgres::Connection), } - -impl Future for TokioPgConnection -{ +impl Future for TokioPgConnection { type Output = Result<(), tokio_postgres::Error>; - fn poll(self: std::pin::Pin<&mut Self>, cx: &mut core::task::Context<'_>) -> core::task::Poll { + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll { // SAFETY: We're simply projecting the Pin from the outer enum to the inner connection field. // The inner connection is never moved out, so this is safe. unsafe { @@ -381,7 +388,6 @@ impl Future for TokioPgConnection } } - impl PgDatabase { pub fn to_uri(&self) -> String { let sslmode = match self.sslmode.as_deref() { @@ -405,32 +411,46 @@ impl PgDatabase { format!( "dbname={dbname} {user} host={host} {password} {port} {sslmode}", dbname = self.dbname, - user = self.user.as_ref().map(|u| format!("user={}", urlencoding::encode(u))).unwrap_or_default(), + user = self + .user + .as_ref() + .map(|u| format!("user={}", urlencoding::encode(u))) + .unwrap_or_default(), host = self.host, - password = self.password - .as_ref().map(|p| format!("password={}", urlencoding::encode(p))) + password = self + .password + .as_ref() + .map(|p| format!("password={}", urlencoding::encode(p))) .unwrap_or_default(), port = self.port.map(|p| format!("port={}", p)).unwrap_or_default(), - sslmode = self.sslmode - .as_ref().map(|s| format!("sslmode={}", s.clone())) + sslmode = self + .sslmode + .as_ref() + .map(|s| format!("sslmode={}", s.clone())) .unwrap_or_default(), ) } - pub async fn connect(&self) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { - use tokio_postgres::tls::{ NoTls }; - use postgres_native_tls::MakeTlsConnector; + pub async fn connect( + &self, + ) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { use native_tls::{Certificate, TlsConnector}; - let ssl_mode_is_require = matches!(self.sslmode.as_deref(), Some("require") | Some("verify-ca") | Some("verify-full")); - + use postgres_native_tls::MakeTlsConnector; + use tokio_postgres::tls::NoTls; + let ssl_mode_is_require = matches!( + self.sslmode.as_deref(), + Some("require") | Some("verify-ca") | Some("verify-full") + ); + if ssl_mode_is_require { tracing::info!("Creating new connection"); let mut connector = TlsConnector::builder(); if let Some(root_certificate_pem) = &self.root_certificate_pem { if !root_certificate_pem.is_empty() { connector.add_root_certificate( - Certificate::from_pem(root_certificate_pem.as_bytes()) - .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, + Certificate::from_pem(root_certificate_pem.as_bytes()).map_err(|e| { + error::Error::BadConfig(format!("Invalid Certs: {e:#}")) + })?, ); } else { connector.danger_accept_invalid_certs(true); @@ -441,7 +461,7 @@ impl PgDatabase { .danger_accept_invalid_certs(true) .danger_accept_invalid_hostnames(true); } - + let (client, connection) = tokio::time::timeout( std::time::Duration::from_secs(20), tokio_postgres::connect( @@ -452,7 +472,7 @@ impl PgDatabase { .await .map_err(to_anyhow)? .map_err(to_anyhow)?; - + Ok((client, TokioPgConnection::Tls(connection))) } else { tracing::info!("Creating new connection"); @@ -463,17 +483,19 @@ impl PgDatabase { .await .map_err(to_anyhow)? .map_err(to_anyhow)?; - + Ok((client, TokioPgConnection::NoTls(connection))) } } pub fn parse_uri(url: &str) -> Result { - let parsed_url = - url::Url::parse(url).map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?; + let parsed_url = url::Url::parse(url) + .map_err(|_| Error::BadConfig("Invalid PostgreSQL URL".to_string()))?; let username = parsed_url.username().to_string(); - let username = urlencoding::decode(&username).map_err(to_anyhow)?.to_string(); + let username = urlencoding::decode(&username) + .map_err(to_anyhow)? + .to_string(); let password = parsed_url.password().map(|p| p.to_string()); let password = match password { Some(p) => Some(urlencoding::decode(&p).map_err(to_anyhow)?.to_string()), @@ -541,10 +563,8 @@ impl DatabaseUrl { let guard = rds_url.read().await; Ok(guard.connect_options()) } - DatabaseUrl::Static(url) => { - sqlx::postgres::PgConnectOptions::from_str(url) - .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))) - } + DatabaseUrl::Static(url) => sqlx::postgres::PgConnectOptions::from_str(url) + .map_err(|e| Error::InternalErr(format!("Failed to parse database URL: {}", e))), } } @@ -781,7 +801,9 @@ pub async fn connect( } }) .connect_with( - database_url.connect_options().await? + database_url + .connect_options() + .await? .statement_cache_capacity(400), ) .await @@ -793,7 +815,11 @@ type Tag = String; pub use db::DB; use crate::{ - auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, db::{AuthedRef, UserDbWithAuthed}, error::to_anyhow, runnable_settings::RunnableSettings, scripts::{ScriptHash, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline} + auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, + db::{AuthedRef, UserDbWithAuthed}, + error::to_anyhow, + runnable_settings::RunnableSettings, + scripts::{ScriptHash, ScriptRunnableSettingsHandle, ScriptRunnableSettingsInline}, }; #[derive(Clone)] @@ -822,19 +848,20 @@ pub struct ScriptHashInfo { } impl ScriptHashInfo { - pub async fn prefetch_cached<'a>(self, db: &DB) -> error::Result> { + pub async fn prefetch_cached<'a>( + self, + db: &DB, + ) -> error::Result> { + let (debouncing_settings, concurrency_settings) = + RunnableSettings::from_runnable_settings_handle( + self.runnable_settings.runnable_settings_handle, + db, + ) + .await? + .prefetch_cached(db) + .await?; - let (debouncing_settings, concurrency_settings) = - RunnableSettings::from_runnable_settings_handle( - self.runnable_settings.runnable_settings_handle, - db, - ) - .await? - .prefetch_cached(db) - .await?; - - Ok( - ScriptHashInfo { + Ok(ScriptHashInfo { path: self.path, hash: self.hash, tag: self.tag, @@ -848,17 +875,17 @@ impl ScriptHashInfo { has_preprocessor: self.has_preprocessor, on_behalf_of_email: self.on_behalf_of_email, created_by: self.created_by, - runnable_settings: ScriptRunnableSettingsInline{ - concurrency_settings: concurrency_settings - .maybe_fallback( + runnable_settings: ScriptRunnableSettingsInline { + concurrency_settings: concurrency_settings.maybe_fallback( self.runnable_settings.concurrency_key, self.runnable_settings.concurrent_limit, self.runnable_settings.concurrency_time_window_s, ), - debouncing_settings: debouncing_settings - .maybe_fallback(self.runnable_settings.debounce_key, self.runnable_settings.debounce_delay_s), + debouncing_settings: debouncing_settings.maybe_fallback( + self.runnable_settings.debounce_key, + self.runnable_settings.debounce_delay_s, + ), }, - }) } } @@ -1329,4 +1356,3 @@ impl KillpillSender { // self.already_sent.load(Ordering::SeqCst) // } } - diff --git a/backend/windmill-common/src/notify_events.rs b/backend/windmill-common/src/notify_events.rs index e00b6d7d49..84e3c8a4ec 100644 --- a/backend/windmill-common/src/notify_events.rs +++ b/backend/windmill-common/src/notify_events.rs @@ -51,7 +51,10 @@ pub async fn get_latest_event_id(db: &Pool) -> Result { /// Delete events older than the specified number of minutes. /// Returns the number of deleted rows. -pub async fn cleanup_old_events(db: &Pool, older_than_minutes: i32) -> Result { +pub async fn cleanup_old_events( + db: &Pool, + older_than_minutes: i32, +) -> Result { let result = sqlx::query( "DELETE FROM notify_event WHERE created_at < now() - make_interval(mins => $1)", ) diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index b3980aa596..18eeecd2ed 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -792,10 +792,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result error::Result Result<()> { diff --git a/backend/windmill-common/src/secret_backend/vault_oss.rs b/backend/windmill-common/src/secret_backend/vault_oss.rs index b338388312..eba15bff67 100644 --- a/backend/windmill-common/src/secret_backend/vault_oss.rs +++ b/backend/windmill-common/src/secret_backend/vault_oss.rs @@ -16,7 +16,10 @@ use std::sync::Arc; use crate::db::DB; use crate::error::{Error, Result}; -use super::{database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, VaultSettings}; +use super::{ + database::DatabaseBackend, SecretBackend, SecretBackendConfig, SecretMigrationReport, + VaultSettings, +}; /// Stub VaultBackend for OSS - all operations return EE required error pub struct VaultBackend; diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 950f13c185..701a0887d0 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -47,8 +47,8 @@ pub const VERBOSE_TARGET: &str = "windmill_verbose"; /// Creates a Targets filter that optionally filters out verbose logs when quiet mode is enabled. fn create_targets_filter(default_env_filter: LevelFilter) -> Targets { - let targets = Targets::new() - .with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF); + let targets = + Targets::new().with_target("windmill:job_log", tracing::level_filters::LevelFilter::OFF); if *QUIET_MODE { targets diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 4b2794c399..8a52d34669 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -163,6 +163,16 @@ pub struct Pagination { pub per_page: Option, } +#[derive(Deserialize)] +pub struct WithStarredInfoQuery { + pub with_starred_info: Option, +} + +#[derive(Deserialize)] +pub struct BulkDeleteRequest { + pub paths: Vec, +} + #[derive(Debug, Serialize, Deserialize)] pub struct StripPath(pub String); diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs new file mode 100644 index 0000000000..44097339ac --- /dev/null +++ b/backend/windmill-common/src/webhook.rs @@ -0,0 +1,154 @@ +use std::time::Duration; + +use quick_cache::sync::Cache; +use serde::Serialize; +use tokio::{select, sync::mpsc}; + +#[cfg(feature = "prometheus")] +use crate::METRICS_ENABLED; + +use crate::db::DB; +use crate::oauth2::InstanceEvent; +use crate::utils::configure_client; + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens. + static ref WEBHOOK_REQUEST_COUNT: prometheus::Histogram = prometheus::register_histogram!( + "webhook_request", + "Histogram of webhook requests made" + ) + .unwrap(); + +} + +lazy_static::lazy_static! { + + pub static ref INSTANCE_EVENTS_WEBHOOK: Option = std::env::var("INSTANCE_EVENTS_WEBHOOK").ok(); + + pub static ref WEBHOOK_CACHE: Cache> = Cache::new(100); + +} + +pub enum WebhookPayload { + WorkspaceEvent(String, WebhookMessage), + InstanceEvent(InstanceEvent), +} + +#[derive(Serialize)] +#[serde(tag = "type")] +pub enum WebhookMessage { + // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON + CreateApp { workspace: String, path: String }, + DeleteApp { workspace: String, path: String }, + UpdateApp { workspace: String, old_path: String, new_path: String }, + CreateFlow { workspace: String, path: String }, + UpdateFlow { workspace: String, old_path: String, new_path: String }, + ArchiveFlow { workspace: String, path: String }, + DeleteFlow { workspace: String, path: String }, + CreateFolder { workspace: String, name: String }, + UpdateFolder { workspace: String, name: String }, + DeleteFolder { workspace: String, name: String }, + DeleteResource { workspace: String, path: String }, + CreateResource { workspace: String, path: String }, + UpdateResource { workspace: String, old_path: String, new_path: String }, + CreateResourceType { name: String }, + DeleteResourceType { name: String }, + UpdateResourceType { name: String }, + CreateScript { workspace: String, path: String, hash: String }, + UpdateScript { workspace: String, path: String, hash: String }, + DeleteScript { workspace: String, hash: String }, + DeleteScriptPath { workspace: String, path: String }, + CreateVariable { workspace: String, path: String }, + UpdateVariable { workspace: String, old_path: String, new_path: String }, + DeleteVariable { workspace: String, path: String }, +} + +#[derive(Clone)] +pub struct WebhookShared { + pub channel: mpsc::UnboundedSender, +} + +impl WebhookShared { + pub fn new(mut shutdown_rx: tokio::sync::broadcast::Receiver<()>, db: DB) -> Self { + let (tx, mut rx) = mpsc::unbounded_channel::(); + let _process = tokio::spawn(async move { + let client = configure_client( + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + // TODO: investigate pool timeouts and such if TCP load is high + .timeout(Duration::from_secs(5)), + ) + .build() + .unwrap(); + + loop { + select! { + biased; + _ = shutdown_rx.recv() => break, + r = rx.recv() => match r { + Some(WebhookPayload::WorkspaceEvent(workspace_id, message)) => { + let webhook_opt = match WEBHOOK_CACHE.get(&workspace_id) { + Some(guard) => { + guard + }, + None => { + let Ok(mut webhook_opt) = + sqlx::query_scalar!( + "SELECT webhook FROM workspace_settings WHERE workspace_id = $1", + workspace_id + ) + .fetch_one( + &db, + ) + .await else { + tracing::error!("Webhook Message to send - but cannot get workspace settings! Workspace: {workspace_id}"); + continue; + }; + if webhook_opt.as_ref().is_some_and(|x| x.is_empty()) { + webhook_opt = None; + } + WEBHOOK_CACHE.insert(workspace_id, webhook_opt.clone()); + webhook_opt + } + }; + if let Some(url) = webhook_opt { + #[cfg(feature = "prometheus")] + let timer = if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; + tracing::info!("Sending webhook message to {}", url); + let _ = client.post(url).json(&message).send().await; + #[cfg(feature = "prometheus")] + timer.map(|x| x.stop_and_record()); + } + }, + Some(WebhookPayload::InstanceEvent(event)) => { + #[cfg(feature = "prometheus")] + if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { Some(WEBHOOK_REQUEST_COUNT.start_timer()) } else { None }; + let r = client.post(INSTANCE_EVENTS_WEBHOOK.as_ref().unwrap()).json(&event).send().await; + if let Err(e) = r { + tracing::error!("Error sending instance event: {}", e); + } + }, + None => break, + }, + } + } + }); + + Self { channel: tx } + } + + pub fn send_message(&self, workspace_id: String, message: WebhookMessage) { + let _ = self.channel.send(WebhookPayload::WorkspaceEvent( + workspace_id.clone(), + message, + )); + } + + pub fn send_instance_event(&self, event: InstanceEvent) { + if INSTANCE_EVENTS_WEBHOOK.is_none() { + return; + } + let _ = self.channel.send(WebhookPayload::InstanceEvent(event)); + } +} diff --git a/backend/windmill-mcp/Cargo.toml b/backend/windmill-mcp/Cargo.toml index afe2547f19..3968ea0836 100644 --- a/backend/windmill-mcp/Cargo.toml +++ b/backend/windmill-mcp/Cargo.toml @@ -11,7 +11,7 @@ path = "src/lib.rs" [features] default = [] server = ["rmcp/transport-streamable-http-server", "rmcp/transport-streamable-http-server-session", "rmcp/transport-worker", "dep:sqlx", "dep:async-trait", "dep:http", "dep:tokio-util", "dep:tokio"] -auth = ["rmcp/auth", "dep:oauth2"] +auth = ["rmcp/auth", "dep:oauth2", "dep:sqlx", "dep:chrono"] [dependencies] oauth2 = { version = "5.0", optional = true } @@ -23,6 +23,7 @@ serde_json.workspace = true tracing.workspace = true rmcp.workspace = true sqlx = { workspace = true, optional = true } +chrono = { workspace = true, optional = true } async-trait = { workspace = true, optional = true } http = { workspace = true, optional = true } tokio-util = { workspace = true, features = ["rt"], optional = true } diff --git a/backend/windmill-mcp/src/client_registration.rs b/backend/windmill-mcp/src/client_registration.rs new file mode 100644 index 0000000000..87736f2915 --- /dev/null +++ b/backend/windmill-mcp/src/client_registration.rs @@ -0,0 +1,208 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! MCP OAuth client registration and credential management. +//! +//! Handles dynamic client registration (DCR) and Client Identifier Metadata Document (CIMD) +//! flows for MCP OAuth, including caching registered credentials in the database. + +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use windmill_common::db::DB; +use windmill_common::error; +use windmill_common::variables::{build_crypt, decrypt, encrypt}; + +use crate::oauth::AuthorizationManager; + +/// MCP client credentials returned by [`get_or_refresh_mcp_client`]. +pub struct McpClientCredentials { + pub client_id: String, + pub client_secret: Option, + pub token_endpoint: String, +} + +#[derive(FromRow)] +struct McpOAuthClient { + #[allow(dead_code)] + mcp_server_url: String, + client_id: String, + client_secret: Option, + client_secret_expires_at: Option, + token_endpoint: String, +} + +impl McpOAuthClient { + fn is_expired(&self) -> bool { + if let Some(expires_at) = self.client_secret_expires_at { + expires_at < chrono::Utc::now().naive_utc() + } else { + false + } + } +} + +async fn encrypt_client_secret(db: &DB, client_secret: &str) -> error::Result { + let mc = build_crypt(db, "admins").await?; + Ok(encrypt(&mc, client_secret)) +} + +async fn decrypt_client_secret(db: &DB, encrypted_secret: &str) -> error::Result { + let mc = build_crypt(db, "admins").await?; + decrypt(&mc, encrypted_secret.to_string()) +} + +#[derive(Serialize)] +struct DcrRequest { + client_name: String, + redirect_uris: Vec, + grant_types: Vec, + response_types: Vec, + token_endpoint_auth_method: String, +} + +#[derive(Deserialize)] +struct DcrResponse { + client_id: String, + client_secret: Option, + client_secret_expires_at: Option, +} + +async fn register_client( + registration_endpoint: &str, + redirect_uri: &str, + client_name: &str, +) -> Result { + let client = reqwest::Client::new(); + let request = DcrRequest { + client_name: client_name.to_string(), + redirect_uris: vec![redirect_uri.to_string()], + grant_types: vec![ + "authorization_code".to_string(), + "refresh_token".to_string(), + ], + response_types: vec!["code".to_string()], + token_endpoint_auth_method: "none".to_string(), + }; + + let response = client + .post(registration_endpoint) + .json(&request) + .send() + .await + .map_err(|e| error::Error::BadRequest(format!("DCR request failed: {e}")))?; + + if !response.status().is_success() { + let error_text = response.text().await.unwrap_or_default(); + return Err(error::Error::BadRequest(format!( + "DCR failed: {error_text}" + ))); + } + + response + .json::() + .await + .map_err(|e| error::Error::BadRequest(format!("DCR response parse failed: {e}"))) +} + +/// Get MCP client credentials from cache, re-registering if expired. +/// +/// Supports two modes: +/// - DCR (Dynamic Client Registration): If the server's metadata includes a registration endpoint +/// - CIMD (Client Identifier Metadata Document): Otherwise, uses our metadata URL as client_id +pub async fn get_or_refresh_mcp_client( + db: &DB, + mcp_server_url: &str, +) -> Result { + let base_url = windmill_common::BASE_URL.read().await.clone(); + let redirect_uri = format!("{}/api/mcp/oauth/callback", base_url); + + let cached_client: Option = + sqlx::query_as("SELECT * FROM mcp_oauth_client WHERE mcp_server_url = $1") + .bind(mcp_server_url) + .fetch_optional(db) + .await + .map_err(|e| error::Error::InternalErr(format!("Database error: {e}")))?; + + if let Some(client) = cached_client { + if !client.is_expired() { + tracing::debug!("Using cached MCP client for {}", mcp_server_url); + let decrypted_secret = if let Some(ref encrypted_secret) = client.client_secret { + Some(decrypt_client_secret(db, encrypted_secret).await?) + } else { + None + }; + return Ok(McpClientCredentials { + client_id: client.client_id, + client_secret: decrypted_secret, + token_endpoint: client.token_endpoint, + }); + } + tracing::debug!("Cached MCP client expired, re-registering"); + } + + let manager = AuthorizationManager::new(mcp_server_url) + .await + .map_err(|e| error::Error::BadRequest(format!("Failed to create auth manager: {e}")))?; + + let metadata = manager + .discover_metadata() + .await + .map_err(|e| error::Error::BadRequest(format!("OAuth discovery failed: {e}")))?; + + let supports_dynamic_registration = metadata.registration_endpoint.is_some(); + + let (client_id, client_secret, expires_at) = if supports_dynamic_registration { + let reg_endpoint = metadata.registration_endpoint.as_ref().unwrap(); + tracing::debug!("Performing DCR at {}", reg_endpoint); + let dcr = register_client(reg_endpoint, &redirect_uri, "Windmill").await?; + + let expires_at = dcr.client_secret_expires_at.and_then(|ts| { + if ts == 0 { + None + } else { + chrono::DateTime::from_timestamp(ts, 0).map(|dt| dt.naive_utc()) + } + }); + + (dcr.client_id, dcr.client_secret, expires_at) + } else { + let client_metadata_url = format!("{}/api/mcp/oauth/client-metadata.json", base_url); + tracing::debug!( + "Using CIMD with metadata URL as client_id: {}", + client_metadata_url + ); + + (client_metadata_url, None, None) + }; + + let encrypted_secret = if let Some(ref secret) = client_secret { + Some(encrypt_client_secret(db, secret).await?) + } else { + None + }; + + sqlx::query( + "INSERT INTO mcp_oauth_client (mcp_server_url, client_id, client_secret, client_secret_expires_at, token_endpoint) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (mcp_server_url) DO UPDATE + SET client_id = EXCLUDED.client_id, + client_secret = EXCLUDED.client_secret, + client_secret_expires_at = EXCLUDED.client_secret_expires_at, + token_endpoint = EXCLUDED.token_endpoint", + ) + .bind(mcp_server_url) + .bind(&client_id) + .bind(&encrypted_secret) + .bind(expires_at) + .bind(&metadata.token_endpoint) + .execute(db) + .await + .map_err(|e| error::Error::InternalErr(format!("Database error: {e}")))?; + + Ok(McpClientCredentials { client_id, client_secret, token_endpoint: metadata.token_endpoint }) +} diff --git a/backend/windmill-mcp/src/lib.rs b/backend/windmill-mcp/src/lib.rs index 9026837493..a75b545154 100644 --- a/backend/windmill-mcp/src/lib.rs +++ b/backend/windmill-mcp/src/lib.rs @@ -29,6 +29,10 @@ pub use rmcp::model::Tool as McpTool; #[cfg(feature = "server")] pub mod server; +// MCP OAuth client registration (when auth feature is enabled) +#[cfg(feature = "auth")] +pub mod client_registration; + // Re-export rmcp auth types when auth feature is enabled #[cfg(feature = "auth")] pub mod oauth { diff --git a/backend/windmill-native-triggers/Cargo.toml b/backend/windmill-native-triggers/Cargo.toml new file mode 100644 index 0000000000..126c22c7c0 --- /dev/null +++ b/backend/windmill-native-triggers/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "windmill-native-triggers" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_native_triggers" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] +native_trigger = ["dep:windmill-oauth", "dep:backon"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true +windmill-queue.workspace = true +windmill-oauth = { workspace = true, optional = true } +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +http.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +lazy_static.workspace = true +reqwest.workspace = true +chrono.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true +hmac.workspace = true +sha2.workspace = true +urlencoding.workspace = true +uuid.workspace = true +strum.workspace = true +backon = { workspace = true, optional = true } diff --git a/backend/windmill-api/src/native_triggers/handler.rs b/backend/windmill-native-triggers/src/handler.rs similarity index 93% rename from backend/windmill-api/src/native_triggers/handler.rs rename to backend/windmill-native-triggers/src/handler.rs index 3c42b5261b..ef054d4664 100644 --- a/backend/windmill-api/src/native_triggers/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -1,13 +1,8 @@ use crate::{ - db::ApiAuthed, - native_triggers::{ - delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, - get_workspace_integration, list_native_triggers, store_native_trigger, - update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, - NativeTriggerData, ServiceName, - }, - users::{create_token_internal, NewToken}, - utils::check_scopes, + delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, + get_workspace_integration, list_native_triggers, store_native_trigger, + update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, NativeTriggerData, + ServiceName, }; use axum::{ extract::{Path, Query}, @@ -17,6 +12,9 @@ use axum::{ use serde::{Deserialize, Serialize}; use sqlx::PgConnection; use std::sync::Arc; +use windmill_api_auth::{ + check_scopes, create_token_internal, require_is_writer, ApiAuthed, NewToken, +}; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, @@ -33,9 +31,25 @@ async fn require_is_writer_on_runnable( db: DB, ) -> Result<()> { if is_flow { - crate::flows::require_is_writer(authed, path, w_id, db).await + require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM flow WHERE path = $1 AND workspace_id = $2", + "flow", + ) + .await } else { - crate::scripts::require_is_writer(authed, path, w_id, db).await + require_is_writer( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "script", + ) + .await } } @@ -548,11 +562,11 @@ pub fn generate_native_trigger_routers() -> Router { #[cfg(feature = "native_trigger")] { - use crate::native_triggers::nextcloud::NextCloud; + use crate::nextcloud::NextCloud; // Register all service routes here // When adding a new service: - // 1. Import the handler: use crate::native_triggers::newservice::NewServiceHandler; + // 1. Import the handler: use crate::newservice::NewServiceHandler; // 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler)) return router.nest("/nextcloud", service_routes(NextCloud)); // Add new services here: diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs new file mode 100644 index 0000000000..456580b3ef --- /dev/null +++ b/backend/windmill-native-triggers/src/lib.rs @@ -0,0 +1,1053 @@ +//! Native Triggers Module +//! +//! This module provides integration with external services (like Nextcloud) that can +//! trigger Windmill scripts/flows via webhooks. +//! +//! ## Adding a New Native Trigger Service +//! +//! When adding a new service (e.g., "NewService"), you need to update the following locations: +//! +//! ### 1. This file (lib.rs): +//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block +//! - Add `NewService` variant to `ServiceName` enum +//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` +//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` +//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) +//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) +//! - Update `ServiceName::fmt()` (Display impl) - add match arm +//! +//! ### 2. sync.rs: +//! - Add `sync_service!()` macro call in `sync_all_triggers()` +//! +//! ### 3. handler.rs: +//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` +//! +//! ### 4. Database migration: +//! - Add `'newservice'` to the `native_trigger_service` enum type +//! +//! ### 5. windmill_common (if needed): +//! - Add `NewService` variant to `TriggerKind` enum +//! - Add `'newservice'` to `job_trigger_kind` enum type in migration +//! +//! The generic code (trait definitions, route handlers, database operations) does NOT +//! need modification when adding new services. + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use http::StatusCode; +use itertools::Itertools; +use reqwest::{Client, Method}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::json; +use serde_json::value::RawValue; +use sqlx::{FromRow, PgConnection, Postgres}; +use std::{collections::HashMap, fmt::Debug}; +use strum::{EnumIter, IntoEnumIterator}; +use tokio::task; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + variables::{build_crypt, decrypt, encrypt}, + DB, +}; +use windmill_queue::PushArgsOwned; + +#[cfg(feature = "native_trigger")] +use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; + +use windmill_api_auth::ApiAuthed; +pub mod handler; +pub mod sync; +pub mod workspace_integrations; + +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod nextcloud; +// #[cfg(feature = "native_trigger")] +// pub mod newservice; + +/// Enum of all supported native trigger services. +/// When adding a new service, add a variant here (e.g., `NewService`). +#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ServiceName { + Nextcloud, + // Add new services here: + // NewService, +} + +impl TryFrom for ServiceName { + type Error = Error; + fn try_from(value: String) -> std::result::Result { + // Add new service match arms here: + let service = match value.as_str() { + "nextcloud" => ServiceName::Nextcloud, + // "newservice" => ServiceName::NewService, + _ => { + return Err(anyhow::anyhow!( + "Unknown service, currently supported services are: [{}]", + ServiceName::iter().join(",") + ) + .into()) + } + }; + + Ok(service) + } +} + +impl ServiceName { + /// Returns the lowercase string identifier for this service. + /// Add new service match arms here. + pub fn as_str(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "nextcloud", + // ServiceName::NewService => "newservice", + } + } + + /// Returns the corresponding TriggerKind for this service. + /// Requires adding the variant to TriggerKind in windmill_common. + pub fn as_trigger_kind(&self) -> TriggerKind { + match self { + ServiceName::Nextcloud => TriggerKind::Nextcloud, + // ServiceName::NewService => TriggerKind::NewService, + } + } + + /// Returns the corresponding JobTriggerKind for this service. + /// Requires adding the variant to JobTriggerKind in windmill_common. + pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { + match self { + ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, + // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, + } + } + + /// Returns the OAuth token endpoint path for this service. + /// Used for building OAuth clients dynamically. + pub fn token_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", + // ServiceName::NewService => "/oauth/token", + } + } + + /// Returns the OAuth authorization endpoint path for this service. + /// Used for building OAuth authorization URLs. + pub fn auth_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/authorize", + // ServiceName::NewService => "/oauth/authorize", + } + } +} + +impl std::fmt::Display for ServiceName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct NativeTrigger { + pub external_id: String, + pub workspace_id: String, + pub service_name: ServiceName, + pub script_path: String, + pub is_flow: bool, + pub webhook_token_prefix: String, + pub service_config: Option, + pub error: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NativeTriggerConfig { + pub script_path: String, + pub is_flow: bool, + pub webhook_token: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct NativeTriggerData { + pub script_path: String, + pub is_flow: bool, + pub service_config: C, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WorkspaceIntegration { + pub workspace_id: String, + pub service_name: ServiceName, + pub oauth_data: serde_json::Value, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +#[async_trait] +pub trait External: Send + Sync + 'static { + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + #[allow(unused)] + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn list_all( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + db: &DB, + tx: &mut PgConnection, + ) -> Result>; + + async fn prepare_webhook( + &self, + _db: &DB, + _w_id: &str, + _header: HashMap, + _body: String, + _script_path: &str, + _is_flow: bool, + ) -> Result { + Ok(PushArgsOwned { extra: None, args: HashMap::new() }) + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option); + + fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; + + /// Extracts the service-specific config from trigger data (from external service). + /// Used for comparison during sync to detect config drift. + /// Default implementation converts the trigger data to a JSON value + /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. + fn extract_service_config_from_trigger_data( + &self, + data: &Self::TriggerData, + ) -> Result { + serde_json::to_value(data).map_err(|e| { + Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) + }) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn http_client_request( + &self, + url: &str, + method: Method, + workspace_id: &str, + tx: &mut PgConnection, + db: &DB, + headers: Option>, + body: Option<&B>, + ) -> Result { + let oauth_config: OAuthConfig = + decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; + + let result = make_http_request( + url, + method.clone(), + headers.clone(), + body.as_ref(), + &oauth_config.access_token, + ) + .await; + + match result { + Ok(response) => Ok(response), + Err(err) + if err.status() == Some(StatusCode::UNAUTHORIZED) + || err.status() == Some(StatusCode::FORBIDDEN) => + { + tracing::info!( + "HTTP auth error ({}), attempting token refresh", + err.status().unwrap() + ); + + let refreshed_oauth_config = + refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; + + task::spawn({ + let db_clone = db.clone(); + let workspace_id_clone = workspace_id.to_string(); + let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); + async move { + update_workspace_integration_tokens_helper( + db_clone, + workspace_id_clone, + Self::SERVICE_NAME, + refreshed_json, + ) + .await; + } + }); + + let response = make_http_request( + url, + method, + headers, + body.as_ref(), + &refreshed_oauth_config.access_token, + ) + .await + .map_err(to_anyhow)?; + Ok(response) + } + Err(e) => Err(to_anyhow(e).into()), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, + pub refresh_token: Option, + pub client_id: String, + pub client_secret: String, +} + +pub async fn make_http_request( + url: &str, + method: Method, + headers: Option>, + body: Option<&B>, + access_token: &str, +) -> std::result::Result { + let client = Client::new(); + let mut request = client.request(method, url); + + request = request + .header("Accept", "application/json") + .header("Authorization", format!("Bearer {}", access_token)); + + if body.is_some() { + request = request.header("Content-Type", "application/json"); + } + + if let Some(custom_headers) = headers { + for (key, value) in custom_headers { + request = request.header(key, value); + } + } + + if let Some(body_content) = body { + request = request.json(body_content); + } + + let response = request.send().await?.error_for_status()?; + + let response_json = response.json().await?; + + Ok(response_json) +} + +pub async fn decrypt_oauth_data< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, + T: DeserializeOwned, +>( + tx: E, + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = get_workspace_integration(tx, workspace_id, service_name).await?; + + let mc = build_crypt(db, workspace_id).await?; + let mut oauth_data: serde_json::Value = integration.oauth_data; + + if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { + let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; + oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); + } + + if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) + { + let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; + oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); + } + + serde_json::from_value(oauth_data) + .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) +} + +#[allow(unused)] +pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { + let base_url = oauth_data + .get("base_url") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? + .to_string(); + + let access_token = oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? + .to_string(); + + let refresh_token = oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let client_id = oauth_data + .get("client_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? + .to_string(); + + let client_secret = oauth_data + .get("client_secret") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? + .to_string(); + + Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) +} + +#[inline] +pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { + let mut json = json!({ + "base_url": config.base_url, + "access_token": config.access_token, + "client_id": config.client_id, + "client_secret": config.client_secret, + }); + + if let Some(refresh_token) = &config.refresh_token { + json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); + } + + json +} + +/// Token refresh response +#[cfg(feature = "native_trigger")] +#[derive(Debug, Deserialize)] +struct RefreshTokenResponse { + access_token: String, + refresh_token: Option, +} + +/// Refresh OAuth tokens using windmill-oauth. +#[cfg(feature = "native_trigger")] +pub async fn refresh_oauth_tokens( + oauth_config: &OAuthConfig, + refresh_endpoint: &str, +) -> Result { + let refresh_token_str = oauth_config + .refresh_token + .as_ref() + .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; + + // Build OAuth client for token refresh + // Auth URL is not used for refresh, but required by the client constructor + let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + + let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); + client.set_client_secret(oauth_config.client_secret.clone()); + + let token_response: RefreshTokenResponse = client + .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) + .with_client(&*OAUTH_HTTP_CLIENT) + .execute() + .await + .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; + + Ok(OAuthConfig { + base_url: oauth_config.base_url.clone(), + access_token: token_response.access_token, + refresh_token: token_response + .refresh_token + .or_else(|| oauth_config.refresh_token.clone()), + client_id: oauth_config.client_id.clone(), + client_secret: oauth_config.client_secret.clone(), + }) +} + +/// Fallback refresh without native_triggers feature +#[cfg(not(feature = "native_trigger"))] +pub async fn refresh_oauth_tokens( + _oauth_config: &OAuthConfig, + _refresh_endpoint: &str, +) -> Result { + Err(Error::InternalErr( + "Native triggers feature is not enabled".to_string(), + )) +} + +async fn update_workspace_integration_tokens_helper( + db: DB, + workspace_id: String, + service_name: ServiceName, + oauth_data: serde_json::Value, +) { + let result = async { + let mut tx = db.begin().await?; + let mc = build_crypt(&db, &workspace_id).await?; + let mut encrypted_oauth_data = oauth_data; + + if let Some(access_token) = encrypted_oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + { + let encrypted_access_token = encrypt(&mc, access_token); + encrypted_oauth_data["access_token"] = + serde_json::Value::String(encrypted_access_token); + } + + if let Some(refresh_token) = encrypted_oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + { + let encrypted_refresh_token = encrypt(&mc, refresh_token); + encrypted_oauth_data["refresh_token"] = + serde_json::Value::String(encrypted_refresh_token); + } + + sqlx::query!( + r#" + UPDATE workspace_integrations + SET oauth_data = $1, updated_at = now() + WHERE workspace_id = $2 AND service_name = $3 + "#, + encrypted_oauth_data, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(e) = result { + tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", + service_name, workspace_id, e); + } +} + +/// Look up the full token from the token table using its prefix +pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result> { + let token = sqlx::query_scalar!( + r#" + SELECT token + FROM token + WHERE token LIKE concat($1::text, '%') + LIMIT 1 + "#, + token_prefix + ) + .fetch_optional(db) + .await?; + + Ok(token) +} + +/// Delete a token from the token table using its prefix +pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM token + WHERE token LIKE concat($1::text, '%') + "#, + token_prefix + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: C, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + INSERT INTO native_trigger ( + external_id, + workspace_id, + service_name, + script_path, + is_flow, + webhook_token_prefix, + service_config + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + ON CONFLICT (external_id, workspace_id, service_name) + DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() + "#, + external_id, + workspace_id, + service_name as ServiceName, + config.script_path, + config.is_flow, + webhook_token_prefix, + sqlx::types::Json(service_config) as _, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: Option<&RawValue>, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + UPDATE native_trigger + SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() + WHERE + workspace_id = $5 + AND service_name = $6 + AND external_id = $7 + "#, + config.script_path, + config.is_flow, + webhook_token_prefix, + service_config.map(sqlx::types::Json) as _, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} +pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + script_path: &str, + is_flow: bool, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND script_path = $3 + AND is_flow = $4 + LIMIT 1 + "#, + workspace_id, + service_name as ServiceName, + script_path, + is_flow + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + page: Option, + per_page: Option, + path: Option<&str>, + is_flow: Option, +) -> Result> { + let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; + let limit = per_page.unwrap_or(100) as i64; + + let triggers = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + nt.external_id, + nt.workspace_id, + nt.service_name AS "service_name!: ServiceName", + nt.script_path, + nt.is_flow, + nt.webhook_token_prefix, + nt.service_config, + nt.error, + nt.created_at, + nt.updated_at + FROM + native_trigger nt + WHERE + nt.workspace_id = $1 AND + nt.service_name = $2 AND + ($5::text IS NULL OR nt.script_path = $5) AND + ($6::bool IS NULL OR nt.is_flow = $6) AND + ( + (nt.is_flow = false AND EXISTS ( + SELECT 1 FROM script s + WHERE s.workspace_id = nt.workspace_id + AND s.path = nt.script_path + )) + OR + (nt.is_flow = true AND EXISTS ( + SELECT 1 FROM flow f + WHERE f.workspace_id = nt.workspace_id + AND f.path = nt.script_path + )) + ) + LIMIT $3 + OFFSET $4 + "#, + workspace_id, + service_name as ServiceName, + limit, + offset, + path, + is_flow + ) + .fetch_all(db) + .await?; + + Ok(triggers) +} + +pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + error: Option<&str>, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET error = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + error, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger_service_config< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, +>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + service_config: &serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET service_config = $1, updated_at = NOW() + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + service_config, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn store_workspace_integration( + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + service_name: ServiceName, + oauth_data: serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO workspace_integrations ( + workspace_id, + service_name, + oauth_data, + created_by, + created_at, + updated_at + ) VALUES ( + $1, $2, $3, $4, now(), now() + ) + ON CONFLICT (workspace_id, service_name) + DO UPDATE SET + oauth_data = $3, + updated_at = now() + "#, + workspace_id, + service_name as ServiceName, + oauth_data, + authed.username, + ) + .execute(&mut *tx) + .await?; + + Ok(()) +} + +pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = sqlx::query_as!( + WorkspaceIntegration, + r#" + SELECT + workspace_id, + service_name AS "service_name!: ServiceName", + oauth_data, + created_at, + updated_at, + created_by + FROM + workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .fetch_one(db) + .await?; + + Ok(integration) +} + +pub async fn delete_workspace_integration( + tx: &mut PgConnection, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +/// Generates the webhook URL that external services will call. +/// +/// `external_id` is optional because during CREATE we don't have it yet +/// (it's returned by the external service). During UPDATE, we have it. +pub fn generate_webhook_service_url( + base_url: &str, + w_id: &str, + script_path: &str, + is_flow: bool, + external_id: Option<&str>, + service_name: ServiceName, + webhook_token: &str, +) -> String { + let runnable_prefix = if is_flow { "f" } else { "p" }; + + let mut url = format!( + "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", + base_url, + w_id, + runnable_prefix, + script_path, + &webhook_token, + service_name.as_str(), + ); + + if let Some(id) = external_id { + url.push_str(&format!("&trigger_external_id={}", id)); + } + + url +} diff --git a/backend/windmill-api/src/native_triggers/nextcloud/external.rs b/backend/windmill-native-triggers/src/nextcloud/external.rs similarity index 99% rename from backend/windmill-api/src/native_triggers/nextcloud/external.rs rename to backend/windmill-native-triggers/src/nextcloud/external.rs index 028ba131c3..54b7578229 100644 --- a/backend/windmill-api/src/native_triggers/nextcloud/external.rs +++ b/backend/windmill-native-triggers/src/nextcloud/external.rs @@ -8,7 +8,7 @@ use windmill_common::{ BASE_URL, DB, }; -use crate::native_triggers::{ +use crate::{ generate_webhook_service_url, nextcloud::{ routes, NextCloud, NextCloudOAuthData, NextCloudTriggerData, NextcloudServiceConfig, diff --git a/backend/windmill-api/src/native_triggers/nextcloud/mod.rs b/backend/windmill-native-triggers/src/nextcloud/mod.rs similarity index 100% rename from backend/windmill-api/src/native_triggers/nextcloud/mod.rs rename to backend/windmill-native-triggers/src/nextcloud/mod.rs diff --git a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs b/backend/windmill-native-triggers/src/nextcloud/routes.rs similarity index 90% rename from backend/windmill-api/src/native_triggers/nextcloud/routes.rs rename to backend/windmill-native-triggers/src/nextcloud/routes.rs index 40633dc248..29534b9642 100644 --- a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs +++ b/backend/windmill-native-triggers/src/nextcloud/routes.rs @@ -9,13 +9,11 @@ use windmill_common::{ }; use crate::{ - db::ApiAuthed, - native_triggers::{ - get_workspace_integration, - nextcloud::{NextCloudEventType, OcsResponse}, - External, OAuthConfig, ServiceName, - }, + get_workspace_integration, + nextcloud::{NextCloudEventType, OcsResponse}, + External, OAuthConfig, ServiceName, }; +use windmill_api_auth::ApiAuthed; async fn list_available_events( authed: ApiAuthed, diff --git a/backend/windmill-api/src/native_triggers/sync.rs b/backend/windmill-native-triggers/src/sync.rs similarity index 98% rename from backend/windmill-api/src/native_triggers/sync.rs rename to backend/windmill-native-triggers/src/sync.rs index ca955630aa..6e7b70c098 100644 --- a/backend/windmill-api/src/native_triggers/sync.rs +++ b/backend/windmill-native-triggers/src/sync.rs @@ -4,9 +4,12 @@ use windmill_common::DB; use serde::Serialize; -use crate::native_triggers::{ +use crate::ServiceName; + +#[cfg(feature = "native_trigger")] +use crate::{ decrypt_oauth_data, list_native_triggers, update_native_trigger_error, - update_native_trigger_service_config, External, ServiceName, + update_native_trigger_service_config, External, }; #[derive(Debug, Serialize)] @@ -44,6 +47,7 @@ pub struct ServiceSyncResult { pub errors: Vec, } +#[allow(unused_variables, unused_mut)] pub async fn sync_all_triggers(db: &DB) -> Result { tracing::info!("Starting native triggers sync"); @@ -56,7 +60,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { // Each service only syncs workspaces that have the corresponding integration configured #[cfg(feature = "native_trigger")] { - use crate::native_triggers::nextcloud::NextCloud; + use crate::nextcloud::NextCloud; let (service_name, result) = sync_service_triggers(db, NextCloud).await; total_synced += result.synced_triggers.len(); @@ -64,7 +68,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { service_results.insert(service_name, result); // Add new services here: - // use crate::native_triggers::newservice::NewService; + // use crate::newservice::NewService; // let (service_name, result) = sync_service_triggers(db, NewService).await; // total_synced += result.synced_triggers.len(); // total_errors += result.errors.len(); @@ -93,6 +97,7 @@ pub async fn sync_all_triggers(db: &DB) -> Result { Ok(result) } +#[cfg(feature = "native_trigger")] async fn sync_service_triggers( db: &DB, handler: T, diff --git a/backend/windmill-api/src/native_triggers/workspace_integrations.rs b/backend/windmill-native-triggers/src/workspace_integrations.rs similarity index 87% rename from backend/windmill-api/src/native_triggers/workspace_integrations.rs rename to backend/windmill-native-triggers/src/workspace_integrations.rs index dd823b1344..3891905411 100644 --- a/backend/windmill-api/src/native_triggers/workspace_integrations.rs +++ b/backend/windmill-native-triggers/src/workspace_integrations.rs @@ -1,17 +1,25 @@ +use axum::Router; + +#[cfg(feature = "native_trigger")] use axum::{ extract::Path, routing::{delete, get, post}, - Extension, Json, Router, + Extension, Json, }; #[cfg(feature = "native_trigger")] use serde_json::to_value; -use sqlx::prelude::FromRow; -use strum::IntoEnumIterator; +#[cfg(feature = "native_trigger")] use serde::{de::DeserializeOwned, Deserialize, Serialize}; + +#[cfg(feature = "native_trigger")] +use sqlx::prelude::FromRow; + #[cfg(feature = "native_trigger")] use windmill_audit::{audit_oss::audit_log, ActionKind}; + +#[cfg(feature = "native_trigger")] use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, @@ -21,20 +29,28 @@ use windmill_common::{ }; #[cfg(feature = "native_trigger")] -use crate::{ - db::ApiAuthed, - native_triggers::{delete_workspace_integration, store_workspace_integration, ServiceName}, -}; +use windmill_api_auth::ApiAuthed; + +#[cfg(feature = "native_trigger")] +use crate::ServiceName; + +#[cfg(feature = "native_trigger")] +use crate::{delete_workspace_integration, store_workspace_integration}; #[cfg(feature = "native_trigger")] use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT}; +#[cfg(feature = "native_trigger")] use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +#[cfg(feature = "native_trigger")] use hmac::{Hmac, Mac}; +#[cfg(feature = "native_trigger")] use sha2::Sha256; +#[cfg(feature = "native_trigger")] type HmacSha256 = Hmac; +#[cfg(feature = "native_trigger")] const STATE_EXPIRATION_SECONDS: i64 = 600; // 10 minutes /// Generate a signed OAuth state that is cluster-safe. @@ -60,8 +76,8 @@ async fn generate_signed_state( // Get workspace key for signing let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + let mut mac = HmacSha256::new_from_slice(key.as_bytes()) + .map_err(|e| Error::InternalErr(e.to_string()))?; mac.update(payload.as_bytes()); let signature = mac.finalize().into_bytes(); @@ -121,8 +137,8 @@ async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Resu // Verify signature let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + let mut mac = HmacSha256::new_from_slice(key.as_bytes()) + .map_err(|e| Error::InternalErr(e.to_string()))?; mac.update(payload.as_bytes()); let received_signature = match URL_SAFE_NO_PAD.decode(encoded_signature) { @@ -133,6 +149,7 @@ async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Resu Ok(mac.verify_slice(&received_signature).is_ok()) } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct IntegrationStatusResponse { pub connected: bool, @@ -141,24 +158,28 @@ pub struct IntegrationStatusResponse { pub created_by: Option, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct ListIntegrationsResponse { pub integrations: Vec, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct ConnectIntegrationResponse { pub auth_url: String, } +#[cfg(feature = "native_trigger")] #[derive(FromRow, Debug, Clone, Serialize, Deserialize)] pub struct WorkspaceOAuthConfig { pub client_id: String, pub client_secret: String, pub base_url: String, - pub access_token: Option + pub access_token: Option, } +#[cfg(feature = "native_trigger")] #[derive(Debug, Serialize)] pub struct OAuthConfigResponse { pub configured: bool, @@ -222,6 +243,7 @@ async fn delete_integration( ))) } +#[cfg(feature = "native_trigger")] #[derive(FromRow, Debug, Deserialize, Serialize)] struct WorkspaceIntegrations { service_name: ServiceName, @@ -240,12 +262,12 @@ async fn list_integrations( let integrations = sqlx::query_as!( WorkspaceIntegrations, r#" - SELECT + SELECT oauth_data as "oauth_data!: sqlx::types::Json", service_name as "service_name!: ServiceName" - FROM - workspace_integrations - WHERE + FROM + workspace_integrations + WHERE workspace_id = $1 "#, workspace_id @@ -258,6 +280,7 @@ async fn list_integrations( .map(|integration| (integration.service_name, integration.oauth_data)) .collect::>(); + use strum::IntoEnumIterator; let integrations = ServiceName::iter() .map(|service_name| WorkspaceIntegrations { service_name: service_name, @@ -270,6 +293,7 @@ async fn list_integrations( Ok(Json(integrations)) } +#[cfg(feature = "native_trigger")] async fn integration_exist( authed: ApiAuthed, Extension(user_db): Extension, @@ -282,7 +306,7 @@ async fn integration_exist( SELECT 1 FROM workspace_integrations WHERE workspace_id = $1 - AND service_name = $2 + AND service_name = $2 AND oauth_data IS NOT NULL ) "#, @@ -296,6 +320,7 @@ async fn integration_exist( Ok(Json(exists)) } +#[cfg(feature = "native_trigger")] #[derive(Debug, Deserialize)] struct RedirectUri { redirect_uri: String, @@ -366,6 +391,7 @@ async fn oauth_callback( } /// Token response from OAuth token exchange +#[cfg(feature = "native_trigger")] #[derive(Debug, Deserialize)] struct TokenResponse { access_token: String, @@ -381,10 +407,18 @@ fn build_native_oauth_client( service_name: ServiceName, redirect_uri: &str, ) -> Result { - let auth_url = Url::parse(&format!("{}{}", config.base_url, service_name.auth_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", config.base_url, service_name.token_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + let auth_url = Url::parse(&format!( + "{}{}", + config.base_url, + service_name.auth_endpoint() + )) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!( + "{}{}", + config.base_url, + service_name.token_endpoint() + )) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; let redirect = Url::parse(redirect_uri).map_err(|e| { Error::BadRequest(format!( "Invalid redirect URI '{}': {}. The redirect URI must be an absolute URL (e.g., https://example.com/callback)", @@ -419,6 +453,7 @@ async fn exchange_code_for_token( Ok(token_response) } +#[cfg(feature = "native_trigger")] async fn get_workspace_oauth_config( db: &DB, workspace_id: &str, @@ -426,11 +461,11 @@ async fn get_workspace_oauth_config( ) -> Result { let oauth_configs = sqlx::query_scalar!( r#" - SELECT - oauth_data - FROM - workspace_integrations - WHERE + SELECT + oauth_data + FROM + workspace_integrations + WHERE workspace_id = $1 AND service_name = $2 "#, @@ -461,7 +496,7 @@ pub async fn create_workspace_integration( let mut tx = user_db.begin(&authed).await?; - store_workspace_integration( + crate::store_workspace_integration( &mut tx, &authed, &workspace_id, @@ -475,6 +510,7 @@ pub async fn create_workspace_integration( Ok(()) } +#[cfg(feature = "native_trigger")] #[inline] async fn get_workspace_oauth_config_as_oauth_config( db: &DB, @@ -484,6 +520,7 @@ async fn get_workspace_oauth_config_as_oauth_config( get_workspace_oauth_config::(db, workspace_id, service_name).await } +#[cfg(feature = "native_trigger")] fn build_authorization_url( config: &WorkspaceOAuthConfig, state: &str, @@ -506,6 +543,7 @@ fn build_authorization_url( format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string) } +#[cfg(feature = "native_trigger")] pub fn workspaced_service() -> Router { let router = Router::new() .route("/list", get(list_integrations)) @@ -520,3 +558,8 @@ pub fn workspaced_service() -> Router { Router::new().nest("/integrations", router) } + +#[cfg(not(feature = "native_trigger"))] +pub fn workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index c89cb190b7..82511819f8 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -36,8 +36,7 @@ pub type DB = sqlx::Pool; // Re-export oauth2 types that consumers need (also used internally) pub use oauth2::{ - AccessToken, AuthType, Client as OClient, RefreshToken, Scope, State, Url, - helpers, + helpers, AccessToken, AuthType, Client as OClient, RefreshToken, Scope, State, Url, }; // Re-export reqwest Client (version 0.12 compatible with async-oauth2) @@ -55,6 +54,12 @@ lazy_static::lazy_static! { .timeout(std::time::Duration::from_secs(30)) .build() .expect("Failed to create OAuth HTTP client"); + + pub static ref OAUTH_CLIENTS: Arc> = Arc::new(RwLock::new(AllClients { + logins: HashMap::new(), + connects: HashMap::new(), + slack: None + })); } /// OAuth client with associated scopes and configuration @@ -221,10 +226,7 @@ pub async fn build_oauth_clients( .collect() }; - tracing::info!( - "OAuth loaded clients: {}", - oauths.keys().join(", ") - ); + tracing::info!("OAuth loaded clients: {}", oauths.keys().join(", ")); let logins = login_configs .into_iter() @@ -446,9 +448,7 @@ pub async fn build_client_credentials_oauth_client( } else { let static_configs = serde_json::from_str::>(connect_configs_json).map_err( - |e| { - error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)) - }, + |e| error::Error::InternalErr(format!("Failed to parse oauth_connect.json: {}", e)), )?; static_configs.get(client_name).cloned().ok_or_else(|| { @@ -532,14 +532,12 @@ pub async fn exchange_token( http_client: &reqwest::Client, ) -> Result { let token_json = match grant_type { - "authorization_code" => { - client - .exchange_refresh_token(&RefreshToken::from(refresh_token)) - .with_client(http_client) - .execute::() - .await - .map_err(to_anyhow)? - } + "authorization_code" => client + .exchange_refresh_token(&RefreshToken::from(refresh_token)) + .with_client(http_client) + .execute::() + .await + .map_err(to_anyhow)?, "client_credentials" => { let mut token_request = client.exchange_client_credentials(); @@ -557,14 +555,12 @@ pub async fn exchange_token( .await .map_err(to_anyhow)? } - "" | _ if grant_type.is_empty() => { - client - .exchange_refresh_token(&RefreshToken::from(refresh_token)) - .with_client(http_client) - .execute::() - .await - .map_err(to_anyhow)? - } + "" | _ if grant_type.is_empty() => client + .exchange_refresh_token(&RefreshToken::from(refresh_token)) + .with_client(http_client) + .execute::() + .await + .map_err(to_anyhow)?, _ => { return Err(Error::BadRequest(format!( "Unsupported grant type: {}", @@ -829,11 +825,9 @@ pub async fn http_get_user_info( res.text().await.unwrap_or_default(), ))); } - Ok(res - .json::() - .await - .map_err(to_anyhow) - .map_err(|e| error::Error::InternalErr(format!("failed to decode json from user info: {}", e)))?) + Ok(res.json::().await.map_err(to_anyhow).map_err(|e| { + error::Error::InternalErr(format!("failed to decode json from user info: {}", e)) + })?) } /// GitHub email info response diff --git a/backend/windmill-store/Cargo.toml b/backend/windmill-store/Cargo.toml new file mode 100644 index 0000000000..3eddd95a39 --- /dev/null +++ b/backend/windmill-store/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "windmill-store" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_store" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] +private = ["windmill-common/private"] +no_auth = [] +oauth2 = ["dep:windmill-oauth"] +mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] +http_trigger = [] +postgres_trigger = [] +mqtt_trigger = [] +sqs_trigger = [] +gcp_trigger = [] +kafka = [] +nats = [] +openidconnect = ["windmill-common/openidconnect"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +windmill-mcp = { workspace = true, optional = true } +windmill-oauth = { workspace = true, optional = true } +axum.workspace = true +hyper.workspace = true +http.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +uuid.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true +sql-builder.workspace = true +async-recursion.workspace = true +futures.workspace = true +chrono.workspace = true +reqwest.workspace = true +anyhow.workspace = true diff --git a/backend/windmill-store/src/lib.rs b/backend/windmill-store/src/lib.rs new file mode 100644 index 0000000000..c7e606de43 --- /dev/null +++ b/backend/windmill-store/src/lib.rs @@ -0,0 +1,16 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(all(feature = "oauth2", feature = "private"))] +mod oauth_refresh_ee; +#[cfg(feature = "oauth2")] +pub mod oauth_refresh_oss; +pub mod resources; +pub mod secret_backend_ext; +pub mod var_resource_cache; +pub mod variables; diff --git a/backend/windmill-store/src/oauth_refresh_oss.rs b/backend/windmill-store/src/oauth_refresh_oss.rs new file mode 100644 index 0000000000..7402e66e49 --- /dev/null +++ b/backend/windmill-store/src/oauth_refresh_oss.rs @@ -0,0 +1,38 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +pub use crate::oauth_refresh_ee::_refresh_token; + +#[cfg(not(feature = "private"))] +use sqlx::{Postgres, Transaction}; +#[cfg(not(feature = "private"))] +use windmill_common::db::DB; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn _refresh_token<'c>( + tx: Transaction<'c, Postgres>, + path: &str, + w_id: &str, + id: i32, + db: &DB, +) -> error::Result { + windmill_oauth::refresh_token( + tx, + path, + w_id, + id, + db, + &*windmill_oauth::OAUTH_CLIENTS.read().await, + &windmill_oauth::OAUTH_HTTP_CLIENT, + include_str!("../../oauth_connect.json"), + ) + .await +} diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs new file mode 100644 index 0000000000..b5934d2a44 --- /dev/null +++ b/backend/windmill-store/src/resources.rs @@ -0,0 +1,1702 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use std::collections::HashMap; + +use windmill_api_auth::{ + check_scopes, maybe_refresh_folders, require_owner_of_path, require_super_admin, ApiAuthed, +}; +use windmill_common::db::DB; + +use crate::secret_backend_ext::rename_vault_secret; +use crate::var_resource_cache::{cache_resource, get_cached_resource}; +use windmill_common::utils::BulkDeleteRequest; +use windmill_common::webhook::{WebhookMessage, WebhookShared}; + +use axum::{ + body::Body, + extract::{Extension, Path, Query}, + response::Response, + routing::{delete, get, post}, + Json, Router, +}; +use futures::future::try_join_all; +use hyper::{header, StatusCode}; +use serde::{Deserialize, Serialize}; +use serde_json::{value::RawValue, Value}; +use sql_builder::{bind::Bind, quote, SqlBuilder}; +use sqlx::{Acquire, FromRow, Postgres, Transaction}; +use std::process::Stdio; +use tokio::process::Command; +use uuid::Uuid; +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; +use windmill_audit::ActionKind; +use windmill_common::{ + db::{DbWithOptAuthed, UserDB}, + error::{self, Error, JsonResult, Result}, + get_database_url, + utils::{ + get_custom_pg_instance_password, not_found_if_none, paginate, require_admin, Pagination, + StripPath, + }, + variables, + worker::{CLOUD_HOSTED, TMP_DIR}, + PgDatabase, +}; + +use async_recursion::async_recursion; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_resources)) + .route("/list_search", get(list_search_resources)) + .route("/list_names/:type", get(list_names)) + .route("/get/*path", get(get_resource)) + .route("/exists/*path", get(exists_resource)) + .route("/get_value/*path", get(get_resource_value)) + .route("/update/*path", post(update_resource)) + .route("/update_value/*path", post(update_resource_value)) + .route("/delete/*path", delete(delete_resource)) + .route("/delete_bulk", delete(delete_resources_bulk)) + .route("/create", post(create_resource)) + .route("/git_commit_hash/*path", get(get_git_commit_hash)) + .route("/type/list", get(list_resource_types)) + .route("/type/listnames", get(list_resource_types_names)) + .route("/type/get/:name", get(get_resource_type)) + .route("/type/exists/:name", get(exists_resource_type)) + .route("/type/update/:name", post(update_resource_type)) + .route("/type/delete/:name", delete(delete_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) + .route("/type/create", post(create_resource_type)) +} + +pub fn public_service() -> Router { + Router::new().route("/custom_component/:name", get(custom_component)) +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct ResourceType { + pub workspace_id: String, + pub name: String, + pub schema: Option, + pub description: Option, + pub created_by: Option, + pub edited_at: Option>, + pub format_extension: Option, +} + +#[derive(Deserialize)] +pub struct CreateResourceType { + pub name: String, + pub schema: Option, + pub description: Option, + pub format_extension: Option, +} + +#[derive(Deserialize)] +pub struct EditResourceType { + pub schema: Option, + pub description: Option, +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct Resource { + pub workspace_id: String, + pub path: String, + pub value: Option, + pub description: Option, + pub resource_type: String, + pub extra_perms: serde_json::Value, + pub created_by: Option, + pub edited_at: Option>, +} + +#[derive(FromRow, Serialize, Deserialize)] +pub struct ListableResource { + pub workspace_id: String, + pub path: String, + pub value: Option, + pub description: Option, + pub resource_type: String, + pub extra_perms: serde_json::Value, + pub created_by: Option, + pub edited_at: Option>, + pub is_linked: Option, + pub is_refreshed: Option, + pub is_oauth: Option, + pub is_expired: Option, + pub refresh_error: Option, + pub account: Option, +} + +#[derive(Deserialize)] +pub struct CreateResource { + pub path: String, + pub value: Option>, + pub description: Option, + pub resource_type: String, +} +#[derive(Deserialize)] +struct EditResource { + path: Option, + description: Option, + value: Option>, +} + +#[derive(Deserialize)] +pub struct ListResourceQuery { + resource_type: Option, + resource_type_exclude: Option, + path_start: Option, +} + +#[derive(Serialize, FromRow)] +pub struct NamePath { + name: String, + path: String, +} +async fn list_names( + authed: ApiAuthed, + Path((w_id, rt)): Path<(String, String)>, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + "SELECT value->>'name' as name, path from resource WHERE resource_type = $1 AND workspace_id = $2", + rt, + &w_id + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .filter_map(|x| x.name.map(|name| NamePath { name, path: x.path })) + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Serialize, FromRow)] +pub struct SearchResource { + path: String, + value: serde_json::Value, +} +async fn list_search_resources( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + #[cfg(feature = "enterprise")] + let n = 1000; + + #[cfg(not(feature = "enterprise"))] + let n = 3; + + let rows = sqlx::query_as!( + SearchResource, + "SELECT path, value from resource WHERE workspace_id = $1 LIMIT $2", + &w_id, + n + ) + .fetch_all(&mut *tx) + .await? + .into_iter() + .collect::>(); + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_resources( + authed: ApiAuthed, + Query(lq): Query, + Query(pagination): Query, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let mut sqlb = SqlBuilder::select_from("resource") + .fields(&[ + "resource.workspace_id", + "resource.path", + "null::JSONB as value", + "resource.description", + "resource_type", + "resource.extra_perms", + "(now() > account.expires_at) as is_expired", + "variable.path IS NOT NULL as is_linked", + "account.refresh_token != '' as is_refreshed", + "variable.is_oauth", + "variable.account", + "account.refresh_error", + "resource.created_by", + "resource.edited_at", + ]) + .left() + .join("variable") + .on("variable.path = resource.path AND variable.workspace_id = resource.workspace_id") + .left() + .join("account") + .on("variable.account = account.id AND account.workspace_id = variable.workspace_id") + .order_by("path", true) + .and_where("resource.workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + + if let Some(rt) = &lq.resource_type { + let resource_type_filters = rt.split(',').collect::>(); + if resource_type_filters.len() == 1 { + sqlb.and_where_eq("resource_type", "?".bind(rt)); + } else { + let mut list = Vec::new(); + for rt in resource_type_filters { + let quoted_value = quote(rt); + list.push(quoted_value); + } + sqlb.and_where_in("resource_type", list.as_slice()); + } + } + if let Some(rt) = &lq.resource_type_exclude { + for rt in rt.split(',') { + sqlb.and_where_ne("resource_type", "?".bind(&rt)); + } + } + + if let Some(path_start) = &lq.path_start { + sqlb.and_where_like_left("resource.path", path_start); + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as::<_, ListableResource>(&sql) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(rows)) +} + +async fn get_resource( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let resource_o = sqlx::query_as!( + ListableResource, + "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed, + account.refresh_error, + variable.path IS NOT NULL as is_linked, + variable.is_oauth as \"is_oauth?\", + variable.account + FROM resource + LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2 + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 + WHERE resource.path = $1 AND resource.workspace_id = $2", + path.to_owned(), + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if resource_o.is_none() { + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + } + let resource = not_found_if_none(resource_o, "Resource", path)?; + Ok(Json(resource)) +} + +async fn exists_resource( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn get_resource_value( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; + + let value_o = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path.to_owned(), + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + if value_o.is_none() { + explain_resource_perm_error(&path, &w_id, &db, &authed).await?; + } + + let value = not_found_if_none(value_o, "Resource", path)?; + + Ok(Json(value)) +} + +pub async fn explain_resource_perm_error( + path: &str, + w_id: &str, + db: &sqlx::Pool, + authed: &ApiAuthed, +) -> windmill_common::error::Result<()> { + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Resource {} not found", path)))?; + if path.starts_with("f/") { + let folder = path.split("/").nth(1).ok_or_else(|| { + Error::BadRequest(format!( + "path {} should have at least 2 components separated by /", + path + )) + })?; + let folder_extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(db) + .await?; + return Err(Error::NotAuthorized(format!( + "Resource exists but you don't have access to it:\nresource perms: {}\nfolder perms: {}\nauthed as: {authed:?}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() + ))); + } else { + return Err(Error::NotAuthorized(format!( + "Resource exists but you don't have access to it:\nresource perms: {}\nauthed as: {authed:?}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default() + ))); + } +} + +async fn custom_component( + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + let cc_o = sqlx::query_scalar!( + "SELECT value->>'js' FROM resource + WHERE path = $1 AND workspace_id = $2", + format!("f/app_custom/{name}"), + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + + let cc = not_found_if_none(cc_o, "Custom Component", name)?; + let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript"); + + Ok(res.body(Body::from(cc)).unwrap()) +} + +#[derive(Deserialize)] +pub struct JobInfo { + pub job_id: Option, + pub allow_cache: Option, +} + +pub async fn get_resource_value_interpolated_internal<'a>( + db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, + workspace: &str, + path: &str, + job_id: Option, + token_for_context: Option<&str>, + allow_cache: bool, +) -> Result> { + // This is a special syntax to help debugging custom instance databases + if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { + let db = db_with_opt_authed.db(); + require_super_admin(db_with_opt_authed.db(), &db_with_opt_authed.email()).await?; + let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + pg_creds.dbname = dbname.to_string(); + pg_creds.password = Some(get_custom_pg_instance_password(&db).await?); + pg_creds.user = Some("custom_instance_user".to_string()); + let pg_creds = serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?; + return Ok(Some(pg_creds)); + } + + if allow_cache { + if let Some(cached_value) = get_cached_resource(&workspace, &path) { + return Ok(Some(cached_value)); + } + } + use sqlx::Acquire; + let mut tx = db_with_opt_authed.begin().await?; + + let value_o = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path, + workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + if value_o.is_none() { + if let Some(authed) = db_with_opt_authed.authed() { + let db = db_with_opt_authed.db(); + explain_resource_perm_error(path, workspace, db, authed).await?; + } + } + + let value = not_found_if_none(value_o, "Resource", path)?; + if let Some(value) = value { + let r = transform_json_value( + &db_with_opt_authed, + workspace, + value, + &job_id, + token_for_context, + ) + .await?; + if allow_cache { + cache_resource(&workspace, &path, r.clone()); + } + Ok(Some(r)) + } else { + Ok(None) + } +} + +#[async_recursion] +pub async fn transform_json_value( + db_with_opt_authed: &DbWithOptAuthed, + workspace: &str, + v: Value, + job_id: &Option, + token: Option<&str>, +) -> Result { + match v { + Value::String(y) if y.starts_with("$var:") => { + let path = y.strip_prefix("$var:").unwrap(); + + let v = + crate::variables::get_value_internal(&db_with_opt_authed, workspace, path, false) + .await?; + Ok(Value::String(v)) + } + Value::String(y) if y.starts_with("$res:") => { + let path = y.strip_prefix("$res:").unwrap(); + if path.split("/").count() < 2 { + return Err(Error::internal_err(format!( + "Invalid resource path: {path}" + ))); + } + let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; + let v = sqlx::query_scalar!( + "SELECT value from resource WHERE path = $1 AND workspace_id = $2", + path, + &workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let v = not_found_if_none(v, "Resource", path)?; + if let Some(v) = v { + transform_json_value(db_with_opt_authed, workspace, v, job_id, token).await + } else { + Ok(Value::Null) + } + } + Value::String(y) if y.starts_with("$") && job_id.is_some() => { + let mut tx = db_with_opt_authed.begin().await?; + let job_id = job_id.unwrap(); + let job = sqlx::query!( + "SELECT + v2_job.permissioned_as_email, + v2_job.created_by, + v2_job.parent_job, + v2_job.permissioned_as, + v2_job.runnable_path, + CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.root_job, + v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\" + FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id + WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", + job_id, + workspace + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let job = not_found_if_none(job, "Job", job_id.to_string())?; + + let flow_path = if let Some(uuid) = job.parent_job { + let mut tx: Transaction<'_, Postgres> = db_with_opt_authed.begin().await?; + let p = sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid) + .fetch_optional(&mut *tx) + .await? + .flatten(); + tx.commit().await?; + p + } else { + None + }; + + let variables = variables::get_reserved_variables( + &db_with_opt_authed.db().into(), + workspace, + token.unwrap_or_else(|| "no_token_available"), + &job.permissioned_as_email, + &job.created_by, + &job_id.to_string(), + &job.permissioned_as, + job.runnable_path.clone(), + job.parent_job.map(|x| x.to_string()), + flow_path, + job.schedule_path.clone(), + job.flow_step_id.clone(), + job.flow_innermost_root_job.map(|x| x.to_string()), + job.root_job.map(|x| x.to_string()), + Some(job.scheduled_for.clone()), + None, + None, + ) + .await; + + let name = y.strip_prefix("$").unwrap(); + + let value = variables + .iter() + .find(|x| x.name == name) + .map(|x| x.value.clone()) + .unwrap_or_else(|| y); + Ok(serde_json::json!(value)) + } + Value::Object(mut m) => { + for (a, b) in m.clone().into_iter() { + let v = + transform_json_value(db_with_opt_authed, workspace, b, job_id, token).await?; + m.insert(a.clone(), v); + } + Ok(Value::Object(m)) + } + a @ _ => Ok(a), + } +} + +async fn check_path_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + path: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Resource {} already exists", + path + ))); + } + return Ok(()); +} + +#[derive(Deserialize)] +struct CreateResourceQuery { + update_if_exists: Option, +} +async fn create_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Query(q): Query, + Json(resource): Json, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("resources:write:{}", resource.path))?; + if *CLOUD_HOSTED { + let nb_resources = sqlx::query_scalar!( + "SELECT COUNT(*) FROM resource WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + if nb_resources.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of resources (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + let authed = maybe_refresh_folders(&resource.path, &w_id, authed, &db).await; + + let mut tx = user_db.begin(&authed).await?; + + let update_if_exists = q.update_if_exists.unwrap_or(false); + if !update_if_exists { + check_path_conflict(&mut tx, &w_id, &resource.path).await?; + } + + let res_value = resource.value.unwrap_or_default(); + let raw_json = sqlx::types::Json(res_value.as_ref()); + + if resource.path.starts_with("f/app_themes/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_custom/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_groups/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } + sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, now()) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now()", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.create", + ActionKind::Create, + &w_id, + Some(&resource.path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: resource.path.clone(), parent_path: None }, + Some(format!("Resource '{}' created", resource.path.clone())), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateResource { workspace: w_id, path: resource.path.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("resource {} created", resource.path), + )) +} + +async fn delete_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + + check_scopes(&authed, || format!("resources:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + let deleted_path = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + not_found_if_none(deleted_path, "Resource", &path)?; + sqlx::query!( + "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Resource '{}' deleted", path)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResource { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("resource {} deleted", path)) +} + +async fn delete_resources_bulk( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult> { + for path in &request.paths { + check_scopes(&authed, || format!("resources:write:{}", path))?; + } + + let mut tx = user_db.begin(&authed).await?; + + let deleted_paths = sqlx::query_scalar!( + "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", + &request.paths, + w_id + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "resources.delete_bulk", + ActionKind::Delete, + &w_id, + Some(&deleted_paths.join(", ")), + None, + ) + .await?; + + tx.commit().await?; + + try_join_all(deleted_paths.iter().map(|path| { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Resource '{}' deleted", path)), + true, + None, + ) + })) + .await?; + + for path in &deleted_paths { + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResource { workspace: w_id.clone(), path: path.to_owned() }, + ); + } + + Ok(Json(deleted_paths)) +} + +async fn update_resource( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + + let mut sqlb = SqlBuilder::update_table("resource"); + sqlb.and_where_eq("path", "?".bind(&path)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(npath) = &ns.path { + sqlb.set_str("path", npath); + } + if let Some(nvalue) = ns.value { + sqlb.set_str("value", nvalue.to_string()); + } + if let Some(ndesc) = ns.description { + sqlb.set_str("description", ndesc); + } + + sqlb.set_str("edited_at", "now()"); + + sqlb.returning("path"); + let authed = maybe_refresh_folders(path, &w_id, authed, &db).await; + + let mut tx = user_db.begin(&authed).await?; + + if let Some(npath) = ns.path.clone() { + if npath != path { + check_path_conflict(&mut tx, &w_id, &npath).await?; + + require_owner_of_path(&authed, path)?; + + // Handle Vault secret rename if the linked variable is a Vault-stored secret + let linked_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = linked_var { + if var.is_secret { + // Check if this is a Vault-stored secret and rename it + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlx::query!( + "UPDATE variable SET value = $1 WHERE path = $2 AND workspace_id = $3", + new_value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + + sqlx::query!( + "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3", + npath, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; + + let npath = not_found_if_none(npath_o, "Resource", path)?; + + audit_log( + &mut *tx, + &authed, + "resources.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + // Detect if this was a rename operation + let old_path_if_renamed = if npath != path { Some(path) } else { None }; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: npath.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Resource '{}' updated", npath)), + true, + old_path_if_renamed, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResource { + workspace: w_id, + old_path: path.to_owned(), + new_path: npath.clone(), + }, + ); + + Ok(format!("resource {} updated (npath: {:?})", path, npath)) +} + +#[derive(FromRow, Serialize, Deserialize)] +struct UpdateResource { + value: Option, +} + +async fn update_resource_value( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(nv): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("resources:write:{}", path))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "UPDATE resource SET value = $1, edited_at = now() WHERE path = $2 AND workspace_id = $3", + nv.value, + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "resources.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Resource { path: path.to_string(), parent_path: Some(path.to_string()) }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResource { + workspace: w_id, + old_path: path.to_owned(), + new_path: path.to_owned(), + }, + ); + + Ok(format!("value of resource {} updated", path)) +} + +async fn file_resource_ext_to_resource_type( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + #[derive(Serialize, sqlx::FromRow)] + struct LocalFileResourceExtension { + name: String, + format_extension: Option, + } + + let r = sqlx::query_as!(LocalFileResourceExtension, " + SELECT name, format_extension FROM resource_type WHERE format_extension IS NOT NULL AND (workspace_id = $1 OR workspace_id = 'admins')", w_id) + .fetch_all(&db) + .await?; + + let hashmap: HashMap = r + .into_iter() + .filter_map(|entry| { + if let Some(format_extension) = entry.format_extension { + Some((entry.name, format_extension)) + } else { + None + } + }) + .collect(); + + Ok(Json(hashmap)) +} + +async fn list_resource_types( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let rows = sqlx::query_as!( + ResourceType, + "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') ORDER \ + BY name", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + +async fn list_resource_types_names( + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let rows = sqlx::query_scalar!( + "SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'admins') \ + ORDER BY name", + &w_id + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + +async fn get_resource_type( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + let resource_type_o = sqlx::query_as!( + ResourceType, + "SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins')", + &name, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let resource_type = not_found_if_none(resource_type_o, "ResourceType", name)?; + Ok(Json(resource_type)) +} + +async fn exists_resource_type( + Extension(db): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'admins'))", + name, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn create_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(resource_type): Json, +) -> Result<(StatusCode, String)> { + let mut tx = user_db.begin(&authed).await?; + + check_rt_path_conflict(&mut tx, &w_id, &resource_type.name).await?; + + sqlx::query!( + "INSERT INTO resource_type + (workspace_id, name, schema, description, created_by, format_extension, edited_at) + VALUES ($1, $2, $3, $4, $5, $6, now())", + w_id, + resource_type.name, + resource_type.schema, + resource_type.description, + authed.username, + resource_type.format_extension, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "resource_types.create", + ActionKind::Create, + &w_id, + Some(&resource_type.name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: resource_type.name.clone() }, + Some(format!( + "Resource Type '{}' created", + resource_type.name.clone() + )), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateResourceType { name: resource_type.name.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("resource_type {} created", resource_type.name), + )) +} + +async fn check_rt_path_conflict<'c>( + tx: &mut Transaction<'c, Postgres>, + w_id: &str, + name: &str, +) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)", + name, + w_id + ) + .fetch_one(&mut **tx) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Resource type {} already exists", + name + ))); + } + return Ok(()); +} + +async fn delete_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + + let deleted_name = sqlx::query_scalar!( + "DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2 RETURNING name", + name, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + not_found_if_none(deleted_name, "ResourceType", &name)?; + + audit_log( + &mut *tx, + &authed, + "resource_types.delete", + ActionKind::Delete, + &w_id, + Some(&name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: name.clone() }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteResourceType { name: name.clone() }, + ); + + Ok(format!("resource_type {} deleted", name)) +} + +async fn update_resource_type( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, name)): Path<(String, String)>, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let mut sqlb = SqlBuilder::update_table("resource_type"); + sqlb.and_where_eq("name", "?".bind(&name)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + if let Some(nschema) = ns.schema { + sqlb.set_str("schema", nschema); + } + if let Some(ndesc) = ns.description { + sqlb.set_str("description", ndesc); + } + sqlb.set_str("edited_at", "now()"); + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + let mut tx = user_db.begin(&authed).await?; + + sqlx::query(&sql).execute(&mut *tx).await?; + audit_log( + &mut *tx, + &authed, + "resource_types.update", + ActionKind::Update, + &w_id, + Some(&name), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::ResourceType { path: name.clone() }, + None, + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateResourceType { name: name.clone() }, + ); + + Ok(format!("resource_type {} updated", name)) +} + +#[cfg(any( + feature = "http_trigger", + feature = "postgres_trigger", + feature = "mqtt_trigger", + all( + feature = "enterprise", + any( + feature = "sqs_trigger", + feature = "gcp_trigger", + feature = "kafka", + feature = "nats" + ) + ) +))] +pub async fn try_get_resource_from_db_as( + authed: &ApiAuthed, + user_db: Option, + db: &DB, + resource_path: &str, + w_id: &str, +) -> Result +where + T: serde::de::DeserializeOwned, +{ + let resource = get_resource_value_interpolated_internal( + &DbWithOptAuthed::from_authed(authed, db.clone(), user_db), + &w_id, + &resource_path, + None, + None, + false, + ) + .await?; + + let resource = match resource { + Some(resource) => serde_json::from_value::(resource) + .map_err(|e| Error::SerdeJson { error: e, location: "resources.rs".to_string() })?, + None => { + return { + Err(Error::NotFound(format!( + "resource at path :{} do not exist", + &resource_path + ))) + } + } + }; + + Ok(resource) +} + +#[derive(Deserialize, Serialize)] +struct GitRepositoryResource { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, +} + +/// Validates a git URL to prevent git option injection attacks. +/// Git URLs starting with '-' could be interpreted as command-line options. +fn validate_git_url(url: &str) -> Result<()> { + let url = url.trim(); + if url.is_empty() { + return Err(Error::BadRequest("Git URL cannot be empty".to_string())); + } + if url.starts_with('-') { + return Err(Error::BadRequest( + "Git URL cannot start with '-' (potential option injection)".to_string(), + )); + } + // Block other potentially dangerous patterns + if url.contains('\0') || url.contains('\n') || url.contains('\r') { + return Err(Error::BadRequest( + "Git URL contains invalid characters".to_string(), + )); + } + Ok(()) +} + +/// Validates a git branch/ref name to prevent injection attacks. +fn validate_git_ref(ref_name: &str) -> Result<()> { + let ref_name = ref_name.trim(); + if ref_name.is_empty() { + return Err(Error::BadRequest("Git ref cannot be empty".to_string())); + } + if ref_name.starts_with('-') { + return Err(Error::BadRequest( + "Git ref cannot start with '-' (potential option injection)".to_string(), + )); + } + // Git ref names have specific rules - block dangerous characters + if ref_name.contains('\0') + || ref_name.contains('\n') + || ref_name.contains('\r') + || ref_name.contains("..") + || ref_name.contains("@{") + || ref_name.ends_with('.') + || ref_name.ends_with('/') + || ref_name.contains("//") + { + return Err(Error::BadRequest( + "Git ref contains invalid characters or patterns".to_string(), + )); + } + Ok(()) +} + +#[derive(Serialize)] +struct GitCommitHashResponse { + commit_hash: String, +} + +#[derive(Deserialize)] +struct GitCommitHashQuery { + git_ssh_identity: Option, +} + +async fn get_git_commit_hash( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, +) -> JsonResult { + let path = path.to_path(); + + check_scopes(&authed, || format!("resources:read:{}", path))?; + + let db_with_opt_authed = + DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + let git_repo_resource_value = get_resource_value_interpolated_internal( + &db_with_opt_authed, + &w_id, + path, + None, + None, + false, + ) + .await + .map_err(|e| Error::NotFound(format!("Access to resource {} denied: ({e})", path)))?; + + let git_resource: GitRepositoryResource = match git_repo_resource_value { + Some(value) => serde_json::from_value(value).map_err(|e| { + Error::BadRequest(format!("Invalid git repository resource format: {}", e)) + })?, + None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), + }; + + let identities: Vec = query + .git_ssh_identity + .map(|s| { + s.split(",") + .filter_map(|s| { + if !s.is_empty() { + Some(s.to_string()) + } else { + None + } + }) + .collect() + }) + .unwrap_or(vec![]); + + let (git_ssh_cmd, filenames) = + get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; + + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + + delete_paths(&filenames).await; + + Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) +} + +async fn write_ssh_file( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + var_path: &str, +) -> std::result::Result { + let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); + let loc = std::path::Path::new(TMP_DIR) + .join("ssh_ids") + .join(id_file_name); + + let userdb_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + let mut content = crate::variables::get_value_internal(&userdb_authed, &w_id, &var_path, false) + .await + .map_err(|e| { + ( + error::Error::NotFound(format!( + "Variable {var_path} not found for git ssh identity: {e:#}" + )), + loc.clone(), + ) + })?; + content.push_str("\n"); + + if let Some(p) = &loc.parent() { + tokio::fs::create_dir_all(p) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + tokio::fs::write(&loc, content) + .await + .map_err(|e| (e.into(), loc.clone()))?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + tokio::fs::set_permissions(&loc, perm) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + + return Ok(loc); +} + +async fn delete_paths(paths: &Vec) { + for path in paths { + let _ = tokio::fs::remove_file(&path).await; + } +} + +async fn get_git_ssh_cmd( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + git_ssh_identity: Vec, +) -> error::Result<(Option, Vec)> { + if git_ssh_identity.len() > 5 { + return Err(error::Error::BadRequest( + "Too many ssh identities, try using at most 1".to_string(), + )); + } + if git_ssh_identity.len() == 0 { + return Ok((None, vec![])); + } + + let mut ssh_id_files = vec![]; + let mut file_paths = vec![]; + for var_path in git_ssh_identity.iter() { + match write_ssh_file(authed, user_db, db, w_id, &var_path).await { + Ok(loc) => { + ssh_id_files.push(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )); + file_paths.push(loc); + } + Err((e, loc)) => { + file_paths.push(loc); + delete_paths(&file_paths).await; + return Err(e); + } + } + } + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok((Some(git_ssh_cmd), file_paths)) +} + +async fn get_repo_latest_commit_hash( + git_resource: &GitRepositoryResource, + git_ssh_command: Option, +) -> Result { + // Validate URL and branch to prevent option injection attacks + validate_git_url(&git_resource.url)?; + + let ref_spec = git_resource + .branch + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or("HEAD"); + + // Validate ref_spec if it's not the default HEAD + if ref_spec != "HEAD" { + validate_git_ref(ref_spec)?; + } + + let mut git_cmd = Command::new("git"); + git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); + if let Some(git_ssh_command) = git_ssh_command { + git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); + } + git_cmd.stderr(Stdio::piped()); + + let output = git_cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute git command: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8(output.stderr) + .unwrap_or_else(|_| "Failed to decode stderr".to_string()); + return Err(Error::BadRequest(format!( + "Error getting git repo commit hash: {}", + stderr + ))); + } + + let stdout = String::from_utf8(output.stdout) + .map_err(|e| Error::internal_err(format!("Failed to decode git output: {}", e)))?; + + let lines: Vec<&str> = stdout.lines().collect(); + + if lines.is_empty() { + return Err(Error::BadRequest(format!( + "No commits found for reference '{}' in repository '{}'", + ref_spec, git_resource.url + ))); + } + + let commit_hash = lines + .first() + .and_then(|line| line.split_whitespace().next()) + .map(|s| s.to_string()) + .ok_or_else(|| { + Error::BadRequest("Unexpected output format for git ls-remote".to_string()) + })?; + + Ok(commit_hash) +} + +#[cfg(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger") +))] +pub async fn interpolate( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + s: String, +) -> std::result::Result { + use serde_json::Value; + use windmill_common::db::DbWithOptAuthed; + let value = Value::String(s); + match transform_json_value( + &DbWithOptAuthed::from_authed(authed, db.clone(), None), + w_id, + value, + &None, + None, + ) + .await? + { + Value::String(s) => Ok(s), + v => Err(anyhow::anyhow!("Expected string, got {:?}", v)), + } +} diff --git a/backend/windmill-store/src/secret_backend_ext.rs b/backend/windmill-store/src/secret_backend_ext.rs new file mode 100644 index 0000000000..5fdc4fdb38 --- /dev/null +++ b/backend/windmill-store/src/secret_backend_ext.rs @@ -0,0 +1,371 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Secret backend extension for the API layer +//! +//! This module provides helper functions for integrating the SecretBackend +//! trait with variable operations in the API. +//! +//! Note: HashiCorp Vault integration requires Enterprise Edition. +//! The OSS version only supports the database backend. + +use std::sync::Arc; + +use windmill_common::{ + db::DB, + error::{Error, Result}, + secret_backend::{database::DatabaseBackend, SecretBackend}, + variables::{build_crypt, decrypt, encrypt}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use windmill_common::{ + global_settings::{load_value_from_global_settings, SECRET_BACKEND_SETTING}, + secret_backend::{SecretBackendConfig, VaultBackend, VaultSettings}, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +use tokio::sync::RwLock; + +// Cached Vault backend to avoid recreating it for every request +// This enables connection pooling and avoids repeated setup overhead +#[cfg(all(feature = "private", feature = "enterprise"))] +struct CachedVaultBackend { + backend: Arc, + settings: VaultSettings, +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +lazy_static::lazy_static! { + static ref VAULT_BACKEND_CACHE: RwLock> = RwLock::new(None); +} + +/// Get the current secret backend based on global settings +/// +/// OSS: Always returns DatabaseBackend +/// EE: Returns configured backend (Database or Vault) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn get_secret_backend(db: &DB) -> Result> { + Ok(Arc::new(DatabaseBackend::new(db.clone()))) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn get_secret_backend(db: &DB) -> Result> { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + match config { + SecretBackendConfig::Database => Ok(Arc::new(DatabaseBackend::new(db.clone()))), + SecretBackendConfig::HashiCorpVault(settings) => { + get_or_create_vault_backend(db, settings).await + } + } +} + +/// Get a cached Vault backend or create a new one if settings changed +#[cfg(all(feature = "private", feature = "enterprise"))] +async fn get_or_create_vault_backend( + _db: &DB, + settings: VaultSettings, +) -> Result> { + // Check if we have a cached backend with matching settings (read lock) + { + let cache = VAULT_BACKEND_CACHE.read().await; + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + } + + // Need to create a new backend - acquire write lock + let mut cache = VAULT_BACKEND_CACHE.write().await; + + // Double-check (another task may have created it while we waited) + if let Some(ref cached) = *cache { + if cached.settings == settings { + return Ok(cached.backend.clone()); + } + } + + // Create new backend + let backend: Arc = { + #[cfg(feature = "openidconnect")] + if settings.token.is_none() { + Arc::new(VaultBackend::new_with_db(settings.clone(), _db.clone())) + } else { + Arc::new(VaultBackend::new(settings.clone())) + } + + #[cfg(not(feature = "openidconnect"))] + Arc::new(VaultBackend::new(settings.clone())) + }; + + // Cache it + *cache = Some(CachedVaultBackend { backend: backend.clone(), settings }); + + Ok(backend) +} + +/// Check if a Vault backend is currently configured +/// +/// OSS: Always returns false +/// EE: Checks global settings +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn is_vault_backend_configured(_db: &DB) -> Result { + Ok(false) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn is_vault_backend_configured(db: &DB) -> Result { + let config = match load_value_from_global_settings(db, SECRET_BACKEND_SETTING).await? { + Some(value) => serde_json::from_value::(value).unwrap_or_default(), + None => SecretBackendConfig::default(), + }; + + Ok(matches!(config, SecretBackendConfig::HashiCorpVault(_))) +} + +/// Get a secret value using the configured backend +/// +/// For database backend: decrypts using workspace key +/// For vault backend (EE only): fetches from Vault directly +pub async fn get_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + encrypted_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database decryption + let mc = build_crypt(db, workspace_id).await?; + decrypt(&mc, encrypted_value.to_string()).map_err(|e| { + Error::internal_err(format!("Error decrypting variable {}: {}", path, e)) + }) + } + "hashicorp_vault" => { + // Fetch from Vault directly + backend.get_secret(workspace_id, path).await + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Store a secret value using the configured backend +/// +/// For database backend: encrypts using workspace key and returns encrypted value +/// For vault backend (EE only): stores in Vault and returns a placeholder for DB storage +pub async fn store_secret_value( + db: &DB, + workspace_id: &str, + path: &str, + plain_value: &str, +) -> Result { + let backend = get_secret_backend(db).await?; + + match backend.backend_name() { + "database" => { + // Use existing database encryption + let mc = build_crypt(db, workspace_id).await?; + Ok(encrypt(&mc, plain_value)) + } + "hashicorp_vault" => { + // Store in Vault and return a marker for DB + backend.set_secret(workspace_id, path, plain_value).await?; + Ok(format!("$vault:{}", path)) + } + _ => Err(Error::internal_err(format!( + "Unknown backend: {}", + backend.backend_name() + ))), + } +} + +/// Delete a secret from the configured backend (if using Vault) +/// +/// For database backend: no-op (DB delete is handled separately) +/// For vault backend (EE only): deletes from Vault +pub async fn delete_secret_from_backend(db: &DB, workspace_id: &str, path: &str) -> Result<()> { + if is_vault_backend_configured(db).await? { + let backend = get_secret_backend(db).await?; + // Ignore NotFound errors during deletion (secret might not exist in Vault) + match backend.delete_secret(workspace_id, path).await { + Ok(()) => Ok(()), + Err(Error::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } else { + Ok(()) + } +} + +/// Check if a value is stored in Vault (indicated by the $vault: prefix) +pub fn is_vault_stored_value(value: &str) -> bool { + value.starts_with("$vault:") +} + +/// Rename a secret in Vault when a variable path changes (EE only) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secret( + _db: &DB, + _workspace_id: &str, + _old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + if is_vault_stored_value(current_value) { + tracing::warn!( + "Variable has $vault: prefix but Vault requires Enterprise Edition. \ + Updating DB reference to {}", + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Ok(None) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secret( + db: &DB, + workspace_id: &str, + old_path: &str, + new_path: &str, + current_value: &str, +) -> Result> { + if !is_vault_stored_value(current_value) { + return Ok(None); + } + + if !is_vault_backend_configured(db).await? { + tracing::warn!( + "Variable value has $vault: prefix but Vault is not configured. \ + Updating DB reference from {} to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + + let backend = get_secret_backend(db).await?; + + let secret_value = match backend.get_secret(workspace_id, old_path).await { + Ok(value) => value, + Err(Error::NotFound(_)) => { + tracing::warn!( + "Secret not found in Vault at path {} during rename to {}", + old_path, + new_path + ); + return Ok(Some(format!("$vault:{}", new_path))); + } + Err(e) => return Err(e), + }; + + backend + .set_secret(workspace_id, new_path, &secret_value) + .await?; + + if let Err(e) = backend.delete_secret(workspace_id, old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename to {}: {}", + old_path, + new_path, + e + ); + } + + Ok(Some(format!("$vault:{}", new_path))) +} + +/// Bulk rename secrets in Vault when a path prefix changes (e.g., user rename) +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub async fn rename_vault_secrets_with_prefix( + _db: &DB, + _workspace_id: &str, + _old_prefix: &str, + _new_prefix: &str, + _variables: Vec<(String, String)>, +) -> Result> { + Ok(vec![]) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub async fn rename_vault_secrets_with_prefix( + db: &DB, + workspace_id: &str, + old_prefix: &str, + new_prefix: &str, + variables: Vec<(String, String)>, +) -> Result> { + if !is_vault_backend_configured(db).await? { + return Ok(vec![]); + } + + let backend = get_secret_backend(db).await?; + let mut updates = Vec::new(); + + for (old_path, value) in variables { + if !is_vault_stored_value(&value) { + continue; + } + + let new_path = if old_path.starts_with(old_prefix) { + format!("{}{}", new_prefix, &old_path[old_prefix.len()..]) + } else { + continue; + }; + + let secret_value = match backend.get_secret(workspace_id, &old_path).await { + Ok(v) => v, + Err(Error::NotFound(_)) => { + updates.push((old_path, format!("$vault:{}", new_path))); + continue; + } + Err(e) => { + tracing::error!( + "Failed to read secret at {} during bulk rename: {}", + old_path, + e + ); + continue; + } + }; + + if let Err(e) = backend + .set_secret(workspace_id, &new_path, &secret_value) + .await + { + tracing::error!( + "Failed to write secret to {} during bulk rename: {}", + new_path, + e + ); + continue; + } + + if let Err(e) = backend.delete_secret(workspace_id, &old_path).await { + tracing::warn!( + "Failed to delete old secret at {} after rename: {}", + old_path, + e + ); + } + + updates.push((old_path, format!("$vault:{}", new_path))); + } + + Ok(updates) +} diff --git a/backend/windmill-api/src/var_resource_cache.rs b/backend/windmill-store/src/var_resource_cache.rs similarity index 96% rename from backend/windmill-api/src/var_resource_cache.rs rename to backend/windmill-store/src/var_resource_cache.rs index d97416b871..f7ce2aeecf 100644 --- a/backend/windmill-api/src/var_resource_cache.rs +++ b/backend/windmill-store/src/var_resource_cache.rs @@ -40,22 +40,20 @@ impl CacheEntry { } } - lazy_static::lazy_static! { /// Cache for individual variable values: key = "workspace_id:path" pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); - - /// Cache for resource values: key = "workspace_id:path" + + /// Cache for resource values: key = "workspace_id:path" pub static ref RESOURCE_CACHE: Cache> = Cache::new(1000); } - /// Generate cache key for variables and resources pub fn cache_key(workspace_id: &str, path: &str) -> String { format!("{}:{}", workspace_id, path) } -/// Get cached variable if available and not expired +/// Get cached variable if available and not expired pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { let key = cache_key(workspace_id, path); VARIABLE_CACHE.get(&key).and_then(|entry| { @@ -119,4 +117,4 @@ pub fn clear_all_caches() { VARIABLE_CACHE.clear(); RESOURCE_CACHE.clear(); tracing::debug!("All variable/resource caches cleared"); -} \ No newline at end of file +} diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs new file mode 100644 index 0000000000..9c3dd652db --- /dev/null +++ b/backend/windmill-store/src/variables.rs @@ -0,0 +1,926 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::{check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed}; +use windmill_common::db::DB; + +use crate::secret_backend_ext::{ + delete_secret_from_backend, get_secret_value, is_vault_stored_value, rename_vault_secret, + store_secret_value, +}; +use windmill_common::utils::BulkDeleteRequest; +use windmill_common::webhook::{WebhookMessage, WebhookShared}; + +use axum::{ + extract::{Extension, Path, Query}, + routing::{delete, get, post}, + Json, Router, +}; +use futures::future::try_join_all; +use hyper::StatusCode; +use serde_json::Value; + +use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; +use windmill_audit::ActionKind; +use windmill_common::{ + db::{DbWithOptAuthed, UserDB}, + error::{Error, JsonResult, Result}, + scripts::ScriptHash, + utils::{not_found_if_none, paginate, Pagination, StripPath, WarnAfterExt}, + variables::{ + build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, + }, + worker::CLOUD_HOSTED, +}; + +use crate::var_resource_cache::{cache_variable, get_cached_variable}; +use lazy_static::lazy_static; +use serde::Deserialize; +use sqlx::{Acquire, Postgres, Transaction}; +use windmill_common::variables::encrypt; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; + +lazy_static! { + pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_variables)) + .route("/list_contextual", get(list_contextual_variables)) + .route("/get/*path", get(get_variable)) + .route("/get_value/*path", get(get_value)) + .route("/exists/*path", get(exists_variable)) + .route("/update/*path", post(update_variable)) + .route("/delete/*path", delete(delete_variable)) + .route("/delete_bulk", delete(delete_variables_bulk)) + .route("/create", post(create_variable)) + .route("/encrypt", post(encrypt_value)) +} + +async fn list_contextual_variables( + Path(w_id): Path, + ApiAuthed { username, email, .. }: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + Ok(Json( + get_reserved_variables( + &db.into(), + &w_id, + "q1A0qcPuO00yxioll7iph76N9CJDqn", + &email, + &username, + "017e0ad5-f499-73b6-5488-92a61c5196dd", + format!("u/{username}").as_str(), + Some("u/user/script_path".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some("u/user/encapsulating_flow_path".to_string()), + Some("u/user/triggering_flow_path".to_string()), + Some("c".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some(chrono::offset::Utc::now()), + Some(ScriptHash(1234567890)), + None, + ) + .await + .to_vec(), + )) +} + +#[derive(Deserialize)] +struct ListVariableQuery { + path_start: Option, +} + +async fn list_variables( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lq): Query, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query_as::<_, ListableVariable>( + "SELECT variable.workspace_id, variable.path, CASE WHEN is_secret IS TRUE THEN null ELSE variable.value::text END as value, + is_secret, variable.description, variable.extra_perms, account, is_oauth, (now() > account.expires_at) as is_expired, + account.refresh_error, + resource.path IS NOT NULL as is_linked, + account.refresh_token != '' as is_refreshed, + variable.expires_at + from variable + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1 + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1 + WHERE variable.workspace_id = $1 AND variable.path NOT LIKE 'u/' || $2 || '/secret_arg/%' + AND variable.path LIKE $3 || '%' + ORDER BY path + LIMIT $4 OFFSET $5 +", + ) + .bind(&w_id) + .bind(&authed.username) + .bind(&lq.path_start.unwrap_or_default()) + .bind(per_page as i32) + .bind(offset as i32) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct GetVariableQuery { + decrypt_secret: Option, + include_encrypted: Option, +} + +async fn get_variable( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Query(q): Query, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; + + let variable_o = sqlx::query_as::<_, ListableVariable>( + "SELECT variable.*, (now() > account.expires_at) as is_expired, account.refresh_error, + resource.path IS NOT NULL as is_linked, + account.refresh_token != '' as is_refreshed + from variable + LEFT JOIN account ON variable.account = account.id + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2 + WHERE variable.path = $1 AND variable.workspace_id = $2 + LIMIT 1", + ) + .bind(&path) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + let variable = if let Some(variable) = variable_o { + variable + } else { + explain_variable_perm_error(&path, &w_id, &db).await?; + unreachable!() + }; + + let decrypt_secret = q.decrypt_secret.unwrap_or(true); + + let r = if variable.is_secret { + if decrypt_secret { + audit_log( + &mut *tx, + &authed, + "variables.decrypt_secret", + ActionKind::Execute, + &w_id, + Some(&variable.path), + None, + ) + .await?; + } + + let value = variable.value.unwrap_or_else(|| "".to_string()); + ListableVariable { + value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() { + #[cfg(feature = "oauth2")] + { + let refresh_tx = db + .begin() + .await + .map_err(|e| Error::InternalErr(e.to_string()))?; + Some( + crate::oauth_refresh_oss::_refresh_token( + refresh_tx, + &variable.path, + &w_id, + variable.account.unwrap(), + &db, + ) + .await?, + ) + } + #[cfg(not(feature = "oauth2"))] + return Err(Error::internal_err("Require oauth2 feature".to_string())); + } else if !value.is_empty() && decrypt_secret { + let _ = tx.commit().await; + // Use secret backend for decryption (supports both DB and Vault) + Some(get_secret_value(&db, &w_id, &variable.path, &value).await?) + } else if q.include_encrypted.unwrap_or(false) { + Some(value) + } else { + None + }, + ..variable + } + } else { + variable + }; + + Ok(Json(r)) +} + +#[derive(Deserialize)] +struct GetValueQuery { + allow_cache: Option, +} +async fn get_value( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(q): Query, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || format!("variables:read:{}", path))?; + let userdb_authed = DbWithOptAuthed::from_authed(&authed, db.clone(), Some(user_db.clone())); + + return get_value_internal(&userdb_authed, &w_id, &path, q.allow_cache.unwrap_or(false)) + .warn_after_seconds(10) + .await + .map(Json); +} + +async fn explain_variable_perm_error( + path: &str, + w_id: &str, + db: &sqlx::Pool, +) -> windmill_common::error::Result<()> { + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Variable {} not found", path)))?; + if path.starts_with("f/") { + let folder = path.split("/").nth(1).ok_or_else(|| { + Error::BadRequest(format!( + "path {} should have at least 2 components separated by /", + path + )) + })?; + let folder_extra_perms = sqlx::query_scalar!( + "SELECT extra_perms from folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(db) + .await?; + return Err(Error::NotAuthorized(format!( + "Variable exists but you don't have access to it:\nvariable perms: {}\nfolder perms: {}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default(), serde_json::to_string_pretty(&folder_extra_perms).unwrap_or_default() + ))); + } else { + return Err(Error::NotAuthorized(format!( + "Variable exists but you don't have access to it:\nvariable perms: {}", + serde_json::to_string_pretty(&extra_perms).unwrap_or_default() + ))); + } +} + +async fn exists_variable( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)", + path, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(false); + if exists { + return Err(Error::BadRequest(format!( + "Variable {} already exists", + path + ))); + } + return Ok(()); +} + +async fn create_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Query(AlreadyEncrypted { already_encrypted }): Query, + Json(variable): Json, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || format!("variables:write:{}", variable.path))?; + if *CLOUD_HOSTED { + let nb_variables = sqlx::query_scalar!( + "SELECT COUNT(*) FROM variable WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + if nb_variables.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of variables (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + let authed = maybe_refresh_folders(&variable.path, &w_id, authed, &db).await; + + check_path_conflict(&db, &w_id, &variable.path).await?; + let value = if variable.is_secret && !already_encrypted.unwrap_or(false) { + // Use secret backend for encryption (supports both DB and Vault) + store_secret_value(&db, &w_id, &variable.path, &variable.value).await? + } else { + variable.value + }; + + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "INSERT INTO variable + (workspace_id, path, value, is_secret, description, account, is_oauth, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + &w_id, + variable.path, + value, + variable.is_secret, + variable.description, + variable.account, + variable.is_oauth.unwrap_or(false), + variable.expires_at + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "variables.create", + ActionKind::Create, + &w_id, + Some(&variable.path), + None, + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: variable.path.clone(), parent_path: None }, + Some(format!("Variable '{}' created", variable.path.clone())), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::CreateVariable { workspace: w_id, path: variable.path.clone() }, + ); + + Ok(( + StatusCode::CREATED, + format!("variable {} created", variable.path), + )) +} + +async fn encrypt_value( + Extension(db): Extension, + Path(w_id): Path, + Json(variable): Json, +) -> Result { + let mc = build_crypt(&db, &w_id).await?; + let value = encrypt(&mc, &variable); + + Ok(value) +} + +async fn delete_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + + check_scopes(&authed, || format!("variables:write:{}", path))?; + + // Check if variable is a secret before deleting (for Vault cleanup) + let is_secret = sqlx::query_scalar!( + "SELECT is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + "DELETE FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "variables.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + // If variable was a secret, also delete from Vault backend (if configured) + if is_secret { + delete_secret_from_backend(&db, &w_id, path).await?; + } + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: path.to_string(), parent_path: Some(path.to_string()) }, + Some(format!("Variable '{}' deleted", path)), + true, + None, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteVariable { workspace: w_id, path: path.to_owned() }, + ); + + Ok(format!("variable {} deleted", path)) +} + +async fn delete_variables_bulk( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path(w_id): Path, + Json(request): Json, +) -> JsonResult> { + for path in &request.paths { + check_scopes(&authed, || format!("variables:write:{}", path))?; + } + + // Query which paths are secrets before deletion (for Vault cleanup) + let secret_paths: Vec = sqlx::query_scalar!( + "SELECT path FROM variable WHERE path = ANY($1) AND workspace_id = $2 AND is_secret = true", + &request.paths, + &w_id + ) + .fetch_all(&db) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let deleted_paths = sqlx::query_scalar!( + "DELETE FROM variable WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", + &request.paths, + w_id + ) + .fetch_all(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2", + &deleted_paths, + w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "variables.delete_bulk", + ActionKind::Delete, + &w_id, + Some(&deleted_paths.join(", ")), + None, + ) + .await?; + + tx.commit().await?; + + // Delete secrets from Vault backend (if configured) + for path in &secret_paths { + if deleted_paths.contains(path) { + delete_secret_from_backend(&db, &w_id, path).await?; + } + } + + try_join_all(deleted_paths.iter().map(|path| { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { + path: path.to_string(), + parent_path: Some(path.to_string()), + }, + Some(format!("Variable '{}' deleted", path)), + true, + None, + ) + })) + .await?; + + for path in &deleted_paths { + webhook.send_message( + w_id.clone(), + WebhookMessage::DeleteVariable { workspace: w_id.clone(), path: path.to_owned() }, + ); + } + + Ok(Json(deleted_paths)) +} + +#[derive(Deserialize)] +struct EditVariable { + path: Option, + value: Option, + is_secret: Option, + description: Option, + account: Option, +} + +#[derive(Deserialize)] +struct AlreadyEncrypted { + already_encrypted: Option, +} + +async fn update_variable( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Extension(webhook): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(AlreadyEncrypted { already_encrypted }): Query, + Json(ns): Json, +) -> Result { + use sql_builder::prelude::*; + + let path = path.to_path(); + check_scopes(&authed, || format!("variables:write:{}", path))?; + let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; + + let mut sqlb = SqlBuilder::update_table("variable"); + sqlb.and_where_eq("path", "?".bind(&path)); + sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); + + if let Some(npath) = &ns.path { + sqlb.set_str("path", npath); + } + let ns_value_is_none = ns.value.is_none(); + // Determine the target path for storing secrets (use new path if provided) + let target_path = ns.path.as_deref().unwrap_or(path); + if let Some(nvalue) = ns.value.clone() { + let is_secret = if ns.is_secret.is_some() { + ns.is_secret.unwrap() + } else { + sqlx::query_scalar!( + "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false) + }; + + let value = if is_secret && !already_encrypted.unwrap_or(false) { + // Use secret backend for encryption (supports both DB and Vault) + // Store at target_path (new path if renaming, otherwise current path) + store_secret_value(&db, &w_id, target_path, &nvalue).await? + } else { + nvalue + }; + sqlb.set_str("value", &value); + } + + if let Some(desc) = ns.description { + sqlb.set_str("description", &desc); + } + + if let Some(account_id) = ns.account { + sqlb.set_str("account", account_id); + } + + if let Some(nbool) = ns.is_secret { + let old_secret = sqlx::query_scalar!( + "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if old_secret != nbool && ns_value_is_none { + return Err(Error::BadRequest( + "cannot change is_secret without updating value too".to_string(), + )); + } + sqlb.set_str("is_secret", nbool); + } + sqlb.returning("path"); + + // Get old account_id if we're updating the account field + let old_account_id = if ns.account.is_some() { + sqlx::query_scalar!( + "SELECT account FROM variable WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + } else { + None + }; + + let mut tx: Transaction<'_, Postgres> = user_db.begin(&authed).await?; + + if let Some(npath) = ns.path.clone() { + if npath != path { + check_path_conflict(&db, &w_id, &npath).await?; + require_owner_of_path(&authed, path)?; + + // Handle Vault secret rename if the variable is a secret stored in Vault + let current_var = sqlx::query!( + "SELECT value, is_secret FROM variable WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(var) = current_var { + if var.is_secret && is_vault_stored_value(&var.value) { + if ns.value.is_some() { + // New value was provided and already stored at new path + // Just delete the old secret from Vault + delete_secret_from_backend(&db, &w_id, path).await?; + } else { + // No new value - rename the secret in Vault + if let Some(new_value) = + rename_vault_secret(&db, &w_id, path, &npath, &var.value).await? + { + // Update the variable's value to point to the new Vault path + sqlb.set_str("value", &new_value); + } + } + } + } + + let mut v = sqlx::query_scalar!( + "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + if let Some(old_v) = v { + v = Some(replace_path( + old_v, + &format!("$var:{path}"), + &format!("$var:{npath}"), + )) + } + + sqlx::query!( + "UPDATE resource SET path = $1, value = $2, edited_at = now() WHERE path = $3 AND workspace_id = $4", + npath, + v, + path, + w_id + ) + .execute(&mut *tx) + .await?; + } + } + + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; + + let npath_o: Option = sqlx::query_scalar(&sql).fetch_optional(&mut *tx).await?; + + let npath = not_found_if_none(npath_o, "Variable", path)?; + + audit_log( + &mut *tx, + &authed, + "variables.update", + ActionKind::Update, + &w_id, + Some(path), + None, + ) + .await?; + + // Clean up old account if it's no longer referenced and different from new account + if let Some(old_acc_id) = old_account_id { + if ns.account.is_some() && ns.account != Some(old_acc_id) { + // Check if old account is still referenced by other variables or resources + let account_still_used = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM variable WHERE account = $1 AND workspace_id = $2)", + old_acc_id, + &w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(true); + + if !account_still_used { + // Delete the orphaned account + sqlx::query!( + "DELETE FROM account WHERE id = $1 AND workspace_id = $2", + old_acc_id, + &w_id + ) + .execute(&mut *tx) + .await?; + } + } + } + + tx.commit().await?; + + // Detect if this was a rename operation + let old_path_if_renamed = if npath != path { Some(path) } else { None }; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Variable { path: npath.clone(), parent_path: Some(path.to_string()) }, + None, + true, + old_path_if_renamed, + ) + .await?; + + webhook.send_message( + w_id.clone(), + WebhookMessage::UpdateVariable { + workspace: w_id, + old_path: path.to_owned(), + new_path: npath.clone(), + }, + ); + + Ok(format!("variable {} updated (npath: {:?})", path, npath)) +} + +fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { + match v { + Value::Object(v) => Value::Object( + v.into_iter() + .map(|(k, v)| (k, replace_path(v, path, npath))) + .collect(), + ), + Value::Array(arr) => Value::Array( + arr.into_iter() + .map(|v| replace_path(v, path, npath)) + .collect(), + ), + Value::String(s) if s == path => Value::String(npath.to_owned()), + _ => v, + } +} + +pub async fn get_value_internal<'a>( + db_with_opt_authed: &'a DbWithOptAuthed<'a, ApiAuthed>, + w_id: &str, + path: &str, + allow_cache: bool, +) -> Result { + if allow_cache { + if let Some(cached_variable) = get_cached_variable(&w_id, &path) { + return Ok(cached_variable); + } + } + + let mut tx = db_with_opt_authed.begin().await?; + let variable_o = sqlx::query!( + "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable + LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id + ) + .fetch_optional(&mut *tx) + .warn_after_seconds(5) + .await?; + drop(tx); + + let variable = if let Some(variable) = variable_o { + variable + } else { + explain_variable_perm_error(path, w_id, &db_with_opt_authed.db()).await?; + unreachable!() + }; + + let r = if variable.is_secret { + // let audit_author = + let mut tx = db_with_opt_authed.db().begin().await?; + audit_log( + &mut *tx, + db_with_opt_authed, + "variables.decrypt_secret", + ActionKind::Execute, + &w_id, + Some(&variable.path), + None, + ) + .await?; + tx.commit().await?; + + let value = variable.value; + if variable.is_expired.unwrap_or(false) && variable.account.is_some() { + #[cfg(feature = "oauth2")] + { + let db = db_with_opt_authed.db(); + let refresh_tx = db + .begin() + .await + .map_err(|e| Error::InternalErr(e.to_string()))?; + crate::oauth_refresh_oss::_refresh_token( + refresh_tx, + &variable.path, + &w_id, + variable.account.unwrap(), + db, + ) + .await? + } + #[cfg(not(feature = "oauth2"))] + return Err(Error::internal_err("Require oauth2 feature".to_string())); + } else if !value.is_empty() { + // Use secret backend for decryption (supports both DB and Vault) + get_secret_value(db_with_opt_authed.db(), &w_id, &variable.path, &value).await? + } else { + "".to_string() + } + } else { + variable.value + }; + + // Cache the result when explicitly allowed and caching appropriate + if allow_cache { + cache_variable(&w_id, &path, db_with_opt_authed.email(), r.clone()); + } + + Ok(r) +} diff --git a/backend/windmill-trigger-email/Cargo.toml b/backend/windmill-trigger-email/Cargo.toml new file mode 100644 index 0000000000..0ed00fa2dd --- /dev/null +++ b/backend/windmill-trigger-email/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "windmill-trigger-email" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_email" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tracing.workspace = true +async-trait.workspace = true +anyhow.workspace = true +base64.workspace = true +regex.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api/src/triggers/email/handler_oss.rs b/backend/windmill-trigger-email/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/email/handler_oss.rs rename to backend/windmill-trigger-email/src/handler_oss.rs index 9a688c2cf1..c2e450d052 100644 --- a/backend/windmill-api/src/triggers/email/handler_oss.rs +++ b/backend/windmill-trigger-email/src/handler_oss.rs @@ -1,5 +1,5 @@ #[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; +use windmill_trigger::TriggerData; #[allow(unused)] #[cfg(feature = "private")] @@ -8,14 +8,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::EmailTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-email/src/lib.rs b/backend/windmill-trigger-email/src/lib.rs new file mode 100644 index 0000000000..836af77fa0 --- /dev/null +++ b/backend/windmill-trigger-email/src/lib.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct EmailTrigger; diff --git a/backend/windmill-trigger-gcp/Cargo.toml b/backend/windmill-trigger-gcp/Cargo.toml new file mode 100644 index 0000000000..1b106171c6 --- /dev/null +++ b/backend/windmill-trigger-gcp/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "windmill-trigger-gcp" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_gcp" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["gcp_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +google-cloud-pubsub.workspace = true +google-cloud-googleapis.workspace = true +tonic.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true +bytes.workspace = true +http.workspace = true +jsonwebtoken.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true +reqwest.workspace = true +tower-http.workspace = true +chrono.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-trigger-gcp/src/handler_oss.rs similarity index 91% rename from backend/windmill-api/src/triggers/gcp/handler_oss.rs rename to backend/windmill-trigger-gcp/src/handler_oss.rs index b75a9721eb..0b9de33ce6 100644 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ b/backend/windmill-trigger-gcp/src/handler_oss.rs @@ -5,14 +5,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::GcpTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + error::{Error, Result}, + DB, + }, windmill_git_sync::DeployedObject, + windmill_trigger::{TriggerCrud, TriggerData}, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-gcp/src/lib.rs b/backend/windmill-trigger-gcp/src/lib.rs new file mode 100644 index 0000000000..ae52ce1eb8 --- /dev/null +++ b/backend/windmill-trigger-gcp/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Clone, Copy)] +pub struct GcpTrigger; diff --git a/backend/windmill-api/src/triggers/gcp/listener_oss.rs b/backend/windmill-trigger-gcp/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/gcp/listener_oss.rs rename to backend/windmill-trigger-gcp/src/listener_oss.rs index a14168968a..b71cf305a1 100644 --- a/backend/windmill-api/src/triggers/gcp/listener_oss.rs +++ b/backend/windmill-trigger-gcp/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::GcpTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for GcpTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Gcp; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for GcpTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for GcpTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-http/Cargo.toml b/backend/windmill-trigger-http/Cargo.toml new file mode 100644 index 0000000000..43420793cf --- /dev/null +++ b/backend/windmill-trigger-http/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "windmill-trigger-http" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_http" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-trigger.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +http.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +lazy_static.workspace = true +quick_cache.workspace = true +regex.workspace = true +matchit.workspace = true +hyper.workspace = true +futures.workspace = true + +# http_trigger_auth deps +base64.workspace = true +hmac.workspace = true +sha1.workspace = true +sha2.workspace = true +constant_time_eq.workspace = true +itertools.workspace = true +thiserror.workspace = true +anyhow.workspace = true +hex.workspace = true diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs new file mode 100644 index 0000000000..43d75f7368 --- /dev/null +++ b/backend/windmill-trigger-http/src/handler.rs @@ -0,0 +1,583 @@ +use super::{ + validate_authentication_method, HttpConfig, HttpConfigRequest, HttpMethod, HttpTrigger, + RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, +}; +use axum::{async_trait, extract::Path, routing::post, Extension, Json, Router}; +use http::StatusCode; +use sqlx::PgConnection; +use std::collections::HashSet; +use windmill_api_auth::ApiAuthed; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + utils::require_admin, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; + +pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { + sqlx::query!("SELECT nextval('http_trigger_version_seq')") + .fetch_one(tx) + .await?; + Ok(()) +} + +pub fn generate_route_path_key(route_path: &str) -> String { + ROUTE_PATH_KEY_RE + .replace_all(route_path, "${1}${2}key") + .to_string() +} + +pub async fn route_path_key_exists( + route_path_key: &str, + http_method: &HttpMethod, + w_id: &str, + trigger_path: Option<&str>, + workspaced_route: Option, + db: &DB, +) -> Result { + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + route_path_key = $1 + AND workspace_id = $2 + AND http_method = $3 + AND ($4::TEXT IS NULL OR path != $4) + ) + "#, + &route_path_key, + w_id, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + } else { + let route_path_key = match workspaced_route { + Some(true) => { + std::borrow::Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))) + } + _ => std::borrow::Cow::Borrowed(route_path_key), + }; + + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) + OR (workspaced_route IS FALSE AND route_path_key = $1)) + AND http_method = $2 + AND ($3::TEXT IS NULL OR path != $3) + ) + "#, + &route_path_key, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + + Ok(exists) +} + +pub async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< + RouteExists, + >, +) -> Result> { + let route_path_key = generate_route_path_key(&route_path); + + let exists = route_path_key_exists( + &route_path_key, + &http_method, + &w_id, + trigger_path.as_deref(), + workspaced_route, + &db, + ) + .await?; + + Ok(Json(exists)) +} + +fn check_no_duplicates( + new_http_triggers: &[TriggerData], + route_path_key: &[String], +) -> Result<()> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert(( + &route_path_key[i], + trigger.config.http_method, + trigger.config.workspaced_route, + )) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.config.route_path + ))); + } + } + + Ok(()) +} + +pub async fn insert_new_trigger_into_db( + authed: &ApiAuthed, + tx: &mut PgConnection, + w_id: &str, + trigger: &TriggerData, + route_path_key: &str, +) -> Result<()> { + require_admin(authed.is_admin, &authed.username)?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + INSERT INTO http_trigger ( + workspace_id, + path, + route_path, + route_path_key, + workspaced_route, + authentication_resource_path, + wrap_body, + raw_string, + script_path, + summary, + description, + is_flow, + mode, + request_type, + authentication_method, + http_method, + static_asset_config, + edited_by, + email, + edited_at, + is_static_website, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + ) + "#, + w_id, + trigger.base.path, + trigger.config.route_path, + route_path_key, + trigger.config.workspaced_route.unwrap_or(false), + trigger.config.authentication_resource_path, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.base.script_path, + trigger.config.summary, + trigger.config.description, + trigger.base.is_flow, + trigger.base.mode() as _, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_many_http_triggers( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>>, +) -> Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let handler = HttpTrigger; + + let error_wrapper = |route_path: &str, error: Error| -> Error { + anyhow::anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + route_path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + handler + .validate_new(&db, &w_id, &new_http_trigger.config) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + let route_path_key = + check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; + + route_path_keys.push(route_path_key.clone()); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { + insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + audit_log( + &mut *tx, + &authed, + "http_trigger.create", + ActionKind::Create, + &w_id, + Some(&new_http_trigger.base.path), + None, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + + increase_trigger_version(&mut tx) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, + Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), + true, + None, + ) + .await + .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; + } + Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) +} + +async fn check_if_route_exist( + db: &DB, + config: &HttpConfigRequest, + workspace_id: &str, + trigger_path: Option<&str>, +) -> Result { + let route_path_key = generate_route_path_key(&config.route_path); + + let exists = route_path_key_exists( + &route_path_key, + &config.http_method, + workspace_id, + trigger_path, + config.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + Ok(route_path_key) +} + +#[async_trait] +impl TriggerCrud for HttpTrigger { + type TriggerConfig = HttpConfig; + type Trigger = Trigger; + type TriggerConfigRequest = HttpConfigRequest; + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = "http_trigger"; + const TRIGGER_TYPE: &'static str = "http"; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/http_triggers"; + const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; + const IS_ALLOWED_ON_CLOUD: bool = true; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "route_path", + "route_path_key", + "request_type", + "authentication_method", + "http_method", + "summary", + "description", + "static_asset_config", + "is_static_website", + "authentication_resource_path", + "workspaced_route", + "wrap_body", + "raw_string", + ]; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::HttpTrigger { path } + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/create_many", post(create_many_http_triggers)) + .route("/route_exists", post(exists_route)) + } + + async fn validate_new( + &self, + _db: &DB, + _workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method(new.authentication_method, new.raw_string)?; + + Ok(()) + } + + async fn validate_edit( + &self, + _db: &DB, + _workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + validate_authentication_method(edit.authentication_method, edit.raw_string)?; + + Ok(()) + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; + + insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + if authed.is_admin { + if trigger.config.route_path.is_empty() { + return Err(Error::BadRequest("route_path is required".to_string())); + }; + + let route_path = &trigger.config.route_path; + if !VALID_ROUTE_PATH_RE.is_match(route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + let route_path_key = + check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + route_path = $1, + route_path_key = $2, + workspaced_route = $3, + wrap_body = $4, + raw_string = $5, + authentication_resource_path = $6, + script_path = $7, + path = $8, + is_flow = $9, + mode = $10, + http_method = $11, + static_asset_config = $12, + edited_by = $13, + email = $14, + request_type = $15, + authentication_method = $16, + summary = $17, + description = $18, + edited_at = now(), + is_static_website = $19, + error_handler_path = $20, + error_handler_args = $21, + retry = $22 + WHERE + workspace_id = $23 AND + path = $24 + "#, + route_path, + &route_path_key, + trigger.config.workspaced_route, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } else { + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + wrap_body = $1, + raw_string = $2, + authentication_resource_path = $3, + script_path = $4, + path = $5, + is_flow = $6, + mode = $7, + http_method = $8, + static_asset_config = $9, + edited_by = $10, + email = $11, + request_type = $12, + authentication_method = $13, + summary = $14, + description = $15, + edited_at = now(), + is_static_website = $16, + error_handler_path = $17, + error_handler_args = $18, + retry = $19 + WHERE + workspace_id = $20 AND + path = $21 + "#, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { + increase_trigger_version(tx).await + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + increase_trigger_version(tx).await?; + + Ok(deleted > 0) + } +} diff --git a/backend/windmill-api/src/triggers/http/http_trigger_auth.rs b/backend/windmill-trigger-http/src/http_trigger_auth.rs similarity index 100% rename from backend/windmill-api/src/triggers/http/http_trigger_auth.rs rename to backend/windmill-trigger-http/src/http_trigger_auth.rs diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs new file mode 100644 index 0000000000..c0f46efc2f --- /dev/null +++ b/backend/windmill-trigger-http/src/lib.rs @@ -0,0 +1,413 @@ +use std::collections::HashMap; + +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use tokio::sync::{RwLock, RwLockReadGuard}; +use windmill_common::{ + error::{Error, Result}, + flows::Retry, + s3_helpers::S3Object, + utils::ExpiringCacheEntry, + worker::CLOUD_HOSTED, + DB, +}; + +use windmill_api_auth::ApiAuthed; +use windmill_trigger::TriggerMode; + +pub mod handler; +pub mod http_trigger_auth; + +lazy_static::lazy_static! { + pub static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); + pub static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); + + pub static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { + routers: HashMap::new(), + version: 0, + }); +} + +#[derive(Debug, Deserialize, Clone)] +pub struct TriggerRoute { + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub route_path: String, + pub workspace_id: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub edited_by: String, + pub email: String, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, + pub error_handler_path: Option, + pub error_handler_args: Option>>, + pub retry: Option>, + pub mode: TriggerMode, +} + +pub struct RoutersCache { + pub routers: HashMap>, + pub version: i64, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum RequestType { + Sync, + Async, + SyncSse, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] +#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] +pub enum AuthenticationMethod { + None, + Windmill, + ApiKey, + BasicHttp, + CustomScript, + Signature, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct HttpConfig { + pub route_path: String, + pub route_path_key: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct HttpConfigRequest { + #[serde(default)] + pub route_path: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: Option, + pub wrap_body: Option, + pub raw_string: Option, +} + +#[derive(Deserialize)] +struct HttpConfigRequestHelper { + #[serde(default)] + route_path: String, + request_type: Option, + is_async: Option, + authentication_method: AuthenticationMethod, + http_method: HttpMethod, + summary: Option, + description: Option, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: Option, + wrap_body: Option, + raw_string: Option, +} + +impl<'de> Deserialize<'de> for HttpConfigRequest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let helper = HttpConfigRequestHelper::deserialize(deserializer)?; + + // Determine request_type with backward compatibility + let request_type = if let Some(mode) = helper.request_type { + mode + } else if let Some(is_async) = helper.is_async { + if is_async { + RequestType::Async + } else { + RequestType::Sync + } + } else { + RequestType::Sync + }; + + Ok(HttpConfigRequest { + route_path: helper.route_path, + request_type, + authentication_method: helper.authentication_method, + http_method: helper.http_method, + summary: helper.summary, + description: helper.description, + static_asset_config: helper.static_asset_config, + is_static_website: helper.is_static_website, + authentication_resource_path: helper.authentication_resource_path, + workspaced_route: helper.workspaced_route, + wrap_body: helper.wrap_body, + raw_string: helper.raw_string, + }) + } +} + +// Regex patterns for route validation +lazy_static::lazy_static! { + // Matches named params like :id or wildcards like :* or * + pub static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); + pub static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); +} + +#[derive(Deserialize)] +pub struct RouteExists { + pub route_path: String, + pub http_method: HttpMethod, + pub trigger_path: Option, + pub workspaced_route: Option, +} + +pub fn validate_authentication_method( + authentication_method: AuthenticationMethod, + raw_string: Option, +) -> Result<()> { + match (authentication_method, raw_string) { + (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { + Err(Error::BadRequest( + "To use custom script authentication, please enable the raw body option." + .to_string(), + )) + } + _ => Ok(()), + } +} + +pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) + .fetch_one(db) + .await?; + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + if routers_cache.version == 0 || version > routers_cache.version { + drop(routers_cache); + let mut routers = HashMap::new(); + + for http_method in [ + HttpMethod::Get, + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, + ] { + let triggers = sqlx::query_as!( + TriggerRoute, + r#" + SELECT + path, + script_path, + is_flow, + route_path, + authentication_resource_path, + workspace_id, + request_type AS "request_type: _", + authentication_method AS "authentication_method: _", + edited_by, + email, + static_asset_config AS "static_asset_config: _", + wrap_body, + raw_string, + workspaced_route, + is_static_website, + error_handler_path, + error_handler_args as "error_handler_args: _", + retry as "retry: _", + mode as "mode: _" + FROM + http_trigger + WHERE + http_method = $1 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + "#, + &http_method as &HttpMethod + ) + .fetch_all(db) + .await?; + + let mut router = matchit::Router::new(); + + for trigger in triggers { + let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { + format!("/{}/{}", trigger.workspace_id, trigger.route_path) + } else { + format!("/{}", trigger.route_path) + }; + + if trigger.is_static_website { + router + .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider HTTP route {}/*wm_subpath: {:?}", + full_path, + e, + ); + }); + } + router + .insert(full_path.clone(), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); + }); + } + + routers.insert(http_method, router); + } + + let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; + *routers_cache = RoutersCache { routers, version }; + + Ok((true, routers_cache.downgrade())) + } else { + tracing::debug!("No HTTP routers refresh needed"); + Ok((false, routers_cache)) + } +} + +pub async fn refresh_routers_loop( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + match refresh_routers(db).await { + Ok(_) => { + tracing::info!("Loaded HTTP routers"); + } + Err(err) => { + tracing::error!("Error loading HTTP routers: {err:#}"); + } + }; + let db = db.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = killpill_rx.recv() => { + break; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + match refresh_routers(&db).await { + Ok((true, _)) => { + tracing::info!("Refreshed HTTP routers"); + } + Err(err) => { + tracing::error!("Error refreshing HTTP routers: {err:#}"); + } + _ => {} + } + } + } + } + }); +} + +pub struct HttpTrigger; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_request_type_backward_compatibility() { + // Test with new request_type field + let json_new = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + + // Test with legacy is_async = true + let json_legacy_async = r#"{ + "route_path": "/test", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); + assert_eq!(config.request_type, RequestType::Async); + + // Test with legacy is_async = false + let json_legacy_sync = r#"{ + "route_path": "/test", + "is_async": false, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test with neither field (default to sync) + let json_default = r#"{ + "route_path": "/test", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test that request_type takes precedence over is_async + let json_both = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + } +} diff --git a/backend/windmill-trigger-kafka/Cargo.toml b/backend/windmill-trigger-kafka/Cargo.toml new file mode 100644 index 0000000000..9887dfc04a --- /dev/null +++ b/backend/windmill-trigger-kafka/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-kafka" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_kafka" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] +kafka-gssapi = ["rdkafka/gssapi"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rdkafka.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +base64.workspace = true +sha2.workspace = true +anyhow.workspace = true diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-trigger-kafka/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/kafka/handler_oss.rs rename to backend/windmill-trigger-kafka/src/handler_oss.rs index 9fda83916f..d85a6caba6 100644 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ b/backend/windmill-trigger-kafka/src/handler_oss.rs @@ -1,5 +1,5 @@ #[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; +use windmill_trigger::TriggerData; #[allow(unused)] #[cfg(feature = "private")] @@ -8,14 +8,15 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::KafkaTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-kafka/src/lib.rs b/backend/windmill-trigger-kafka/src/lib.rs new file mode 100644 index 0000000000..0a2733cc5c --- /dev/null +++ b/backend/windmill-trigger-kafka/src/lib.rs @@ -0,0 +1,23 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct KafkaTrigger; diff --git a/backend/windmill-api/src/triggers/kafka/listener_oss.rs b/backend/windmill-trigger-kafka/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/kafka/listener_oss.rs rename to backend/windmill-trigger-kafka/src/listener_oss.rs index c5dca4ab37..057844efaf 100644 --- a/backend/windmill-api/src/triggers/kafka/listener_oss.rs +++ b/backend/windmill-trigger-kafka/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::KafkaTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for KafkaTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Kafka; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for KafkaTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for KafkaTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-mqtt/Cargo.toml b/backend/windmill-trigger-mqtt/Cargo.toml new file mode 100644 index 0000000000..305880b176 --- /dev/null +++ b/backend/windmill-trigger-mqtt/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-mqtt" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_mqtt" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["mqtt_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rumqttc.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +base64.workspace = true +anyhow.workspace = true +bytes.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-api/src/triggers/mqtt/handler.rs b/backend/windmill-trigger-mqtt/src/handler.rs similarity index 92% rename from backend/windmill-api/src/triggers/mqtt/handler.rs rename to backend/windmill-trigger-mqtt/src/handler.rs index ac4c027fff..7b12a23c5f 100644 --- a/backend/windmill-api/src/triggers/mqtt/handler.rs +++ b/backend/windmill-trigger-mqtt/src/handler.rs @@ -1,16 +1,15 @@ -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; use axum::async_trait; use itertools::Itertools; use sqlx::{types::Json as SqlxJson, PgConnection}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; use windmill_common::{ db::UserDB, error::{Error, Result}, }; use windmill_git_sync::DeployedObject; +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger, @@ -92,16 +91,16 @@ impl TriggerCrud for MqttTrigger { v3_config, v5_config, workspace_id, - path, - script_path, - is_flow, - email, - mode, + path, + script_path, + is_flow, + email, + mode, edited_by, error_handler_path, error_handler_args, retry - ) + ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 )"#, @@ -149,8 +148,8 @@ impl TriggerCrud for MqttTrigger { // Important to set server_id to NULL to stop current mqtt listener sqlx::query!( r#" - UPDATE - mqtt_trigger + UPDATE + mqtt_trigger SET mqtt_resource_path = $1, subscribe_topics = $2, @@ -158,19 +157,19 @@ impl TriggerCrud for MqttTrigger { client_id = $4, v3_config = $5, v5_config = $6, - is_flow = $7, - edited_by = $8, + is_flow = $7, + edited_by = $8, email = $9, script_path = $10, path = $11, - edited_at = now(), + edited_at = now(), error = NULL, server_id = NULL, error_handler_path = $14, error_handler_args = $15, retry = $16 - WHERE - workspace_id = $12 AND + WHERE + workspace_id = $12 AND path = $13 "#, trigger.config.mqtt_resource_path, diff --git a/backend/windmill-trigger-mqtt/src/lib.rs b/backend/windmill-trigger-mqtt/src/lib.rs new file mode 100644 index 0000000000..62e5093f7f --- /dev/null +++ b/backend/windmill-trigger-mqtt/src/lib.rs @@ -0,0 +1,347 @@ +use base64::{engine, prelude::*}; +use itertools::Itertools; +use rumqttc::{ + v5::{ + mqttbytes::{ + v5::{ConnectProperties, Filter}, + QoS as V5QoS, + }, + AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, + }, + AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, + QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow, Type}; +use std::{collections::HashMap, time::Duration}; +use windmill_common::{ + error::{to_anyhow, Error}, + triggers::TriggerKind, + worker::to_raw_value, +}; + +use crate::listener::EventLoop; +use windmill_trigger::trigger_helpers::TriggerJobArgs; + +pub mod handler; +pub mod listener; + +#[derive(Clone, Copy)] +pub struct MqttTrigger; + +impl TriggerJobArgs for MqttTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[serde(rename_all = "lowercase")] +pub enum QualityOfService { + Qos0, + Qos1, + Qos2, +} + +impl From for V3QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V3QoS::AtMostOnce, + QualityOfService::Qos1 => V3QoS::AtLeastOnce, + QualityOfService::Qos2 => V3QoS::ExactlyOnce, + } + } +} + +impl From for V5QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V5QoS::AtMostOnce, + QualityOfService::Qos1 => V5QoS::AtLeastOnce, + QualityOfService::Qos2 => V5QoS::ExactlyOnce, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV3Config { + clean_session: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV5Config { + clean_start: Option, + session_expiry_interval: Option, + topic_alias_maximum: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[sqlx(type_name = "MQTT_CLIENT_VERSION")] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MqttClientVersion { + V3, + V5, +} + +#[derive(Debug, Deserialize)] +pub struct Tls { + enabled: bool, + ca_certificate: String, + pkcs12_client_certificate: Option, + pkcs12_certificate_password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Credentials { + username: Option, + password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct MqttResource { + broker: String, + port: u16, + credentials: Option, + tls: Option, +} + +#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] +pub struct SubscribeTopic { + qos: QualityOfService, + topic: String, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct MqttConfig { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec>, + pub v3_config: Option>, + pub v5_config: Option>, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttConfigRequest { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec, + pub v3_config: Option, + pub v5_config: Option, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestMqttConfig { + pub mqtt_resource_path: String, + pub client_version: Option, + pub v3_config: Option, + pub v5_config: Option, +} + +// Constants +pub const KEEP_ALIVE: u64 = 60; +pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; +pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; +pub const TIMEOUT_DURATION: u64 = 10; +pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +pub struct V3MqttHandler; +pub struct V5MqttHandler; + +pub enum MqttClientResult { + V3((V3MqttHandler, V3EventLoop)), + V5((V5MqttHandler, V5EventLoop)), +} + +#[derive(Debug, thiserror::Error)] +pub enum MqttError { + #[error("{0}")] + Common(#[from] Error), + #[error("{0}")] + V5RumqttClient(#[from] rumqttc::v5::ClientError), + #[error("{0}")] + V5ConnectionError(#[from] rumqttc::v5::ConnectionError), + #[error("{0}")] + V3RumqttClient(#[from] rumqttc::ClientError), + #[error("{0}")] + V3ConnectionError(#[from] rumqttc::ConnectionError), + #[error("{0}")] + Base64Decode(#[from] base64::DecodeError), +} + +pub struct MqttClientBuilder<'client> { + mqtt_resource: MqttResource, + client_id: &'client str, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, +} + +impl<'client> MqttClientBuilder<'client> { + pub fn new( + mqtt_resource: MqttResource, + client_id: Option<&'client str>, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, + ) -> Self { + Self { + mqtt_resource, + client_id: client_id.unwrap_or(""), + subscribe_topics, + v3_config, + v5_config, + mqtt_client_version, + } + } + + pub async fn build_client(&self) -> Result { + match self.mqtt_client_version { + Some(MqttClientVersion::V5) | None => self.build_v5_client().await, + Some(MqttClientVersion::V3) => self.build_v3_client().await, + } + } + + fn get_tls_configuration(&self) -> Result, Error> { + let transport = match self.mqtt_resource.tls { + Some(ref tls) if tls.enabled => { + let transport = match tls.ca_certificate.trim().is_empty() { + true => rumqttc::Transport::Tls(TlsConfiguration::Native), + false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { + ca: tls.ca_certificate.as_bytes().to_vec(), + client_auth: { + match tls.pkcs12_client_certificate.as_ref() { + Some(client_certificate) + if !client_certificate.trim().is_empty() => + { + let client_certificate = BASE64_STANDARD + .decode(client_certificate) + .map_err(to_anyhow)?; + let password = tls + .pkcs12_certificate_password + .clone() + .unwrap_or("".to_string()); + Some((client_certificate, password)) + } + _ => None, + } + }, + }), + }; + + Some(transport) + } + _ => None, + }; + + Ok(transport) + } + + async fn build_v5_client(&self) -> Result { + let mut mqtt_options = V5MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + + mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); + + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + + if let Some(v5_config) = self.v5_config { + mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); + mqtt_options.set_connect_properties(ConnectProperties { + session_expiry_interval: v5_config.session_expiry_interval, + receive_maximum: None, + max_packet_size: None, + topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), + request_response_info: None, + request_problem_info: None, + user_properties: vec![], + authentication_method: None, + authentication_data: None, + }); + } + + let (async_client, mut event_loop) = + V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) + } + + async fn build_v3_client(&self) -> Result { + let mut mqtt_options = V3MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + if let Some(v3_config) = self.v3_config { + mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); + } + + let (async_client, mut event_loop) = + V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-trigger-mqtt/src/listener.rs similarity index 97% rename from backend/windmill-api/src/triggers/mqtt/listener.rs rename to backend/windmill-trigger-mqtt/src/listener.rs index af2d94b113..2098a3c20a 100644 --- a/backend/windmill-api/src/triggers/mqtt/listener.rs +++ b/backend/windmill-trigger-mqtt/src/listener.rs @@ -19,17 +19,14 @@ use windmill_common::{ DB, }; -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - mqtt::{ - MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, - V3MqttHandler, V5MqttHandler, - }, - trigger_helpers::TriggerJobArgs, - Listener, - }, +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::trigger_helpers::TriggerJobArgs; +use windmill_trigger::Listener; + +use super::{ + MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, V3MqttHandler, + V5MqttHandler, }; #[async_trait] diff --git a/backend/windmill-trigger-nats/Cargo.toml b/backend/windmill-trigger-nats/Cargo.toml new file mode 100644 index 0000000000..a0830097bb --- /dev/null +++ b/backend/windmill-trigger-nats/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "windmill-trigger-nats" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_nats" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["nats"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +async-nats.workspace = true +nkeys.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +base64.workspace = true diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-trigger-nats/src/handler_oss.rs similarity index 88% rename from backend/windmill-api/src/triggers/nats/handler_oss.rs rename to backend/windmill-trigger-nats/src/handler_oss.rs index 3ab3430e5f..9c011041b5 100644 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ b/backend/windmill-trigger-nats/src/handler_oss.rs @@ -2,17 +2,21 @@ #[cfg(feature = "private")] pub use super::handler_ee::*; +#[cfg(not(feature = "private"))] +use windmill_trigger::TriggerData; + #[cfg(not(feature = "private"))] use { super::NatsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + db::DB, + error::{Error, Result}, + }, windmill_git_sync::DeployedObject, + windmill_trigger::TriggerCrud, }; #[cfg(not(feature = "private"))] diff --git a/backend/windmill-trigger-nats/src/lib.rs b/backend/windmill-trigger-nats/src/lib.rs new file mode 100644 index 0000000000..668df073c9 --- /dev/null +++ b/backend/windmill-trigger-nats/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct NatsTrigger; diff --git a/backend/windmill-api/src/triggers/nats/listener_oss.rs b/backend/windmill-trigger-nats/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/nats/listener_oss.rs rename to backend/windmill-trigger-nats/src/listener_oss.rs index 80f1befded..588536f6bf 100644 --- a/backend/windmill-api/src/triggers/nats/listener_oss.rs +++ b/backend/windmill-trigger-nats/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::NatsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for NatsTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Nats; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for NatsTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for NatsTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-postgres/Cargo.toml b/backend/windmill-trigger-postgres/Cargo.toml new file mode 100644 index 0000000000..d932274f8e --- /dev/null +++ b/backend/windmill-trigger-postgres/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "windmill-trigger-postgres" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_postgres" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["postgres_trigger"] } +anyhow.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +rust-postgres.workspace = true +rust-postgres-native-tls.workspace = true +pg_escape.workspace = true +native-tls.workspace = true +rust_decimal.workspace = true +byteorder.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +chrono.workspace = true +rand.workspace = true +uuid.workspace = true +bytes.workspace = true +futures.workspace = true +thiserror.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-api/src/triggers/postgres/bool.rs b/backend/windmill-trigger-postgres/src/bool.rs similarity index 100% rename from backend/windmill-api/src/triggers/postgres/bool.rs rename to backend/windmill-trigger-postgres/src/bool.rs diff --git a/backend/windmill-api/src/triggers/postgres/converter.rs b/backend/windmill-trigger-postgres/src/converter.rs similarity index 99% rename from backend/windmill-api/src/triggers/postgres/converter.rs rename to backend/windmill-trigger-postgres/src/converter.rs index f57268c4b8..49162afd82 100644 --- a/backend/windmill-api/src/triggers/postgres/converter.rs +++ b/backend/windmill-trigger-postgres/src/converter.rs @@ -18,10 +18,10 @@ use uuid::Uuid; /** * This implementation is inspired by Postgres replication functionality * from https://github.com/supabase/pg_replicate -* -* Original implementation: +* +* Original implementation: * - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs -* +* */ #[derive(Debug, Error)] diff --git a/backend/windmill-api/src/triggers/postgres/handler.rs b/backend/windmill-trigger-postgres/src/handler.rs similarity index 98% rename from backend/windmill-api/src/triggers/postgres/handler.rs rename to backend/windmill-trigger-postgres/src/handler.rs index c46441cd0c..dc0057ffa4 100644 --- a/backend/windmill-api/src/triggers/postgres/handler.rs +++ b/backend/windmill-trigger-postgres/src/handler.rs @@ -16,20 +16,19 @@ use uuid; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error, Result}, + DB, }; use windmill_git_sync::DeployedObject; -use crate::{ - db::{ApiAuthed, DB}, - triggers::{postgres::PostgresTrigger, Trigger, TriggerCrud, TriggerData}, -}; +use windmill_api_auth::ApiAuthed; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, mapper::{Mapper, MappingInfo}, - PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PublicationData, - Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, + PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PostgresTrigger, + PublicationData, Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; @@ -211,8 +210,8 @@ impl TriggerCrud for PostgresTrigger { sqlx::query!( r#" - UPDATE postgres_trigger - SET + UPDATE postgres_trigger + SET postgres_resource_path = $1, replication_slot_name = $2, publication_name = $3, @@ -227,7 +226,7 @@ impl TriggerCrud for PostgresTrigger { error_handler_path = $11, error_handler_args = $12, retry = $13 - WHERE + WHERE workspace_id = $9 AND path = $10 "#, postgres_resource_path, @@ -419,12 +418,12 @@ pub async fn list_slot_name( let rows = pg_connection .query( r#" - SELECT + SELECT slot_name, active FROM - pg_replication_slots - WHERE + pg_replication_slots + WHERE plugin = 'pgoutput' AND slot_type = 'logical'; "#, @@ -467,11 +466,11 @@ pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &s let row = pg_connection .query_opt( r#" - SELECT - active_pid - FROM - pg_replication_slots - WHERE + SELECT + active_pid + FROM + pg_replication_slots + WHERE slot_name = $1 "#, &[&slot_name], diff --git a/backend/windmill-api/src/triggers/postgres/hex.rs b/backend/windmill-trigger-postgres/src/hex.rs similarity index 96% rename from backend/windmill-api/src/triggers/postgres/hex.rs rename to backend/windmill-trigger-postgres/src/hex.rs index 2ada70dd35..f0d155b554 100644 --- a/backend/windmill-api/src/triggers/postgres/hex.rs +++ b/backend/windmill-trigger-postgres/src/hex.rs @@ -1,15 +1,13 @@ use std::num::ParseIntError; - /** * This implementation is inspired by Postgres replication functionality * from https://github.com/supabase/pg_replicate -* -* Original implementation: +* +* Original implementation: * - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs -* +* */ - use thiserror::Error; #[derive(Debug, Error)] diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs new file mode 100644 index 0000000000..a1fef1cfa8 --- /dev/null +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -0,0 +1,552 @@ +use std::collections::HashMap; + +use windmill_api_auth::ApiAuthed; +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::trigger_helpers::TriggerJobArgs; +use chrono::Utc; +use itertools::Itertools; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; +use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::value::RawValue; +use sqlx::FromRow; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + utils::empty_as_none, + DB, +}; + +mod bool; +mod converter; +pub mod handler; +mod hex; +pub mod listener; +mod mapper; +mod relation; +mod replication_message; + +#[derive(Clone, Copy)] +pub struct PostgresTrigger; + +impl TriggerJobArgs for PostgresTrigger { + type Payload = HashMap>; + const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; + fn v1_payload_fn(payload: &HashMap>) -> HashMap> { + payload.to_owned() + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct PostgresConfig { + pub postgres_resource_path: String, + pub replication_slot_name: String, + pub publication_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub basic_mode: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PostgresConfigRequest { + postgres_resource_path: String, + #[serde(default)] + replication_slot_name: String, + #[serde(default)] + publication_name: String, + publication: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestPostgresConfig { + pub postgres_resource_path: String, +} + +fn check_if_valid_relation<'de, D>( + relations: D, +) -> std::result::Result>, D::Error> +where + D: Deserializer<'de>, +{ + let relations: Option> = Option::deserialize(relations)?; + let mut track_all_table_in_schema = false; + let mut track_specific_columns_in_table = false; + match relations { + Some(relations) => { + for relation in relations.iter() { + if relation.schema_name.is_empty() { + return Err(serde::de::Error::custom( + "Schema Name must not be empty".to_string(), + )); + } + + if !track_all_table_in_schema && relation.table_to_track.is_empty() { + track_all_table_in_schema = true; + continue; + } + + for table_to_track in relation.table_to_track.iter() { + if table_to_track.table_name.trim().is_empty() { + return Err(serde::de::Error::custom( + "Table name must not be empty".to_string(), + )); + } + + if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { + track_specific_columns_in_table = true; + } + } + + if track_all_table_in_schema && track_specific_columns_in_table { + return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); + } + } + + if !relations + .iter() + .map(|relation| relation.schema_name.as_str()) + .all_unique() + { + return Err(serde::de::Error::custom( + "You cannot choose a schema more than one time".to_string(), + )); + } + + Ok(Some(relations)) + } + None => Ok(None), + } +} + +fn check_if_valid_transaction_type<'de, D>( + transaction_type: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; + if transaction_type.len() > 3 { + return Err(serde::de::Error::custom( + "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" + .to_string(), + )); + } + transaction_type.sort_unstable(); + transaction_type.dedup(); + + for transaction in transaction_type.iter() { + match transaction.to_lowercase().as_ref() { + "insert" => {}, + "update" => {}, + "delete" => {}, + _ => { + return Err(serde::de::Error::custom( + "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" + .to_string(), + )) + } + } + } + + Ok(transaction_type) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PublicationData { + #[serde(default, deserialize_with = "check_if_valid_relation")] + pub table_to_track: Option>, + #[serde(deserialize_with = "check_if_valid_transaction_type")] + pub transaction_to_track: Vec, +} + +impl PublicationData { + pub fn new( + table_to_track: Option>, + transaction_to_track: Vec, + ) -> PublicationData { + PublicationData { table_to_track, transaction_to_track } + } +} + +// Slot list struct +#[derive(FromRow, Debug, Serialize)] +pub struct SlotList { + pub slot_name: Option, + pub active: Option, +} + +// Slot struct +#[derive(Debug, Serialize, Deserialize)] +pub struct Slot { + pub name: String, +} + +// Template script struct +#[derive(Debug, Deserialize)] +pub struct TemplateScript { + pub postgres_resource_path: String, + #[serde(deserialize_with = "check_if_valid_relation")] + pub relations: Option>, + pub language: Language, +} + +// Language enum +#[derive(Deserialize, Debug)] +pub enum Language { + #[serde(rename = "typescript", alias = "Typescript")] + Typescript, +} + +// Test postgres struct +#[derive(Serialize, Deserialize)] +pub struct TestPostgres { + pub postgres_resource_path: String, +} + +// PostgreSQL publication replication struct +#[derive(Serialize, Deserialize)] +pub struct PostgresPublicationReplication { + pub publication_name: String, + pub replication_slot_name: String, +} + +impl PostgresPublicationReplication { + pub fn new( + publication_name: String, + replication_slot_name: String, + ) -> PostgresPublicationReplication { + PostgresPublicationReplication { publication_name, replication_slot_name } + } +} + +pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Postgres { + pub user: String, + pub password: String, + pub host: String, + pub port: Option, + pub dbname: String, + #[serde(default)] + pub sslmode: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub root_certificate_pem: Option, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct TableToTrack { + pub table_name: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub where_clause: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub columns_name: Option>, +} + +impl TableToTrack { + pub fn new( + table_name: String, + where_clause: Option, + columns_name: Option>, + ) -> TableToTrack { + TableToTrack { table_name, where_clause, columns_name } + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct Relations { + pub schema_name: String, + pub table_to_track: Vec, +} + +impl Relations { + pub fn new(schema_name: String, table_to_track: Vec) -> Relations { + Relations { schema_name, table_to_track } + } + + pub fn add_new_table(&mut self, table_to_track: TableToTrack) { + self.table_to_track.push(table_to_track); + } +} + +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); + if let Err(e) = connection.await { + tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); + }; + tracing::info!("PostgreSQL trigger connection closed"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + +pub async fn get_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, + logical_mode: bool, +) -> Result { + let database = + try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) + .await?; + + Ok(get_raw_postgres_connection(&database, logical_mode).await?) +} + +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await +} + +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn check_if_valid_publication_for_postgres_version( + pg_connection: &Client, + table_to_track: Option<&[Relations]>, +) -> Result { + use crate::handler::get_postgres_version_internal; + + let postgres_version = get_postgres_version_internal(pg_connection).await?; + + let pg_14 = postgres_version.starts_with("14"); + if pg_14 { + let unsupported_publication = table_to_track + .and_then(|relations| { + relations.iter().find(|relation| { + let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { + table_to_track.where_clause.is_some() + || table_to_track.columns_name.is_some() + }); + + relation.table_to_track.is_empty() || invalid_relation.is_some() + }) + }) + .is_some(); + + if unsupported_publication { + return Err(Error::BadRequest( + "Your PostgreSQL database is running version 14, which does not support the following publication features: \ + - WHERE clause filtering, \ + - selective column tracking, and \ + - tracking all tables within a schema.\n\ + These features are only available in PostgreSQL 15 and above.".to_string(), + )); + } + } + Ok(pg_14) +} + +pub async fn create_pg_publication( + pg_connection: &Client, + publication_name: &str, + table_to_track: Option<&[Relations]>, + transaction_to_track: &[String], +) -> Result<()> { + let pg_14 = + check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; + let mut query = String::from("CREATE PUBLICATION "); + + query.push_str("e_identifier(publication_name)); + + match table_to_track { + Some(database_component) if !database_component.is_empty() => { + query.push_str(" FOR"); + let mut first = true; + for (i, schema) in database_component.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push_str(" TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); + } else { + if pg_14 && first { + query.push_str(" TABLE ONLY "); + first = false + } else if !pg_14 { + query.push_str(" TABLE ONLY "); + } + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push_str(&full_name); + if let Some(columns) = table.columns_name.as_ref() { + query.push_str(" ("); + let columns = columns + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push_str(&columns); + query.push_str(")"); + } + + if let Some(where_clause) = &table.where_clause { + query.push_str(" WHERE ("); + query.push_str(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push_str(", "); + } + } + } + if i < database_component.len() - 1 { + query.push_str(", "); + } + } + } + _ => { + query.push_str(" FOR ALL TABLES "); + } + }; + + if !transaction_to_track.is_empty() { + let transactions = || transaction_to_track.iter().join(", "); + query.push_str(" WITH (publish = '"); + query.push_str(&transactions()); + query.push_str("');"); + } + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { + let mut query = String::from("DROP PUBLICATION IF EXISTS "); + let quoted_publication_name = quote_identifier(publication_name); + query.push_str("ed_publication_name); + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + +pub fn generate_random_string() -> String { + let timestamp = Utc::now().timestamp_millis().to_string(); + let mut rng = rand::rng(); + let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; + + let random_part = (0..10) + .map(|_| { + charset + .chars() + .nth(rng.random_range(0..charset.len())) + .unwrap() + }) + .collect::(); + + format!("{}_{}", timestamp, random_part) +} diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-trigger-postgres/src/listener.rs similarity index 95% rename from backend/windmill-api/src/triggers/postgres/listener.rs rename to backend/windmill-trigger-postgres/src/listener.rs index bfb527e3e7..cc71cedd33 100644 --- a/backend/windmill-api/src/triggers/postgres/listener.rs +++ b/backend/windmill-trigger-postgres/src/listener.rs @@ -15,25 +15,18 @@ use windmill_common::{ DB, }; -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - postgres::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, - handler::drop_logical_replication_slot, - relation::RelationConverter, - replication_message::{ - LogicalReplicationMessage::{ - Begin, Commit, Delete, Insert, Relation, Type, Update, - }, - PrimaryKeepAliveBody, ReplicationMessage, - }, - Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, - }, - trigger_helpers::TriggerJobArgs, - Listener, +use windmill_store::resources::try_get_resource_from_db_as; +use windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}; + +use super::{ + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::drop_logical_replication_slot, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, + PrimaryKeepAliveBody, ReplicationMessage, }, + Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; diff --git a/backend/windmill-api/src/triggers/postgres/mapper.rs b/backend/windmill-trigger-postgres/src/mapper.rs similarity index 98% rename from backend/windmill-api/src/triggers/postgres/mapper.rs rename to backend/windmill-trigger-postgres/src/mapper.rs index a24fb65378..727d6b8bb1 100644 --- a/backend/windmill-api/src/triggers/postgres/mapper.rs +++ b/backend/windmill-trigger-postgres/src/mapper.rs @@ -129,8 +129,7 @@ export async function main( ) {{ }} "#, - &struct_definition, - &struct_definition + &struct_definition, &struct_definition ) } } diff --git a/backend/windmill-api/src/triggers/postgres/relation.rs b/backend/windmill-trigger-postgres/src/relation.rs similarity index 100% rename from backend/windmill-api/src/triggers/postgres/relation.rs rename to backend/windmill-trigger-postgres/src/relation.rs diff --git a/backend/windmill-api/src/triggers/postgres/replication_message.rs b/backend/windmill-trigger-postgres/src/replication_message.rs similarity index 100% rename from backend/windmill-api/src/triggers/postgres/replication_message.rs rename to backend/windmill-trigger-postgres/src/replication_message.rs diff --git a/backend/windmill-trigger-sqs/Cargo.toml b/backend/windmill-trigger-sqs/Cargo.toml new file mode 100644 index 0000000000..a073f6dfc1 --- /dev/null +++ b/backend/windmill-trigger-sqs/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "windmill-trigger-sqs" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_sqs" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-store/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private", "windmill-common/aws_auth", "windmill-common/openidconnect", "windmill-store/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-store = { workspace = true, features = ["sqs_trigger"] } +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +aws-sdk-sqs.workspace = true +aws-sdk-sts.workspace = true +aws-config.workspace = true +aws-credential-types.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +chrono.workspace = true +backon.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-trigger-sqs/src/handler_oss.rs similarity index 90% rename from backend/windmill-api/src/triggers/sqs/handler_oss.rs rename to backend/windmill-trigger-sqs/src/handler_oss.rs index 21200e8e38..6340e95cd1 100644 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ b/backend/windmill-trigger-sqs/src/handler_oss.rs @@ -5,19 +5,19 @@ pub use super::handler_ee::*; #[cfg(not(feature = "private"))] use { super::SqsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, - }, axum::async_trait, sqlx::PgConnection, - windmill_common::error::{Error, Result}, + windmill_api_auth::ApiAuthed, + windmill_common::{ + error::{Error, Result}, + DB, + }, windmill_git_sync::DeployedObject, + windmill_trigger::{Trigger, TriggerCrud, TriggerData}, }; #[cfg(not(feature = "private"))] #[async_trait] -#[cfg(not(feature = "private"))] impl TriggerCrud for SqsTrigger { type Trigger = Trigger; type TriggerConfig = (); diff --git a/backend/windmill-trigger-sqs/src/lib.rs b/backend/windmill-trigger-sqs/src/lib.rs new file mode 100644 index 0000000000..8c7217725c --- /dev/null +++ b/backend/windmill-trigger-sqs/src/lib.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct SqsTrigger; diff --git a/backend/windmill-api/src/triggers/sqs/listener_oss.rs b/backend/windmill-trigger-sqs/src/listener_oss.rs similarity index 61% rename from backend/windmill-api/src/triggers/sqs/listener_oss.rs rename to backend/windmill-trigger-sqs/src/listener_oss.rs index 0aaf5360c3..7081d67b0f 100644 --- a/backend/windmill-api/src/triggers/sqs/listener_oss.rs +++ b/backend/windmill-trigger-sqs/src/listener_oss.rs @@ -1,22 +1,32 @@ #[allow(unused)] - #[cfg(feature = "private")] pub use super::listener_ee::*; #[cfg(not(feature = "private"))] use { super::SqsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, + serde_json::value::RawValue, + std::{collections::HashMap, sync::Arc}, tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, + windmill_common::{error::Result, jobs::JobTriggerKind, triggers::TriggerKind, DB}, + windmill_trigger::{listener::ListeningTrigger, trigger_helpers::TriggerJobArgs, Listener}, }; +#[cfg(not(feature = "private"))] +impl TriggerJobArgs for SqsTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Sqs; + fn v1_payload_fn(_payload: &Self::Payload) -> HashMap> { + HashMap::new() + } +} + #[cfg(not(feature = "private"))] #[async_trait::async_trait] impl Listener for SqsTrigger { type Consumer = (); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; async fn get_consumer( @@ -35,6 +45,7 @@ impl Listener for SqsTrigger { _listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { () } diff --git a/backend/windmill-trigger-websocket/Cargo.toml b/backend/windmill-trigger-websocket/Cargo.toml new file mode 100644 index 0000000000..6b239d237c --- /dev/null +++ b/backend/windmill-trigger-websocket/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "windmill-trigger-websocket" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger_websocket" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise", "windmill-trigger/enterprise"] +private = ["windmill-common/private"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-api-auth.workspace = true +windmill-trigger.workspace = true +windmill-git-sync.workspace = true +windmill-queue.workspace = true +tokio-tungstenite.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +tokio.workspace = true +tracing.workspace = true +async-trait.workspace = true +itertools.workspace = true +anyhow.workspace = true +futures.workspace = true +http.workspace = true diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-trigger-websocket/src/handler.rs similarity index 98% rename from backend/windmill-api/src/triggers/websocket/handler.rs rename to backend/windmill-trigger-websocket/src/handler.rs index bea4f195b2..c50b2dffec 100644 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ b/backend/windmill-trigger-websocket/src/handler.rs @@ -1,20 +1,19 @@ use std::borrow::Cow; -use crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; use axum::async_trait; use itertools::Itertools; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, PgConnection}; use tokio_tungstenite::connect_async; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; use windmill_common::{ db::UserDB, error::{Error, Result}, worker::to_raw_value, }; use windmill_git_sync::DeployedObject; +use windmill_trigger::{Trigger, TriggerCrud, TriggerData}; use super::{ get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, @@ -167,7 +166,7 @@ impl TriggerCrud for WebsocketTrigger { // important to update server_id to NULL to stop current websocket listener sqlx::query!( " - UPDATE + UPDATE websocket_trigger SET url = $1, diff --git a/backend/windmill-trigger-websocket/src/lib.rs b/backend/windmill-trigger-websocket/src/lib.rs new file mode 100644 index 0000000000..ca754d8349 --- /dev/null +++ b/backend/windmill-trigger-websocket/src/lib.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; + +use windmill_api_auth::ApiAuthed; +use windmill_trigger::trigger_helpers::{ + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +pub mod handler; +pub mod listener; + +#[derive(Copy, Clone)] +pub struct WebsocketTrigger; + +impl TriggerJobArgs for WebsocketTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WebsocketConfig { + pub url: String, + #[serde(default)] + pub filters: Vec>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_messages: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub url_runnable_args: Option>>, + #[serde(default)] + pub can_return_message: bool, + #[serde(default)] + pub can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebsocketConfigRequest { + url: String, + filters: Vec, + initial_messages: Option>, + url_runnable_args: Option, + can_return_message: bool, + can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestWebsocketConfig { + url: String, + url_runnable_args: Option, +} + +pub fn value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + let args_map: Option> = serde_json::from_str(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; + + args_map + .unwrap_or_else(HashMap::new) + .into_iter() + .map(|(k, v)| { + let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { + Error::BadRequest(format!("failed to convert to raw value: {}", e)) + })?; + Ok((k, raw_value)) + }) + .collect::>>>() + } else { + Ok(HashMap::new()) + }?; + Ok(args) +} + +pub async fn get_url_from_runnable_value( + path: &str, + is_flow: bool, + db: &DB, + authed: ApiAuthed, + args: Option<&Box>, + workspace_id: &str, +) -> Result { + tracing::info!( + "Running {} {} to get WebSocket URL", + if is_flow { "flow" } else { "script" }, + path + ); + + let args = value_to_args_hashmap(args)?; + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed, + workspace_id, + path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), + ) + .await?; + + serde_json::from_str::(result.get()).map_err(|_| { + Error::BadConfig(format!( + "{} {} did not return a string", + if is_flow { "Flow" } else { "Script" }, + path, + )) + }) +} diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs similarity index 78% rename from backend/windmill-api/src/triggers/websocket/listener.rs rename to backend/windmill-trigger-websocket/src/listener.rs index 0f22dffdc3..f6fba882a8 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -1,14 +1,4 @@ -use super::WebsocketTrigger; -use crate::triggers::{ - filter::{is_value_superset, Filter, JsonFilter}, - listener::ListeningTrigger, - trigger_helpers::{ - trigger_runnable, trigger_runnable_and_wait_for_raw_result, - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, - websocket::{get_url_from_runnable_value, WebsocketConfig}, - Listener, -}; +use super::{get_url_from_runnable_value, WebsocketConfig, WebsocketTrigger}; use anyhow::Context; use async_trait::async_trait; use futures::{stream::SplitSink, SinkExt, StreamExt}; @@ -28,99 +18,109 @@ use windmill_common::{ DB, }; use windmill_queue::PushArgsOwned; +use windmill_trigger::filter::{is_value_superset, Filter, JsonFilter}; +use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::trigger_helpers::{ + trigger_runnable, trigger_runnable_and_wait_for_raw_result, + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; +use windmill_trigger::Listener; -impl ListeningTrigger { - async fn send_initial_messages( - &self, - writer: &mut SplitSink>, Message>, - db: &DB, - ) -> Result<()> { - let initial_messages: Vec = self - .trigger_config - .initial_messages - .as_deref() - .unwrap_or_default() - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); +async fn send_initial_messages( + listening_trigger: &ListeningTrigger, + writer: &mut SplitSink>, Message>, + db: &DB, +) -> Result<()> { + let initial_messages: Vec = listening_trigger + .trigger_config + .initial_messages + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); - let WebsocketConfig { ref url, .. } = self.trigger_config; - let runnable_kind = if self.is_flow { "flow" } else { "script" }; - let mut authed_o = None; - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() - } else { - msg - }; - tracing::info!( - "Sending raw message initial message to WebSocket {}: {}", - url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; - } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running {} {} for initial message to WebSocket {}", - runnable_kind, - path, - url, - ); - - let args = raw_value_to_args_hashmap(Some(&args))?; - - if authed_o.is_none() { - authed_o = Some(self.authed(db, "ws").await?); - } - let authed = authed_o.clone().unwrap(); - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed.clone(), - &self.workspace_id, - &path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(self.path.to_owned()), JobTriggerKind::Websocket), - ) + let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; + let runnable_kind = if listening_trigger.is_flow { + "flow" + } else { + "script" + }; + let mut authed_o = None; + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to WebSocket {}: {}", + url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) .await - .map(|r| r.get().to_owned())?; + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running {} {} for initial message to WebSocket {}", + runnable_kind, + path, + url, + ); - tracing::info!( - "Sending {} {} result to WebSocket {}", - runnable_kind, - path, - url - ); + let args = raw_value_to_args_hashmap(Some(&args))?; - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!("Failed to send {} {} result", runnable_kind, path) - })?; + if authed_o.is_none() { + authed_o = Some(listening_trigger.authed(db, "ws").await?); } + let authed = authed_o.clone().unwrap(); + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed.clone(), + &listening_trigger.workspace_id, + &path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new( + Some(listening_trigger.path.to_owned()), + JobTriggerKind::Websocket, + ), + ) + .await + .map(|r| r.get().to_owned())?; + + tracing::info!( + "Sending {} {} result to WebSocket {}", + runnable_kind, + path, + url + ); + + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| format!("Failed to send {} {} result", runnable_kind, path))?; } } - Ok(()) } + Ok(()) } #[async_trait] @@ -205,7 +205,7 @@ impl Listener for WebsocketTrigger { _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { return; }, - result = listening_trigger.send_initial_messages(&mut writer, &db) => { + result = send_initial_messages(listening_trigger, &mut writer, &db) => { if let Err(err) = result { self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; return diff --git a/backend/windmill-trigger/Cargo.toml b/backend/windmill-trigger/Cargo.toml new file mode 100644 index 0000000000..c35a6ffeaa --- /dev/null +++ b/backend/windmill-trigger/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "windmill-trigger" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_trigger" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +cloud = ["windmill-common/cloud"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-api-jobs.workspace = true +windmill-api-sse.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +axum.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +uuid.workspace = true +chrono.workspace = true +http.workspace = true +anyhow.workspace = true +tracing.workspace = true +tokio.workspace = true +async-trait.workspace = true +sql-builder.workspace = true +reqwest.workspace = true +lazy_static.workspace = true +hyper.workspace = true +rand.workspace = true +itertools.workspace = true +windmill-parser.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-py.workspace = true diff --git a/backend/windmill-trigger/src/capture.rs b/backend/windmill-trigger/src/capture.rs new file mode 100644 index 0000000000..7831adef55 --- /dev/null +++ b/backend/windmill-trigger/src/capture.rs @@ -0,0 +1,127 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use sqlx::types::Json as SqlxJson; +use windmill_common::{ + db::DB, error::Result, triggers::TriggerKind, utils::not_found_if_none, worker::CLOUD_HOSTED, +}; +use windmill_queue::{PushArgs, PushArgsOwned}; + +const KEEP_LAST: i64 = 20; + +struct ActiveCaptureOwner { + owner: String, + email: String, +} + +pub async fn get_active_capture_owner_and_email( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + kind: &TriggerKind, +) -> Result<(String, String)> { + let capture_config = sqlx::query_as!( + ActiveCaptureOwner, + r#" + SELECT + owner, + email + FROM + capture_config + WHERE + workspace_id = $1 + AND path = $2 + AND is_flow = $3 + AND trigger_kind = $4 + AND last_client_ping > NOW() - INTERVAL '10 seconds' + "#, + &w_id, + &path, + is_flow, + kind as &TriggerKind, + ) + .fetch_optional(db) + .await?; + + let capture_config = not_found_if_none( + capture_config, + &format!("capture config for {} trigger", kind), + path, + )?; + + Ok((capture_config.owner, capture_config.email)) +} + +async fn clear_captures_history(db: &DB, w_id: &str) -> Result<()> { + if *CLOUD_HOSTED { + /* Retain only KEEP_LAST most recent captures in this workspace. */ + sqlx::query!( + r#" + DELETE FROM + capture + WHERE + workspace_id = $1 + AND created_at <= ( + SELECT + created_at + FROM + capture + WHERE + workspace_id = $1 + ORDER BY + created_at DESC + OFFSET $2 + LIMIT 1 + ) + "#, + &w_id, + KEEP_LAST, + ) + .execute(db) + .await?; + } + Ok(()) +} + +pub async fn insert_capture_payload( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + trigger_kind: &TriggerKind, + main_args: PushArgsOwned, + preprocessor_args: PushArgsOwned, + owner: &str, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO + capture ( + workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + "#, + &w_id, + path, + is_flow, + trigger_kind as &TriggerKind, + SqlxJson(PushArgs { args: &main_args.args, extra: main_args.extra }) as SqlxJson, + SqlxJson(PushArgs { args: &preprocessor_args.args, extra: preprocessor_args.extra }) + as SqlxJson, + owner, + ) + .execute(db) + .await?; + + clear_captures_history(db, &w_id).await?; + + Ok(()) +} diff --git a/backend/windmill-api/src/triggers/filter.rs b/backend/windmill-trigger/src/filter.rs similarity index 94% rename from backend/windmill-api/src/triggers/filter.rs rename to backend/windmill-trigger/src/filter.rs index 553e0a70f5..57ef45839f 100644 --- a/backend/windmill-api/src/triggers/filter.rs +++ b/backend/windmill-trigger/src/filter.rs @@ -53,13 +53,11 @@ impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } + (Value::Object(json_map), Value::Object(check_map)) => check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }), (Value::Array(json_array), Value::Array(check_array)) => { check_array.iter().all(|check_item| { json_array diff --git a/backend/windmill-api/src/triggers/global_handler.rs b/backend/windmill-trigger/src/global_handler.rs similarity index 97% rename from backend/windmill-api/src/triggers/global_handler.rs rename to backend/windmill-trigger/src/global_handler.rs index 3143d6806f..8c8ab70d5c 100644 --- a/backend/windmill-api/src/triggers/global_handler.rs +++ b/backend/windmill-trigger/src/global_handler.rs @@ -1,8 +1,4 @@ -use crate::{ - db::{ApiAuthed, DB}, - jobs::cancel_jobs, - triggers::trigger_helpers::trigger_runnable_inner, -}; +use crate::trigger_helpers::trigger_runnable_inner; use axum::{ extract::{Extension, Path}, response::Json, @@ -13,8 +9,10 @@ use serde_json::value::RawValue; use sqlx::PgConnection; use std::collections::HashMap; use uuid::Uuid; +use windmill_api_auth::ApiAuthed; +use windmill_api_jobs::execution::cancel_jobs; use windmill_common::{ - db::UserDB, + db::{UserDB, DB}, error::{self, Error, Result}, jobs::JobTriggerKind, triggers::TriggerMetadata, @@ -62,12 +60,12 @@ async fn get_suspended_trigger( ]; let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND path = $2 "#, fields.join(", "), diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs new file mode 100644 index 0000000000..be9e92228d --- /dev/null +++ b/backend/windmill-trigger/src/handler.rs @@ -0,0 +1,693 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::types::{StandardTriggerQuery, TriggerData, TriggerMode}; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::{FromRow, PgConnection}; +use std::fmt::Debug; +use windmill_api_auth::{check_scopes, ApiAuthed}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{paginate, Pagination, StripPath}, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::DeployedObject; + +use axum::{ + extract::{Path, Query}, + http::StatusCode, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use std::sync::Arc; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_git_sync::handle_deployment_metadata; + +#[async_trait] +pub trait TriggerCrud: Send + Sync + 'static { + type Trigger: Serialize + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Send + + Sync + + Unpin; + + type TriggerConfig: Debug + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Serialize + + Send + + Sync + + Unpin; + + type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; + type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + + const TABLE_NAME: &'static str; + const TRIGGER_TYPE: &'static str; + const SUPPORTS_SERVER_STATE: bool; + const SUPPORTS_TEST_CONNECTION: bool; + const ROUTE_PREFIX: &'static str; + const DEPLOYMENT_NAME: &'static str; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; + const IS_ALLOWED_ON_CLOUD: bool; + + fn get_deployed_object(path: String) -> DeployedObject; + + async fn validate_new( + &self, + db: &DB, + workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + self.validate_config(db, new, workspace_id).await + } + + async fn validate_edit( + &self, + db: &DB, + workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + self.validate_config(db, edit, workspace_id).await + } + + async fn validate_config( + &self, + _db: &DB, + _config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + Ok(()) + } + + fn scope_domain_name() -> &'static str { + &Self::ROUTE_PREFIX[1..] + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn test_connection( + &self, + _db: &DB, + _authed: &ApiAuthed, + _user_db: &UserDB, + _workspace_id: &str, + _config: Self::TestConnectionConfig, + ) -> Result<()> { + Err( + anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) + .into(), + ) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn get_trigger_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let sql = format!( + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND + path = $2 + "#, + fields.join(", "), + Self::TABLE_NAME + ); + + sqlx::query_as(&sql) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) + } + + async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { + let exists = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .fetch_one(db) + .await?; + + Ok(exists) + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) + } + + async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { + Ok(()) + } + + async fn set_trigger_mode( + &self, + authed: &ApiAuthed, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + mode: &TriggerMode, + ) -> Result { + let updated = if Self::SUPPORTS_SERVER_STATE { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + } else { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now() + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + }; + + self.set_trigger_mode_extra_action(&mut *tx).await?; + + Ok(updated > 0) + } + + #[allow(unused)] + async fn trigger_count( + &self, + tx: &mut PgConnection, + workspace_id: &str, + is_flow: bool, + script_path: &str, + ) -> i64 { + let count = sqlx::query_scalar(&format!( + r#" + SELECT + COUNT(*) + FROM + {} + WHERE + workspace_id = $1 AND + is_flow = $2 AND + script_path = $3 + "#, + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(is_flow) + .bind(script_path) + .fetch_one(&mut *tx) + .await + .unwrap_or(0); + + count + } + + async fn list_triggers( + &self, + tx: &mut PgConnection, + workspace_id: &str, + query: Option<&StandardTriggerQuery>, + ) -> Result> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&workspace_id)); + + if let Some(query) = query { + let (per_page, offset) = + paginate(Pagination { per_page: query.per_page, page: query.page }); + if let Some(path) = &query.path { + sqlb.and_where_eq("script_path", "?".bind(path)); + } + + if let Some(is_flow) = query.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + + if let Some(path_start) = &query.path_start { + sqlb.and_where_like_left("path", path_start); + } + + sqlb.offset(offset).limit(per_page); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; + + Ok(triggers) + } +} + +pub fn trigger_routes() -> Router { + let mut router = Router::new() + .route("/create", post(create_trigger::)) + .route("/list", get(list_triggers::)) + .route("/get/*path", get(get_trigger::)) + .route("/update/*path", post(update_trigger::)) + .route("/delete/*path", delete(delete_trigger::)) + .route("/exists/*path", get(exists_trigger::)) + .route("/setmode/*path", post(set_trigger_mode::)); + + if T::SUPPORTS_TEST_CONNECTION { + router = router.route("/test", post(test_connection::)); + } + + router +} + +async fn create_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(new_trigger): Json>, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &new_trigger.base.path + ) + })?; + + if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { + return Err(Error::BadRequest(format!( + "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", + T::TRIGGER_TYPE + ))); + } + + handler + .validate_new(&db, &workspace_id, &new_trigger.config) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = new_trigger.base.path.clone(); + + handler + .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.create", T::TRIGGER_TYPE), + ActionKind::Create, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, new_path)) +} + +async fn list_triggers( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path(workspace_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let triggers = handler + .list_triggers(&mut *tx, &workspace_id, Some(&query)) + .await?; + tx.commit().await?; + + Ok(Json(triggers)) +} + +async fn get_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let trigger = handler + .get_trigger_by_path(&mut *tx, &workspace_id, path) + .await?; + + tx.commit().await?; + + Ok(Json(trigger)) +} + +async fn update_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(edit_trigger): Json>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &edit_trigger.base.path + ) + })?; + + handler + .validate_edit(&db, &workspace_id, &edit_trigger.config, path) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = edit_trigger.base.path.to_string(); + + handler + .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.update", T::TRIGGER_TYPE), + ActionKind::Update, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' updated", path)) +} + +async fn delete_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:write:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let deleted = handler + .delete_by_path(&mut *tx, &workspace_id, path) + .await?; + + if !deleted { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.delete", T::TRIGGER_TYPE), + ActionKind::Delete, + &workspace_id, + Some(&path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' deleted", path)) +} + +async fn exists_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), path) + })?; + let exists = handler.exists(&db, &workspace_id, path).await?; + + Ok(Json(exists)) +} + +#[derive(serde::Deserialize)] +struct SetTriggerModePayload { + mode: TriggerMode, +} + +async fn set_trigger_mode( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; + + let mut tx = user_db.begin(&authed).await?; + let updated = handler + .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) + .await?; + + if !updated { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(path.to_owned()), + Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), + true, + None, + ) + .await?; + + Ok(format!( + "Trigger '{}' {}", + path, + if payload.mode == TriggerMode::Enabled { + "enabled" + } else if payload.mode == TriggerMode::Disabled { + "disabled" + } else { + "suspended" + } + )) +} + +async fn test_connection( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(config): Json, +) -> Result<()> { + let connect_f = async move { + handler + .test_connection(&db, &authed, &user_db, &workspace_id, config) + .await + }; + + tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) + .await + .map_err(|_| { + Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) + })??; + Ok(()) +} + +#[allow(unused)] +pub fn complete_trigger_routes(handler: T) -> Router { + let standard_routes = trigger_routes::(); + + let additional_routes = handler.additional_routes(); + + standard_routes + .merge(additional_routes) + .layer(Extension(Arc::new(handler))) +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggerPrimarySchedule { + pub schedule: String, +} + +// generate_trigger_routers(), get_triggers_count_internal(), TriggersCount stay in windmill-api diff --git a/backend/windmill-trigger/src/lib.rs b/backend/windmill-trigger/src/lib.rs new file mode 100644 index 0000000000..690d9cc022 --- /dev/null +++ b/backend/windmill-trigger/src/lib.rs @@ -0,0 +1,19 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod capture; +pub mod filter; +pub mod global_handler; +pub mod handler; +pub mod listener; +pub mod trigger_helpers; +pub mod types; + +pub use handler::TriggerCrud; +pub use listener::Listener; +pub use types::*; diff --git a/backend/windmill-trigger/src/listener.rs b/backend/windmill-trigger/src/listener.rs new file mode 100644 index 0000000000..d09bb5e55a --- /dev/null +++ b/backend/windmill-trigger/src/listener.rs @@ -0,0 +1,864 @@ +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +use crate::{ + capture::insert_capture_payload, + handler::TriggerCrud, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, + types::{Trigger, TriggerErrorHandling, TriggerMode}, +}; +use async_trait::async_trait; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sql_builder::SqlBuilder; +use sqlx::{FromRow, Row}; +use tokio::sync::RwLock; +use windmill_api_auth::{fetch_api_authed, ApiAuthed}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + utils::report_critical_error, + DB, INSTANCE_NAME, +}; + +#[allow(unused)] +#[async_trait] +pub trait Listener: TriggerCrud + TriggerJobArgs { + type Consumer: Send; + type Extra: Send + Sync; + type ExtraState: Send + Sync; + + const JOB_TRIGGER_KIND: JobTriggerKind; + const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; + const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result>; + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, + ); + async fn fetch_enabled_unlistened_triggers( + &self, + db: &DB, + ) -> Result>> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + "error_handler_path", + "error_handler_args", + "retry", + ]; + + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + ); + + for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let triggers = triggers + .into_iter() + .map(|trigger| ListeningTrigger { + path: trigger.base.path, + workspace_id: trigger.base.workspace_id, + is_flow: trigger.base.is_flow, + username: trigger.base.edited_by, + email: trigger.base.email, + script_path: trigger.base.script_path, + trigger_config: trigger.config, + error_handling: Some(trigger.error_handling), + trigger_mode: true, + suspended_mode: trigger.base.mode == TriggerMode::Suspended, + }) + .collect_vec(); + + Ok(triggers) + } + + async fn fetch_unlistened_captures( + &self, + db: &DB, + ) -> Result>> { + let fields = vec![ + "path", + "is_flow", + "workspace_id", + "owner AS username", + "email", + "trigger_config", + ]; + + let mut sqlb = SqlBuilder::select_from("capture_config"); + sqlb.fields(&fields) + .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) + .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") + .and_where("trigger_config IS NOT NULL") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", + ); + + for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb.sql().expect("failed to build SQL"); + + let captures: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let captures = captures + .into_iter() + .map(|capture| ListeningTrigger { + username: capture.username, + path: capture.path, + workspace_id: capture.workspace_id, + script_path: "".to_string(), + email: capture.email, + trigger_config: capture.trigger_config, + trigger_mode: false, + is_flow: capture.is_flow, + error_handling: None, + suspended_mode: false, + }) + .collect_vec(); + + Ok(captures) + } + + async fn get_extra_state(&self) -> Option { + None + } + + async fn cleanup( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, + ) -> Result<()> { + Ok(()) + } + + async fn loop_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + status: Arc>>, + error_message: Option, + ) { + update_rw_lock(status.clone(), error_message).await; + loop { + if let None = self + .update_ping(db, listening_trigger, status.read().await.as_deref()) + .await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn update_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + if listening_trigger.trigger_mode { + self.update_trigger_ping(db, listening_trigger, error).await + } else { + self.update_capture_ping(db, listening_trigger, error).await + } + } + + async fn update_ping_and_loop_ping_status( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + loop_ping_status: Arc>>, + error: Option, + ) -> Option<()> { + // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status + update_rw_lock(loop_ping_status.clone(), error.clone()).await; + if let None = self + .update_ping(db, listening_trigger, error.as_deref()) + .await + { + return None; + } + Some(()) + } + + async fn update_trigger_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar::<_, i32>(&format!( + r#" + UPDATE + {} + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + server_id = $4 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + RETURNING 1 + "#, + Self::TABLE_NAME + )) + .bind(error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .bind(&*INSTANCE_NAME) + .fetch_optional(db) + .await; + + self.handle_ping_result(updated, db, listening_trigger, "trigger") + .await + } + + async fn update_capture_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + server_id = $6 AND + last_client_ping > NOW() - INTERVAL '10 seconds' + RETURNING 1 + "#, + error, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind, + &*INSTANCE_NAME + ) + .fetch_optional(db) + .await + .map(|result| result.flatten()); + + self.handle_ping_result(updated, db, listening_trigger, "capture") + .await + } + + async fn handle_ping_result( + &self, + result: sqlx::Result>, + db: &DB, + listening_trigger: &ListeningTrigger, + entity_type: &str, + ) -> Option<()> { + match result { + Ok(updated) => { + if updated.is_none() { + self.reset_ping_for_restart(db, listening_trigger).await; + tracing::info!( + "{} {} {} changed, disabled, or deleted, stopping...", + Self::TRIGGER_KIND, + entity_type, + listening_trigger.path + ); + return None; + } + } + Err(error) => { + tracing::warn!( + "Error updating ping of {} {} {}: {:?}", + Self::TRIGGER_KIND, + entity_type, + &listening_trigger.path, + error + ); + } + } + + Some(()) + } + + async fn reset_ping_for_restart( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) { + if listening_trigger.trigger_mode { + let _ = sqlx::query(&format!( + r#" + UPDATE + {} + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + server_id IS NULL + "#, + Self::TABLE_NAME + )) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + } else { + let _ = sqlx::query!( + r#" + UPDATE + capture_config + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + is_flow = $3 AND + trigger_kind = $4 AND + server_id IS NULL + "#, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + } + } + + async fn disable_with_error( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: String, + ) { + if listening_trigger.trigger_mode { + let report_status = sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = 'disabled'::TRIGGER_MODE, + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 + "#, + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + + match report_status { + Ok(_) => { + report_critical_error( + format!( + "Disabling {} trigger {} because of error: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + error + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + } + Err(disable_err) => { + report_critical_error( + format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ).await; + } + } + return; + } + + let report_status = sqlx::query!( + r#" + UPDATE + capture_config + SET + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 + "#, + error, + listening_trigger.workspace_id, + listening_trigger.path, + listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + + if let Err(disable_err) = report_status { + tracing::error!( + "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", + Self::TRIGGER_KIND, + listening_trigger.path, + listening_trigger.workspace_id, + disable_err, + error + ) + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + _extra: Option, + ) -> Result<()> { + let args = Self::build_job_args( + &listening_trigger.script_path, + listening_trigger.is_flow, + &listening_trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let (retry, error_handler_path, error_handler_args) = + match listening_trigger.error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + + tracing::debug!( + "Triggering job from {} event {} with args {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + args + ); + + trigger_runnable( + db, + None, + authed, + &listening_trigger.workspace_id, + &listening_trigger.script_path, + listening_trigger.is_flow, + args, + retry, + error_handler_path.as_deref(), + error_handler_args, + format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), + None, + listening_trigger.suspended_mode, + TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), + ) + .await?; + + Ok(()) + } + + async fn handle_event( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + if listening_trigger.trigger_mode { + if let Err(err) = self + .handle_trigger(db, listening_trigger, payload, trigger_info, extra) + .await + { + report_critical_error( + format!( + "Failed to trigger job from {} event {}: {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + return Err(err); + }; + return Ok(()); + } + + let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); + if let Err(err) = insert_capture_payload( + db, + &listening_trigger.workspace_id, + &listening_trigger.path, + listening_trigger.is_flow, + &Self::TRIGGER_KIND, + main_args, + preprocessor_args, + &listening_trigger.username, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); + return Err(err); + } + Ok(()) + } +} + +#[allow(unused)] +pub async fn listening( + db: DB, + listener: T, + listening_trigger: ListeningTrigger, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let killpill_rx_consumer = killpill_rx.resubscribe(); + let killpill_rx_get_consumer = killpill_rx.resubscribe(); + + let loop_ping_status = Arc::new(RwLock::new(None)); + let extra_state = listener.get_extra_state().await; + let path = listening_trigger.path.clone(); + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + consumer = { + tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) + } => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { + tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = async { + match consumer { + Ok(Some(consumer)) => { + listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; + tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; + tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); + } + Err(error) => { + tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); + listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; + } + Ok(None) => { + tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); + } + } + } => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + } + } + } +} + +#[allow(unused)] +pub async fn listen_to_unlistened_events( + listener: T, + db: DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; + + match unlistend_enabled_triggers { + Ok(mut unlistend_enabled_triggers) => { + unlistend_enabled_triggers.shuffle(&mut rand::rng()); + for trigger in unlistend_enabled_triggers { + let has_lock = sqlx::query_scalar(&format!( + r#" + UPDATE + {} + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + T::TABLE_NAME, + )) + .bind(&*INSTANCE_NAME) + .bind(&trigger.workspace_id) + .bind(&trigger.path) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!( + "Spawning new task to listen for {} event", + T::TABLE_NAME + ); + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, trigger, killpill_rx).await } + }); + } else { + tracing::info!( + "{} trigger {} already being listened to", + T::TRIGGER_KIND, + trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for {} trigger {}: {:?}", + T::TRIGGER_KIND, + trigger.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); + } + } + + let unlisted_captures = listener.fetch_unlistened_captures(&db).await; + + match unlisted_captures { + Ok(unlistened_captures) => { + for capture in unlistened_captures { + let has_lock = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + last_client_ping > NOW() - INTERVAL '10 seconds' AND + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') + RETURNING true + "#, + *INSTANCE_NAME, + &capture.workspace_id, + &capture.path, + &capture.is_flow, + T::TRIGGER_KIND as TriggerKind + ) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, capture, killpill_rx).await } + }); + } else { + tracing::info!( + "{} capture {} already being listened to", + T::TRIGGER_KIND.to_string(), + capture.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for capture {} {}: {:?}", + T::TRIGGER_KIND, + capture.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!( + "Error fetching captures {} triggers: {:?}", + T::TRIGGER_KIND, + err + ); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + path: String, + is_flow: bool, + workspace_id: String, + username: String, + email: String, + #[serde(flatten)] + trigger_config: T, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let trigger_config_value = row.try_get("trigger_config")?; + let trigger_config: T = serde_json::from_value(trigger_config_value) + .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; + Ok(Capture { + path: row.try_get("path")?, + is_flow: row.try_get("is_flow")?, + workspace_id: row.try_get("workspace_id")?, + username: row.try_get("username")?, + email: row.try_get("email")?, + trigger_config, + }) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListeningTrigger { + pub path: String, + pub is_flow: bool, + pub workspace_id: String, + pub username: String, + pub email: String, + pub trigger_config: T, + pub script_path: String, + pub trigger_mode: bool, + pub error_handling: Option, + pub suspended_mode: bool, +} + +impl ListeningTrigger { + pub async fn authed(&self, db: &DB, username: &str) -> Result { + fetch_api_authed( + self.username.clone(), + self.email.clone(), + &self.workspace_id, + db, + Some(format!("{}-{}", username, self.path)), + ) + .await + } +} + +#[allow(unused)] +pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { + let mut w = lock.write().await; + *w = value; +} + +#[allow(unused)] +fn listen_to( + trigger: T, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tokio::spawn(async move { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await + } + } + } + }); +} + +// start_all_listeners() stays in windmill-api since it references concrete trigger types diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs similarity index 99% rename from backend/windmill-api/src/triggers/trigger_helpers.rs rename to backend/windmill-trigger/src/trigger_helpers.rs index c533dcaaeb..f1b7d18fa5 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use std::future::Future; use uuid::Uuid; use windmill_common::{ - db::{UserDB, UserDbWithAuthed}, + db::{UserDB, UserDbWithAuthed, DB}, error::Result, flows::{FlowModuleValue, Retry}, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, @@ -19,22 +19,21 @@ use windmill_common::{ RUNNABLE_FORMAT_VERSION_CACHE, }, users::username_to_permissioned_as, - utils::StripPath, + utils::{StripPath, HTTP_CLIENT}, worker::to_raw_value, }; use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; +use windmill_api_auth::{check_scopes, ApiAuthed}; #[cfg(feature = "enterprise")] -use crate::jobs::check_license_key_valid; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ +use windmill_api_jobs::execution::check_license_key_valid; +use windmill_api_jobs::{ + execution::{ check_tag_available_for_workspace, delete_job_metadata_after_use, push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response, - run_wait_result_internal, RunJobQuery, + run_wait_result_internal, }, - utils::check_scopes, - HTTP_CLIENT, + types::RunJobQuery, }; struct ScriptInfo { diff --git a/backend/windmill-trigger/src/types.rs b/backend/windmill-trigger/src/types.rs new file mode 100644 index 0000000000..07feae5831 --- /dev/null +++ b/backend/windmill-trigger/src/types.rs @@ -0,0 +1,158 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use std::{collections::HashMap, fmt::Debug}; +use windmill_common::jobs::JobTriggerKind; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HandlerAction { + Trigger { path: String, trigger_kind: JobTriggerKind }, + // Future variants can be added here (e.g., Script, Flow, etc.) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StandardTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct BaseTrigger { + pub workspace_id: String, + pub path: String, + pub script_path: String, + pub mode: TriggerMode, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: DateTime, + pub extra_perms: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct ServerState { + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct TriggerErrorHandling { + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + #[serde(flatten)] + pub base: BaseTrigger, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub server_state: Option, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let base = BaseTrigger::from_row(row)?; + + Ok(Trigger { + base, + config: T::from_row(row)?, + server_state: ServerState::from_row(row).ok(), + error_handling: TriggerErrorHandling::from_row(row)?, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BaseTriggerData { + pub path: String, + pub script_path: String, + pub is_flow: bool, + #[deprecated(note = "Use mode instead")] + enabled: Option, // Kept for backwards compatibility, use mode instead + mode: Option, +} + +impl BaseTriggerData { + pub fn mode(&self) -> &TriggerMode { + self.mode.as_ref().unwrap_or( + #[allow(deprecated)] + if self.enabled.unwrap_or(true) { + &TriggerMode::Enabled + } else { + &TriggerMode::Disabled + }, + ) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TriggerData { + #[serde(flatten)] + pub base: BaseTriggerData, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl StandardTriggerQuery { + pub fn offset(&self) -> i64 { + let page = self.page.unwrap_or(0); + let per_page = self.per_page.unwrap_or(100); + (page * per_page) as i64 + } + + pub fn limit(&self) -> i64 { + self.per_page.unwrap_or(100) as i64 + } +} + +impl Default for StandardTriggerQuery { + fn default() -> Self { + Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] +#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum TriggerMode { + Enabled, + Disabled, + Suspended, +}