From f68ecffd6d2e2cca58e063c2d5aa3abf2d9345e0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 21 Sep 2023 15:07:15 +0200 Subject: [PATCH 01/32] update docker-compose --- docker-compose.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4f236ee163..1ecfd7492b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: windmill_server: image: ${WM_IMAGE} - pull_policy: always + # pull_policy: always deploy: replicas: 1 restart: unless-stopped @@ -41,12 +41,15 @@ services: db: condition: service_healthy - windmill_worker: image: ${WM_IMAGE} - pull_policy: always + # pull_policy: always deploy: replicas: 3 + resources: + limits: + cpus: "1" + memory: 2048M restart: unless-stopped environment: - DATABASE_URL=${DATABASE_URL} @@ -66,30 +69,28 @@ services: - /var/run/docker.sock:/var/run/docker.sock - worker_dependency_cache:/tmp/windmill/cache - ## This worker is specialized for "native" jobs. They run in-process and can thus be parallelized to more than 1 at a time on a given worker which is why NUM_WORKERS is set to 4 + ## This worker is specialized for "native" jobs. Native jobs run in-process and thus are much more lightweight than other jobs windmill_worker_native: # Use ghcr.io/windmill-labs/windmill-ee:main for the ee image: ${WM_IMAGE} - pull_policy: always + # pull_policy: always deploy: - replicas: 1 + replicas: 2 resources: - limits: - cpus: "0.25" - memory: 512M + limits: + cpus: "0.1" + memory: 128M restart: unless-stopped environment: - DATABASE_URL=${DATABASE_URL} - RUST_LOG=info - DISABLE_SERVER=true - METRICS_ADDR=false # (ee only, if set to true, metrics will be exposed on port 8001) - - NUM_WORKERS=4 - WORKER_GROUP=native depends_on: db: condition: service_healthy - lsp: image: ghcr.io/windmill-labs/windmill-lsp:latest restart: unless-stopped From b9ab5d8fc9a625f43b7c3a41797e17711e348a8a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 21 Sep 2023 15:08:02 +0200 Subject: [PATCH 02/32] update docker-compose --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1ecfd7492b..e997e0e0e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: windmill_server: image: ${WM_IMAGE} - # pull_policy: always + pull_policy: always deploy: replicas: 1 restart: unless-stopped @@ -43,7 +43,7 @@ services: windmill_worker: image: ${WM_IMAGE} - # pull_policy: always + pull_policy: always deploy: replicas: 3 resources: @@ -73,7 +73,7 @@ services: windmill_worker_native: # Use ghcr.io/windmill-labs/windmill-ee:main for the ee image: ${WM_IMAGE} - # pull_policy: always + pull_policy: always deploy: replicas: 2 resources: From 95194abeacc42416174ee9dd79b75f2204a40d33 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 21 Sep 2023 17:11:06 +0200 Subject: [PATCH 03/32] feat: ai flow trigger menu (#2317) * feat: add trigger ai menu * fix: ai specify resource param name format --- .../src/lib/components/FlowBuilder.svelte | 2 +- .../copilot/FlowCopilotDrawer.svelte | 2 +- .../src/lib/components/copilot/StepGen.svelte | 10 ++-- frontend/src/lib/components/copilot/flow.ts | 4 +- .../lib/components/copilot/prompts/edit.yaml | 8 +-- .../components/copilot/prompts/editPrompt.ts | 8 +-- .../lib/components/copilot/prompts/fix.yaml | 8 +-- .../components/copilot/prompts/fixPrompt.ts | 8 +-- .../lib/components/copilot/prompts/gen.yaml | 8 +-- .../components/copilot/prompts/genPrompt.ts | 8 +-- .../flows/map/InsertTriggerButton.svelte | 51 +++++++++++++++++++ .../components/flows/map/VirtualItem.svelte | 31 +++++++---- 12 files changed, 105 insertions(+), 43 deletions(-) create mode 100644 frontend/src/lib/components/flows/map/InsertTriggerButton.svelte diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 1c4251403c..8387e59f3a 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -421,7 +421,7 @@ let abortController: AbortController | undefined = undefined let copilotLoading = false - let flowCopilotMode: 'trigger' | 'sequence' = 'trigger' + let flowCopilotMode: 'trigger' | 'sequence' = 'sequence' let copilotStatus: string = '' let copilotFlowInputs: Record = {} let copilotFlowRequiredInputs: string[] = [] diff --git a/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte b/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte index 00b67c2869..fdb75419ec 100644 --- a/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte +++ b/frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte @@ -37,8 +37,8 @@ } }} > - + {#each $modulesStore as copilotModule, i}
diff --git a/frontend/src/lib/components/copilot/StepGen.svelte b/frontend/src/lib/components/copilot/StepGen.svelte index 9c64779f2d..3c0887164f 100644 --- a/frontend/src/lib/components/copilot/StepGen.svelte +++ b/frontend/src/lib/components/copilot/StepGen.svelte @@ -14,6 +14,7 @@ export let close: () => void export let funcDesc: string export let modules: FlowModule[] + export let trigger = false // state let input: HTMLInputElement | undefined @@ -31,7 +32,8 @@ const ts = Date.now() const scriptIds = await ScriptService.queryHubScripts({ text: `${text}`, - limit: 3 + limit: 3, + kind: trigger ? 'trigger' : 'script' }) if (ts < doneTs) return doneTs = ts @@ -60,7 +62,7 @@ $copilotModulesStore = [ { id: nextId($flowStateStore, $flowStore), - type: 'script', + type: trigger ? 'trigger' : 'script', description: funcDesc, code: '', source: selectedCompletion ? 'hub' : 'custom', @@ -96,7 +98,7 @@ hubCompletions = [] } }} - placeholder="AI Gen or search hub scripts" + placeholder="AI Gen or search hub {trigger ? 'triggers' : 'scripts'}" /> {#if funcDesc.length === 0} 0}
-

Hub Scripts

+

Hub {trigger ? 'Triggers' : 'Scripts'}

    {#each hubCompletions as item (item.path)}
  • diff --git a/frontend/src/lib/components/copilot/flow.ts b/frontend/src/lib/components/copilot/flow.ts index f34a2eed82..e37dc3e28b 100644 --- a/frontend/src/lib/components/copilot/flow.ts +++ b/frontend/src/lib/components/copilot/flow.ts @@ -56,7 +56,7 @@ const additionalInfos: { bun: ` We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. If needed, the standard fetch method is available globally, do not import it. -You can take as parameters resources which are dictionaries containing credentials or configuration information. +You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}_resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -65,7 +65,7 @@ Only define the type for resources that are actually needed to achieve the funct `, python3: ` We have to export a "main" function and specify the parameter types but do not call it. -You can take as parameters resources which are dictionaries containing credentials or configuration information. +You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified (has to be IN LOWERCASE). {resourceTypes} diff --git a/frontend/src/lib/components/copilot/prompts/edit.yaml b/frontend/src/lib/components/copilot/prompts/edit.yaml index f043d104b5..f9bd3b4ae8 100644 --- a/frontend/src/lib/components/copilot/prompts/edit.yaml +++ b/frontend/src/lib/components/copilot/prompts/edit.yaml @@ -16,7 +16,7 @@ prompts: ``` We have to export a "main" function and specify the parameter types but do not call it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}_resource". The resource type name has to be exactly as specified (has to be IN LOWERCASE). {resourceTypes} @@ -33,7 +33,7 @@ prompts: We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -133,7 +133,7 @@ prompts: ``` We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -150,7 +150,7 @@ prompts: We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} diff --git a/frontend/src/lib/components/copilot/prompts/editPrompt.ts b/frontend/src/lib/components/copilot/prompts/editPrompt.ts index 85870d023e..639083f045 100644 --- a/frontend/src/lib/components/copilot/prompts/editPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/editPrompt.ts @@ -2,10 +2,10 @@ export const EDIT_PROMPT = { "system": "You write code as instructed by the user. Only output code. Wrap the code in a code block. \nPut explanations directly in the code as comments.\n\nHere's how interactions have to look like:\nuser: {sample_question}\nassistant: ```language\n{code}\n```", "prompts": { "python3": { - "prompt": "Here's my python3 code: \n```python\n{code}\n```\n\nWe have to export a \"main\" function and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" + "prompt": "Here's my python3 code: \n```python\n{code}\n```\n\nWe have to export a \"main\" function and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}_resource\". \nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" }, "deno": { - "prompt": "Here's my TypeScript code in a deno running environment:\n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" + "prompt": "Here's my TypeScript code in a deno running environment:\n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" }, "go": { "prompt": "Here's my go code: \n```go\n{code}\n```\n\nWe have to export a \"main\" function. Import the packages you need. The return type of the function has to be ({return_type}, error). The file package has to be \"inner\"\n\nMy instructions: {description}" @@ -32,10 +32,10 @@ export const EDIT_PROMPT = { "prompt": "Here's my powershell code: \n```powershell\n{code}\n```\n\nArguments can be obtained by calling the param function on the first line like that: `param($ParamName1, $ParamName2 = \"default value\", [{type}]$ParamName3, ...)`\nI get the following error: {error}\n\nMy instructions: {description}" }, "nativets": { - "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" + "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" }, "bun": { - "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" + "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nMy instructions: {description}" }, "frontend": { "prompt": "Here's my client-side javascript code: \n```javascript\n{code}\n```\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nUse the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nUse the recompute function to recompute a component: recompute(id: string)\nUse the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any).\n\nMy instructions: {description}" diff --git a/frontend/src/lib/components/copilot/prompts/fix.yaml b/frontend/src/lib/components/copilot/prompts/fix.yaml index 5b4a58a1e9..6afd0b77f2 100644 --- a/frontend/src/lib/components/copilot/prompts/fix.yaml +++ b/frontend/src/lib/components/copilot/prompts/fix.yaml @@ -18,7 +18,7 @@ prompts: ``` We have to export a "main" function and specify the parameter types but do not call it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}_resource". The resource type name has to be exactly as specified (has to be IN LOWERCASE). {resourceTypes} @@ -36,7 +36,7 @@ prompts: We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -141,7 +141,7 @@ prompts: ``` We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -159,7 +159,7 @@ prompts: We have to export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} diff --git a/frontend/src/lib/components/copilot/prompts/fixPrompt.ts b/frontend/src/lib/components/copilot/prompts/fixPrompt.ts index 956bd7adfa..44cd016dbc 100644 --- a/frontend/src/lib/components/copilot/prompts/fixPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/fixPrompt.ts @@ -2,10 +2,10 @@ export const FIX_PROMPT = { "system": "You fix the code shared by the user. Only output code. Wrap the code in a code block. \nExplain the error and the fix after generating the code inside an tag.\nAlso put explanations directly in the code as comments.\n\nHere's how interactions have to look like:\nuser: {sample_question}\nassistant: ```language\n{code}\n```\n{explanation}", "prompts": { "python3": { - "prompt": "Here's my python3 code: \n```python\n{code}\n```\n\nWe have to export a \"main\" function and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my python3 code: \n```python\n{code}\n```\n\nWe have to export a \"main\" function and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}_resource\". \nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." }, "deno": { - "prompt": "Here's my TypeScript code in a deno running environment:\n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my TypeScript code in a deno running environment:\n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." }, "go": { "prompt": "Here's my go code: \n```go\n{code}\n```\n\nWe have to export a \"main\" function. Import the packages you need. The return type of the function has to be ({return_type}, error). The file package has to be \"inner\"\n\nI get the following error: {error}\nFix my code." @@ -32,10 +32,10 @@ export const FIX_PROMPT = { "prompt": "Here's my powershell code: \n```powershell\n{code}\n```\n\nArguments can be obtained by calling the param function on the first line like that: `param($ParamName1, $ParamName2 = \"default value\", [{type}]$ParamName3, ...)`\n\nI get the following error: {error}\nFix my code." }, "nativets": { - "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." }, "bun": { - "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my TypeScript code: \n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\n\nI get the following error: {error}\nFix my code." } } }; \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/prompts/gen.yaml b/frontend/src/lib/components/copilot/prompts/gen.yaml index 8725eaf4c4..9b79e5b8f9 100644 --- a/frontend/src/lib/components/copilot/prompts/gen.yaml +++ b/frontend/src/lib/components/copilot/prompts/gen.yaml @@ -11,7 +11,7 @@ prompts: python3: prompt: |- Write a function in python called "main". The function should {description}. Specify the parameter types. Do not call the main function. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}_resource". The resource type name has to be exactly as specified (has to be IN LOWERCASE). {resourceTypes} @@ -21,7 +21,7 @@ prompts: prompt: |- Write a function in TypeScript called "main". The function should {description}. Specify the parameter types. You are in a Deno environment. You can import deno libraries or you can also import npm libraries like that: "import ... from "npm:{package}";". Export the "main" function like this: "export async function main(...)". Do not call the main function. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -54,7 +54,7 @@ prompts: nativets: prompt: |- Write a function in TypeScript called "main". The function should {description}. Specify the parameter types. You should use fetch and are not allowed to import any libraries. Export the "main" function like this: "export async function main(...)". Do not call the main function. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} @@ -64,7 +64,7 @@ prompts: prompt: |- Write a function in TypeScript called "main". The function should {description}. Specify the parameter types. You can import npm libraries. Export the "main" function like this: "export async function main(...)". Do not call the main function. If needed, the standard fetch method is available globally, do not import it. - You can take as parameters resources which are dictionaries containing credentials or configuration information. + You can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: "{resource_type}Resource". The resource type name has to be exactly as specified. {resourceTypes} diff --git a/frontend/src/lib/components/copilot/prompts/genPrompt.ts b/frontend/src/lib/components/copilot/prompts/genPrompt.ts index 034c4456a1..cd088a7c20 100644 --- a/frontend/src/lib/components/copilot/prompts/genPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/genPrompt.ts @@ -2,10 +2,10 @@ export const GEN_PROMPT = { "system": "You write code as instructed by the user. Only output code. Wrap the code in a code block. \nPut explanations directly in the code as comments.\n\nHere's how interactions have to look like:\nuser: {sample_question}\nassistant: ```language\n{code}\n```", "prompts": { "python3": { - "prompt": "Write a function in python called \"main\". The function should {description}. Specify the parameter types. Do not call the main function.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." + "prompt": "Write a function in python called \"main\". The function should {description}. Specify the parameter types. Do not call the main function.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}_resource\".\nThe resource type name has to be exactly as specified (has to be IN LOWERCASE).\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." }, "deno": { - "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You are in a Deno environment. You can import deno libraries or you can also import npm libraries like that: \"import ... from \"npm:{package}\";\". Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." + "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You are in a Deno environment. You can import deno libraries or you can also import npm libraries like that: \"import ... from \"npm:{package}\";\". Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\".\nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." }, "go": { "prompt": "Write a function in go called \"main\". The function should {description}. Import the packages you need. The return type of the function has to be ({return_type}, error). The file package has to be \"inner\"." @@ -32,10 +32,10 @@ export const GEN_PROMPT = { "prompt": "Write powershell code that should {description}. Arguments can be obtained by calling the param function on the first line like that: `param($ParamName1, $ParamName2 = \"default value\", [{type}]$ParamName3, ...)`" }, "nativets": { - "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You should use fetch and are not allowed to import any libraries. Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information.\nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." + "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You should use fetch and are not allowed to import any libraries. Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\".\nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." }, "bun": { - "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You can import npm libraries. Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." + "prompt": "Write a function in TypeScript called \"main\". The function should {description}. Specify the parameter types. You can import npm libraries. Export the \"main\" function like this: \"export async function main(...)\". Do not call the main function.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\".\nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE." }, "frontend": { "prompt": "Write client-side javascript code that should {description}. You have access to a few helpers:\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nUse the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nUse the recompute function to recompute a component: recompute(id: string)\nUse the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)." diff --git a/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte b/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte new file mode 100644 index 0000000000..2ee13c058c --- /dev/null +++ b/frontend/src/lib/components/flows/map/InsertTriggerButton.svelte @@ -0,0 +1,51 @@ + + + + + + {#if funcDesc.length === 0} +
    + +
    + {/if} +     diff --git a/frontend/src/lib/components/flows/map/VirtualItem.svelte b/frontend/src/lib/components/flows/map/VirtualItem.svelte index cb984dde3c..4c7d9d9517 100644 --- a/frontend/src/lib/components/flows/map/VirtualItem.svelte +++ b/frontend/src/lib/components/flows/map/VirtualItem.svelte @@ -2,7 +2,7 @@ import { Badge } from '$lib/components/common' import type { FlowModule } from '$lib/gen' import { classNames } from '$lib/utils' - import { faBolt, faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons' + import { faMagicWandSparkles } from '@fortawesome/free-solid-svg-icons' import { ClipboardCopy, ExternalLink, X } from 'lucide-svelte' import { createEventDispatcher, getContext } from 'svelte' import { Icon } from 'svelte-awesome' @@ -10,6 +10,7 @@ import type { FlowCopilotContext } from '$lib/components/copilot/flow' import { existsOpenaiResourcePath } from '$lib/stores' import Menu from '$lib/components/common/menu/Menu.svelte' + import InsertTriggerButton from './InsertTriggerButton.svelte' export let label: string export let modules: FlowModule[] | undefined @@ -34,6 +35,7 @@ deleteBranch: { module: FlowModule; index: number } }>() let openMenu = false + let triggerOpenMenu = false let openNoCopilot = false const { drawerStore: copilotDrawerStore, currentStepStore: copilotCurrentStepStore } = @@ -57,6 +59,7 @@
{/if} +
-
- + index={0} + modules={modules ?? []} + />
{/if} From fc93c2a7cece95c00070a3a3391ae2bcb4513e85 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 21 Sep 2023 17:12:00 +0200 Subject: [PATCH 04/32] feat: improved dedicated benchmarks + buffer fix (#2313) * feat: improved dedicated benchmarks + buffer fix * fixes + limit task spawning to noop/dedicated * fix: cargo test --------- Co-authored-by: Ruben Fiszel --- ...55ddd99c4ad1f46b876dd86e372b84d806ecd.json | 1 - ...17cbcf1013ae311c36b42d423bf6a02fa016c.json | 1 - ...04e797527841cd97dba80c271cbefafae65cc.json | 1 - ...ee85b191263989b0c78b2bfce77e796e96825.json | 1 - ...f6f580f15bca96d9746c9359e98ca793f8f1f.json | 1 - ...10de88172c0c748a45aba4d2d03c3b58f54d.json} | 5 +- ...6e03e870ccc3b353401439bc0ed8ff219249b.json | 1 - ...5eedc09bceb82bca349f3e31c8513ebbf0192.json | 1 - backend/src/monitor.rs | 2 +- backend/windmill-api/src/jobs.rs | 14 +- backend/windmill-common/src/scripts.rs | 1 + backend/windmill-worker/src/bun_executor.rs | 23 +- backend/windmill-worker/src/worker.rs | 260 +++++++++++++++--- benchmarks/benchmark_oneoff.ts | 158 +++++++---- benchmarks/lib.ts | 27 +- 15 files changed, 386 insertions(+), 111 deletions(-) rename backend/.sqlx/{query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json => query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json} (73%) diff --git a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json index 1b8084742c..7718e05ccf 100644 --- a/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json +++ b/backend/.sqlx/query-123c0608e229c29187009b7961355ddd99c4ad1f46b876dd86e372b84d806ecd.json @@ -37,7 +37,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json index bf591ef11c..9d082a6772 100644 --- a/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json +++ b/backend/.sqlx/query-25bef6a248f3ee0ea2cbcc376c217cbcf1013ae311c36b42d423bf6a02fa016c.json @@ -67,7 +67,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json index a5dee163e5..1166260449 100644 --- a/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json +++ b/backend/.sqlx/query-438b5b5d29b05846c2e074cad2404e797527841cd97dba80c271cbefafae65cc.json @@ -28,7 +28,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json index 1517c8d1d4..eabf671894 100644 --- a/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json +++ b/backend/.sqlx/query-5cd89ab614d3cac80fb81627267ee85b191263989b0c78b2bfce77e796e96825.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json index bfe7c41f64..c52efca4c0 100644 --- a/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json +++ b/backend/.sqlx/query-65835f2e5ad38f7cc6b147dadfef6f580f15bca96d9746c9359e98ca793f8f1f.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json similarity index 73% rename from backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json rename to backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json index 9c386c3509..73e6e17c76 100644 --- a/backend/.sqlx/query-2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40.json +++ b/backend/.sqlx/query-6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11)) RETURNING id", + "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $11)\n )\n INSERT INTO queue \n (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id)\n (SELECT uuid, $1, $2, $3, $4, ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) \n RETURNING id", "describe": { "columns": [ { @@ -46,7 +46,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", @@ -70,5 +69,5 @@ false ] }, - "hash": "2de52e1f3226ca9281b6e25f74d4d05f4509cb87c875234bfc7b310a012e4d40" + "hash": "6b9ff3fbca9e825c95d14705082a10de88172c0c748a45aba4d2d03c3b58f54d" } diff --git a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json index 6308bf3bb2..fead4ba250 100644 --- a/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json +++ b/backend/.sqlx/query-9e8c3ff3d6b31e366e15beda1e96e03e870ccc3b353401439bc0ed8ff219249b.json @@ -60,7 +60,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json index 38f81da395..c90719118a 100644 --- a/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json +++ b/backend/.sqlx/query-b224cdd1221fc9e7227ef8e8c025eedc09bceb82bca349f3e31c8513ebbf0192.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 659f9213b9..b2ba7d2b13 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -63,7 +63,7 @@ pub async fn initial_load( db: &Pool, tx: tokio::sync::broadcast::Sender<()>, worker_mode: bool, - server_mode: bool, + server_mode: bool ) { let reload_worker_config_f = async { if worker_mode { diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b97a06c747..2457e543a9 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2332,7 +2332,6 @@ struct BatchInfo { kind: String, flow_value: Option, path: Option, - dedicated_worker: Option, } #[tracing::instrument(level = "trace", skip_all)] @@ -2362,7 +2361,7 @@ async fn add_batch_jobs( batch_info.path, JobKind::Script, Some(script.language), - batch_info.dedicated_worker, + script.dedicated_worker, ) } "flow" => { @@ -2433,15 +2432,22 @@ async fn add_batch_jobs( format!("{}", language.as_str()) }; - let uuids = sqlx::query_scalar!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11)) RETURNING id", + let uuids = sqlx::query_scalar!( + r#"WITH uuid_table as ( + select gen_random_uuid() as uuid from generate_series(1, $11) + ) + INSERT INTO queue + (id, script_hash, script_path, job_kind, language, args, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) + (SELECT uuid, $1, $2, $3, $4, ('{ "uuid": "' || uuid || '" }')::jsonb, $5, $6, $7, $8, $9, $10 FROM uuid_table) + RETURNING id"#, hash.map(|h| h.0), path, job_kind.clone() as JobKind, language as ScriptLang, tag, authed.username, - authed.email, username_to_permissioned_as(&authed.username), + authed.email, Utc::now(), w_id, n diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 960cdf3a98..af17eae543 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -161,6 +161,7 @@ pub struct Script { pub concurrent_limit: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_time_window_s: Option, + pub dedicated_worker: Option, } #[derive(Serialize)] diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d46ab7bbb6..df9e4721d8 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -464,6 +464,10 @@ pub async fn start_worker( mut jobs_rx: Receiver, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> Result<()> { + use std::task::Poll; + + use futures::{future, Future}; + let mut logs = "".to_string(); let _ = write_file(job_dir, "main.ts", inner_content).await?; let common_bun_proc_envs: HashMap = @@ -684,6 +688,21 @@ plugin(p) // let mut i = 0; // let mut j = 0; let mut alive = true; + + fn conditional_polling( + fut: impl Future, + predicate: bool, + ) -> impl Future { + let mut fut = Box::pin(fut); + future::poll_fn(move |cx| { + if predicate { + fut.as_mut().poll(cx) + } else { + Poll::Pending + } + }) + } + loop { tokio::select! { biased; @@ -711,8 +730,8 @@ plugin(p) tracing::info!("dedicated worker process exited"); break; } - } - job = jobs_rx.recv(), if alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS => { + }, + job = conditional_polling(jobs_rx.recv(), alive && jobs.len() < MAX_BUFFERED_DEDICATED_JOBS) => { // i += 1; if let Some(job) = job { tracing::debug!("received job"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b3fa1fe62e..072202134b 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -10,12 +10,16 @@ use anyhow::Result; use const_format::concatcp; use itertools::Itertools; use once_cell::sync::OnceCell; +use prometheus::core::{AtomicU64, GenericCounter}; #[cfg(feature = "benchmark")] use serde::Serialize; use sqlx::{Pool, Postgres}; use std::{ collections::HashMap, - sync::{atomic::Ordering, Arc}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, time::Duration, }; use windmill_api_client::Client; @@ -337,6 +341,52 @@ macro_rules! add_time { }; } +async fn handle_receive_completed_job< + R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static, +>( + jc: JobCompleted, + worker_execution_failed: HashMap, GenericCounter>, + base_internal_url: String, + db: Pool, + worker_dir: String, + same_worker_tx: Sender, + rsmq: Option, +) { + let metrics = build_language_metrics(&worker_execution_failed.clone(), &jc.job.language); + let token = jc.token.clone(); + let workspace = jc.job.workspace_id.clone(); + let client = AuthedClient { + base_internal_url: base_internal_url.to_string(), + workspace, + token, + client: OnceCell::new(), + }; + if let Err(err) = process_completed_job( + &jc, + &client, + &db, + &worker_dir, + metrics.clone(), + same_worker_tx.clone(), + rsmq.clone(), + ) + .await + { + handle_job_error( + &db, + &client, + &jc.job, + err, + metrics, + false, + same_worker_tx.clone(), + &worker_dir, + rsmq.clone(), + ) + .await; + } +} + pub async fn run_worker( db: &Pool, worker_instance: &str, @@ -549,43 +599,168 @@ pub async fn run_worker, + None::, + JobKind::Noop as JobKind, + ScriptLang::Deno as ScriptLang, + "deno", + "admin", + "u/admin", + "admin@windmill.dev", + chrono::Utc::now(), + "admins", + jobs + ) + .execute(db) + .await.unwrap_or_else(|_e| panic!("failed to insert noop jobs")); + } + } + + #[cfg(feature = "benchmark")] + let completed_jobs = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let start = Instant::now(); + #[cfg(feature = "benchmark")] + let main_duration = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let send_duration = Arc::new(AtomicUsize::new(0)); + #[cfg(feature = "benchmark")] + let process_duration = Arc::new(AtomicUsize::new(0)); + + #[cfg(feature = "benchmark")] + let main_duration2 = main_duration.clone(); + #[cfg(feature = "benchmark")] + let send_duration2 = send_duration.clone(); + let send_result = tokio::spawn(async move { while let Some(jc) = job_completed_rx.recv().await { - let metrics = build_language_metrics(&worker_execution_failed2, &jc.job.language); - let token = jc.token.clone(); - let workspace = jc.job.workspace_id.clone(); - let client = AuthedClient { - base_internal_url: base_internal_url2.to_string(), - workspace, - token, - client: OnceCell::new(), - }; - if let Err(err) = process_completed_job( - &jc, - &client, - &db2, - &worker_dir2, - metrics.clone(), - same_worker_tx2.clone(), - rsmq2.clone(), - ) - .await - { - handle_job_error( - &db2, - &client, - &jc.job, - err, - metrics, - false, - same_worker_tx2.clone(), - &worker_dir2, - rsmq2.clone(), + let base_internal_url2 = base_internal_url2.clone(); + let worker_execution_failed2 = worker_execution_failed2.clone(); + let worker_dir2 = worker_dir2.clone(); + let db2 = db2.clone(); + let same_worker_tx2 = same_worker_tx2.clone(); + let rsmq2 = rsmq2.clone(); + + if matches!(jc.job.job_kind, JobKind::Noop) || is_dedicated_worker { + thread_count.fetch_add(1, Ordering::SeqCst); + let thread_count = thread_count.clone(); + + #[cfg(feature = "benchmark")] + let send_duration = send_duration2.clone(); + #[cfg(feature = "benchmark")] + let process_duration = process_duration.clone(); + #[cfg(feature = "benchmark")] + let completed_jobs = completed_jobs.clone(); + #[cfg(feature = "benchmark")] + let main_duration = main_duration2.clone(); + + tokio::spawn(async move { + #[cfg(feature = "benchmark")] + let process_start = Instant::now(); + + handle_receive_completed_job( + jc, + worker_execution_failed2, + base_internal_url2, + db2, + worker_dir2, + same_worker_tx2, + rsmq2, + ) + .await; + #[cfg(feature = "benchmark")] + { + let n = completed_jobs.fetch_add(1, Ordering::SeqCst); + if (n + 1) % 1000 == 0 || n == (jobs - 1) as usize { + let duration_s = start.elapsed().as_secs_f64(); + let jobs_per_sec = n as f64 / duration_s; + tracing::info!( + "completed {} jobs in {}s, {} jobs/s", + n + 1, + duration_s, + jobs_per_sec + ); + + tracing::info!( + "main loop without send {}s", + main_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + + tracing::info!( + "send job completed / send dedicated job duration {}s", + send_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + + tracing::info!( + "job completed process duration {}s", + process_duration.load(Ordering::SeqCst) as f64 / 1000.0 + ); + } + + process_duration.fetch_add( + process_start.elapsed().as_millis() as usize, + Ordering::SeqCst, + ); + } + + thread_count.fetch_sub(1, Ordering::SeqCst); + }); + } else { + handle_receive_completed_job( + jc, + worker_execution_failed2, + base_internal_url2, + db2, + worker_dir2, + same_worker_tx2, + rsmq2, ) .await; } } + tracing::info!("stopped processing new completed jobs"); + while thread_count.load(Ordering::SeqCst) > 0 { + tokio::time::sleep(Duration::from_millis(100)).await; + } + tracing::info!("finished processing all completed jobs"); + // if let Err(e) = // add_completed_job(&db2, &job, success, false, result, logs, rsmq2.clone()).await // { @@ -719,6 +894,9 @@ pub async fn run_worker>, Option>) }; + #[cfg(feature = "benchmark")] + tracing::info!("pre loop time {}s", start.elapsed().as_secs_f64()); + loop { #[cfg(feature = "benchmark")] let loop_start = Instant::now(); @@ -889,11 +1067,27 @@ pub async fn run_worker new TextEncoder().encode(s); + async function getQueueCount() { + return ( + await ( + await fetch( + config.server + "/api/w/" + config.workspace_id + "/jobs/queue/count", + { headers: { ["Authorization"]: "Bearer " + config.token } } + ) + ).json() + ).database_length; + } + + let pastJobs = 0; + async function getCompletedJobsCount(): Promise { + const completedJobs = ( + await ( + await fetch( + host + "/api/w/" + config.workspace_id + "/jobs/completed/count", + { headers: { ["Authorization"]: "Bearer " + config.token } } + ) + ).json() + ).database_length; + return completedJobs - pastJobs; + } + if (["deno", "python", "go", "bash", "dedicated", "bun"].includes(kind)) { await createBenchScript(kind, workspace); } - let jobsSent = jobs; + pastJobs = await getCompletedJobsCount(); + + const jobsSent = jobs; console.log(`Bulk creating ${jobsSent} jobs`); const start_create = Date.now(); @@ -84,9 +137,8 @@ export async function main({ body = JSON.stringify({ kind: "script", path: "f/benchmarks/" + kind, - dedicated_worker: kind === "dedicated", }); - } else if (["2steps", "onebranch", "branchallparrallel"].includes(kind)) { + } else if (["2steps"].includes(kind)) { const payload = getFlowPayload(kind); body = JSON.stringify({ kind: "flow", @@ -113,6 +165,7 @@ export async function main({ if (!response.ok) { throw new Error("Failed to create jobs: " + response.statusText); } + const uuids = await response.json(); const end_create = Date.now(); const create_duration = end_create - start_create; console.log( @@ -122,69 +175,61 @@ export async function main({ ); let start = Date.now(); - let queue_length = jobsSent; + let completedJobs = 0; let lastElapsed = 0; - let lastQueueLength = queue_length; - const updateState = setInterval(async () => { - const elapsed = start ? Date.now() - start : 0; - queue_length = ( - await ( - await fetch( - host + "/api/w/" + config.workspace_id + "/jobs/queue/count", - { headers: { ["Authorization"]: "Bearer " + config.token } } + let lastCompletedJobs = 0; + + let didStart = false; + while (completedJobs < jobsSent) { + if (!didStart) { + const actual_queue = await getQueueCount(); + if (actual_queue < jobsSent) { + start = Date.now(); + didStart = true; + } + } else { + const elapsed = start ? Date.now() - start : 0; + completedJobs = await getCompletedJobsCount(); + if (kind === "2steps") { + completedJobs = Math.floor(completedJobs / 3); + } + const avgThr = ((completedJobs / elapsed) * 1000).toFixed(2); + const instThr = + lastElapsed > 0 + ? ( + ((completedJobs - lastCompletedJobs) / (elapsed - lastElapsed)) * + 1000 + ).toFixed(2) + : 0; + + lastElapsed = elapsed; + lastCompletedJobs = completedJobs; + + await Deno.stdout.write( + enc( + `elapsed: ${(elapsed / 1000).toFixed( + 2 + )} | jobs executed: ${completedJobs}/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | remaining: ${ + jobsSent - completedJobs + } \r` ) - ).json() - ).database_length; - const avgThr = (((jobsSent - queue_length) / elapsed) * 1000).toFixed(2); - const instThr = - lastElapsed > 0 - ? ( - ((lastQueueLength - queue_length) / (elapsed - lastElapsed)) * - 1000 - ).toFixed(2) - : 0; - - lastElapsed = elapsed; - lastQueueLength = queue_length; - - await Deno.stdout.write( - enc( - `elapsed: ${(elapsed / 1000).toFixed(2)} | jobs executed: ${ - jobsSent - queue_length - }/${jobsSent} (thr: inst ${instThr} - avg ${avgThr}) | queue: ${queue_length} \r` - ) - ); - }, 10); - - while (queue_length > 0) { - if (queue_length < jobsSent && jobsSent === jobs) { - // reset start time to when the first job was picked up - start = Date.now(); - jobsSent = queue_length; + ); } - await sleep(0.01); } - clearInterval(updateState); - const total_duration_sec = (Date.now() - start) / 1000.0; - await sleep(0.1); console.log(`\njobs: ${jobsSent}`); console.log(`duration: ${total_duration_sec}s`); console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`); - console.log( - "queue length:", - ( - await ( - await fetch( - host + "/api/w/" + config.workspace_id + "/jobs/queue/count", - { headers: { ["Authorization"]: "Bearer " + config.token } } - ) - ).json() - ).database_length - ); + console.log("completed jobs", await getCompletedJobsCount()); + console.log("queue length:", await getQueueCount()); + + if (!noVerify && kind !== "noop") { + await verifyOutputs(uuids, config.workspace_id); + } + console.log("done"); return { @@ -229,7 +274,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, onebranch, branchallparrallel", + "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps", { required: true, } @@ -237,6 +282,7 @@ if (import.meta.main) { .option("-j --jobs ", "Number of jobs to create.", { default: 10000, }) + .option("--no-verify", "Do not verify the output of the jobs.") .action(main) .command( "upgrade", diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index e40040c351..f9ab85ea83 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -32,16 +32,16 @@ async function waitForDedicatedWorker(workspace: string, path: string) { const query = windmill.JobService.runWaitResultScriptByPath({ workspace, path, - requestBody: { - args: {}, - }, + requestBody: {}, }); - const timeout = new Promise((_, reject) => { - setTimeout(() => { + let timeout; + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { reject("Timeout"); }, 15000); }); - await Promise.race([query, timeout]); + await Promise.race([query, timeoutPromise]); + clearTimeout(timeout); } export async function createBenchScript( @@ -63,6 +63,7 @@ export async function createBenchScript( let scriptContent: string; let language: string; + let schemaProperties = {}; if (scriptPattern === "python") { scriptContent = 'import os\n\ndef main():\n return os.environ.get("WM_JOB_ID")'; @@ -74,9 +75,15 @@ export async function createBenchScript( } else if (scriptPattern === "bash") { scriptContent = "echo $WM_JOB_ID"; language = "bash"; - } else if (scriptPattern === "dedicated" || scriptPattern === "bun") { + } else if (scriptPattern === "bun") { scriptContent = 'export function main(){ return Bun.env["WM_JOB_ID"]; }'; language = "bun"; + } else if (scriptPattern === "dedicated") { + scriptContent = "export function main(uuid){ return uuid; }"; + language = "bun"; + schemaProperties = { + uuid: { default: null, description: "", type: "string" }, + }; } else if (scriptPattern === "deno") { scriptContent = 'export function main(){ return Deno.env.get("WM_JOB_ID"); }'; @@ -96,6 +103,12 @@ export async function createBenchScript( description: "", language: language as api.NewScript.language, dedicated_worker: scriptPattern === "dedicated", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + properties: schemaProperties, + required: [], + type: "object", + }, }, }); From f68cee4ebddbf6e774f80e91a8c89fb8dc213f91 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Thu, 21 Sep 2023 18:20:34 +0200 Subject: [PATCH 05/32] fix: tag id as flow (#2318) --- backend/windmill-queue/src/jobs.rs | 8 ++++---- benchmarks/benchmark_oneoff.ts | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index ca9cabb890..764efc3dba 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1744,11 +1744,11 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( tag = None; } let default = || { - if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview { + if job_kind == JobKind::Flow + || job_kind == JobKind::FlowPreview + || job_kind == JobKind::Identity + { "flow".to_string() - } else if job_kind == JobKind::Identity { - // identity is a light script, nativets is too - "nativets".to_string() } else if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies { "dependency".to_string() } else { diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 1f67d24d21..85e8339389 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -181,6 +181,7 @@ export async function main({ let didStart = false; while (completedJobs < jobsSent) { + let loopStart = Date.now(); if (!didStart) { const actual_queue = await getQueueCount(); if (actual_queue < jobsSent) { @@ -215,6 +216,10 @@ export async function main({ ) ); } + let loopDuration = (Date.now() - loopStart) / 1000.0; + if (loopDuration < 0.05) { + await sleep(0.05 - loopDuration); + } } const total_duration_sec = (Date.now() - start) / 1000.0; @@ -223,7 +228,7 @@ export async function main({ console.log(`duration: ${total_duration_sec}s`); console.log(`avg. throughput (jobs/time): ${jobsSent / total_duration_sec}`); - console.log("completed jobs", await getCompletedJobsCount()); + console.log("completed jobs", completedJobs); console.log("queue length:", await getQueueCount()); if (!noVerify && kind !== "noop") { From 481bcd53cb07e4520d5fd81572cad74340c4eb64 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 22 Sep 2023 00:05:39 +0200 Subject: [PATCH 06/32] fix: benchmark worker tags (#2319) * fix: benchmark worker tags * fix: increase nb of noop/ded jobs --- .github/workflows/benchmark.yml | 3 +++ benchmarks/suite_config.json | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index f4cb3ce1ba..e5f1a48803 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -23,6 +23,8 @@ jobs: env: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} + WORKER_GROUP: main + WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -34,6 +36,7 @@ jobs: env: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill DISABLE_SERVER: true + WORKER_GROUP: dedicated DEDICATED_WORKER: "admins:f/benchmarks/dedicated" LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} options: >- diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index 8ccc26fb0f..f973b907b3 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -3,7 +3,7 @@ { "graph_title": "noop throughput benchmark (single worker)", "kind": "noop", - "jobs": 5000 + "jobs": 30000 }, { "graph_title": "flow throughput benchmark (single worker)", @@ -13,7 +13,7 @@ { "graph_title": "dedicated throughput benchmark (single worker)", "kind": "dedicated", - "jobs": 2000 + "jobs": 30000 }, { "graph_title": "deno throughput benchmark (single worker)", From 3017307fccbe5feb18a694e3cbad882b25653d46 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 22 Sep 2023 14:35:05 +0200 Subject: [PATCH 07/32] add benchmark warm up (#2320) --- benchmarks/benchmark_oneoff.ts | 15 ++++++++++----- benchmarks/benchmark_suite.ts | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 85e8339389..fd3f512063 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -28,7 +28,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { console.log(`Job ${uuid} did not output the correct value`); incorrectResults++; } - } catch (err) { + } catch (_) { console.log(`Job ${uuid} did not complete`); incorrectResults++; } @@ -53,7 +53,7 @@ export async function main({ workspace: string; kind: string; jobs: number; - noVerify: boolean; + noVerify?: boolean; }) { windmill.setClient("", host); @@ -64,6 +64,9 @@ export async function main({ host, email, workspace, + kind, + jobs, + noVerify, }, null, 4 @@ -181,7 +184,7 @@ export async function main({ let didStart = false; while (completedJobs < jobsSent) { - let loopStart = Date.now(); + const loopStart = Date.now(); if (!didStart) { const actual_queue = await getQueueCount(); if (actual_queue < jobsSent) { @@ -216,7 +219,7 @@ export async function main({ ) ); } - let loopDuration = (Date.now() - loopStart) / 1000.0; + const loopDuration = (Date.now() - loopStart) / 1000.0; if (loopDuration < 0.05) { await sleep(0.05 - loopDuration); } @@ -287,7 +290,9 @@ if (import.meta.main) { .option("-j --jobs ", "Number of jobs to create.", { default: 10000, }) - .option("--no-verify", "Do not verify the output of the jobs.") + .option("--no-verify", "Do not verify the output of the jobs.", { + default: false, + }) .action(main) .command( "upgrade", diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index 1572f510bf..0b9e85429b 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -23,6 +23,25 @@ type Config = { ]; }; +async function warmUp( + host: string, + email: string | undefined, + password: string | undefined, + token: string | undefined, + workspace: string +) { + console.log("%cWarming up...", "font-weight: bold;"); + await runBenchmark({ + host, + email, + password, + token, + workspace, + kind: "noop", + jobs: 5000, + }); +} + async function main({ host, email, @@ -47,6 +66,8 @@ async function main({ } } + await warmUp(host, email, password, token, workspace); + try { const config = await getConfig(configPath); for (const benchmark of config.benchmarks) { From 1e629b233c7c327fa6aea314dedb003e36abdb88 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 22 Sep 2023 15:57:46 +0200 Subject: [PATCH 08/32] fix table when empty rows --- .../components/apps/components/display/table/AppTable.svelte | 2 +- frontend/src/lib/components/apps/editor/AppPreview.svelte | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/lib/components/apps/components/display/table/AppTable.svelte b/frontend/src/lib/components/apps/components/display/table/AppTable.svelte index a3c534e7be..22e9f1cf3d 100644 --- a/frontend/src/lib/components/apps/components/display/table/AppTable.svelte +++ b/frontend/src/lib/components/apps/components/display/table/AppTable.svelte @@ -165,7 +165,7 @@ } const headers = Array.from( - new Set(result.flatMap((row) => (typeof row == 'object' ? Object.keys(row) : []))) + new Set(result.flatMap((row) => (typeof row == 'object' ? Object.keys(row ?? {}) : []))) ) $options = { diff --git a/frontend/src/lib/components/apps/editor/AppPreview.svelte b/frontend/src/lib/components/apps/editor/AppPreview.svelte index 9a050cec8e..b172181c39 100644 --- a/frontend/src/lib/components/apps/editor/AppPreview.svelte +++ b/frontend/src/lib/components/apps/editor/AppPreview.svelte @@ -119,7 +119,6 @@ appStore.subscribe(loadTheme) async function loadTheme(currentAppStore: App) { - console.log(currentAppStore) if (!currentAppStore.theme) { return } From ea364ad9602647cbc9e8ee78fb5f17f0012105f6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 22 Sep 2023 18:15:34 +0200 Subject: [PATCH 09/32] feat: add running filter --- backend/tests/worker.rs | 6 +- backend/windmill-api/openapi.yaml | 1 + backend/windmill-api/src/jobs.rs | 23 +++++-- .../src/lib/components/OAuthSetting.svelte | 29 +++++---- .../src/lib/components/runs/RunsFilter.svelte | 7 +- .../(logged)/runs/[...path]/+page.svelte | 65 +++++++++---------- 6 files changed, 76 insertions(+), 55 deletions(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 45b32a0b3f..85fdbdafa0 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5,7 +5,7 @@ use chrono::Timelike; #[cfg(feature = "enterprise")] use futures::StreamExt; -use futures::{stream, Stream}; +use futures::{stream, Stream, StreamExt}; use serde::Deserialize; use serde_json::json; use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres}; @@ -15,7 +15,7 @@ use tokio::sync::RwLock; use tokio::time::{timeout, Duration}; use windmill_api_client::types::{ - CreateFlowBody, RawScript + CreateFlowBody, RawScript, NewScript, NewScriptLanguage }; #[cfg(feature = "enterprise")] @@ -2925,7 +2925,7 @@ async fn run_deployed_relative_imports(db: &Pool, script_content: Stri completed.next().await; // deployed script let script = - query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string()) + sqlx::query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string()) .fetch_one(&db2) .await .unwrap(); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index b3b3fd77fa..2ea6c4f1bf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4117,6 +4117,7 @@ paths: - $ref: "#/components/parameters/StartedBefore" - $ref: "#/components/parameters/StartedAfter" - $ref: "#/components/parameters/CreatedOrStartedBefore" + - $ref: "#/components/parameters/Running" - $ref: "#/components/parameters/CreatedOrStartedAfter" - $ref: "#/components/parameters/JobKinds" - $ref: "#/components/parameters/ArgsFilter" diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 2457e543a9..3e60a2acd6 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -780,7 +780,14 @@ async fn list_jobs( let (per_page, offset) = paginate(pagination); let lqc = lq.clone(); - let sqlc = list_completed_jobs_query( + if lq.success.is_some() && lq.running.is_some_and(|x| x) { + return Err(error::Error::BadRequest( + "cannot specify both success and running".to_string(), + )); + + } + let sqlc = if lq.running.is_none() { + Some(list_completed_jobs_query( &w_id, per_page + offset, 0, @@ -817,7 +824,10 @@ async fn list_jobs( "null as concurrent_limit", "null as concurrency_time_window_s", ], - ); + )) + } else { + None + }; let sql = if lq.success.is_none() { let sqlq = list_queue_jobs_query( @@ -833,7 +843,7 @@ async fn list_jobs( created_after: lq.created_after, created_or_started_before: lq.created_or_started_before, created_or_started_after: lq.created_or_started_after, - running: None, + running: lq.running, parent_job: lq.parent_job, order_desc: Some(true), job_kinds: lq.job_kinds, @@ -876,6 +886,7 @@ async fn list_jobs( ], ); + if let Some(sqlc) = sqlc { format!( "{} UNION ALL {} LIMIT {} OFFSET {};", &sqlq.subquery()?, @@ -883,8 +894,11 @@ async fn list_jobs( per_page, offset ) + } else { + sqlq.query()? + } } else { - sqlc.query()? + sqlc.unwrap().query()? }; let mut tx = user_db.begin(&authed).await?; let jobs: Vec = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; @@ -2721,6 +2735,7 @@ pub struct ListCompletedQuery { pub created_or_started_before: Option>, pub created_or_started_after: Option>, pub success: Option, + pub running: Option, pub parent_job: Option, pub order_desc: Option, pub job_kinds: Option, diff --git a/frontend/src/lib/components/OAuthSetting.svelte b/frontend/src/lib/components/OAuthSetting.svelte index 8de1c5b07c..76ec51f73a 100644 --- a/frontend/src/lib/components/OAuthSetting.svelte +++ b/frontend/src/lib/components/OAuthSetting.svelte @@ -7,11 +7,10 @@ export let value: any export let login = true - $: if (value && value?.['allowed_domains'] == undefined) { - value = { ...(value ?? {}), allowed_domains: [] } - } - $: enabled = value != undefined + + + let allowed_domains = value?.['allowed_domains'] ?? ''
@@ -39,14 +38,19 @@ {#if login} {/if} {#if name == 'google'} @@ -83,3 +87,4 @@
{/if}
+ diff --git a/frontend/src/lib/components/runs/RunsFilter.svelte b/frontend/src/lib/components/runs/RunsFilter.svelte index cf16b8cf85..a132bd1805 100644 --- a/frontend/src/lib/components/runs/RunsFilter.svelte +++ b/frontend/src/lib/components/runs/RunsFilter.svelte @@ -10,7 +10,7 @@ // Filters export let path: string | null = null - export let success: boolean | undefined = undefined + export let success: "running" | "success" | "failure" | undefined = undefined export let isSkipped: boolean | undefined = undefined export let argFilter: string export let argError: string @@ -187,8 +187,9 @@ Status - - + + +
diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 1f4da44c9e..f5135302ed 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -42,10 +42,7 @@ let user: string | null = $page.url.searchParams.get('user') let folder: string | null = $page.url.searchParams.get('folder') // Rest of filters handled by RunsFilter - let success: boolean | undefined = - $page.url.searchParams.get('success') != undefined - ? $page.url.searchParams.get('success') == 'true' - : undefined + let success: "running" | "success" | "failure" | undefined = ($page.url.searchParams.get('success') ?? undefined) as "running" | "success" | "failure" | undefined let isSkipped: boolean | undefined = $page.url.searchParams.get('is_skipped') != undefined ? $page.url.searchParams.get('is_skipped') == 'true' @@ -128,7 +125,8 @@ createdBy: user === null || user === '' ? undefined : user, scriptPathStart: folder === null || folder === '' ? undefined : `f/${folder}/`, jobKinds, - success, + success: success == "success" ? true : (success == 'failure' ? false : undefined), + running: success == 'running' ? true : undefined, isSkipped, isFlowStep: jobKindsCat != 'all' ? false : undefined, args: @@ -166,34 +164,38 @@ async function syncer() { getCount() if (sync && jobs && maxTs == undefined) { - let ts: string | undefined = undefined - let cursor = 0 - while (cursor < jobs.length && minTs == undefined) { - let invCursor = jobs.length - 1 - cursor - let isQueuedJob = cursor == jobs?.length - 1 || jobs[invCursor].type == Job.type.QUEUED_JOB - if (isQueuedJob) { - if (cursor > 0) { - const date = new Date(jobs[invCursor + 1]?.created_at!) - date.setMilliseconds(date.getMilliseconds() + 1) - ts = date.toISOString() + if (success == 'running') { + loadJobs() + } else { + let ts: string | undefined = undefined + let cursor = 0 + while (cursor < jobs.length && minTs == undefined) { + let invCursor = jobs.length - 1 - cursor + let isQueuedJob = cursor == jobs?.length - 1 || jobs[invCursor].type == Job.type.QUEUED_JOB + if (isQueuedJob) { + if (cursor > 0) { + const date = new Date(jobs[invCursor + 1]?.created_at!) + date.setMilliseconds(date.getMilliseconds() + 1) + ts = date.toISOString() + } + break } - break + cursor++ } - cursor++ - } - loading = true - const newJobs = await fetchJobs(maxTs, minTs ?? ts) - if (newJobs && newJobs.length > 0 && jobs) { - const oldJobs = jobs?.map((x) => x.id) - jobs = newJobs.filter((x) => !oldJobs.includes(x.id)).concat(jobs) - newJobs - .filter((x) => oldJobs.includes(x.id)) - .forEach((x) => (jobs![jobs?.findIndex((y) => y.id == x.id)!] = x)) - jobs = jobs - computeCompletedJobs() + loading = true + const newJobs = await fetchJobs(maxTs, minTs ?? ts) + if (newJobs && newJobs.length > 0 && jobs) { + const oldJobs = jobs?.map((x) => x.id) + jobs = newJobs.filter((x) => !oldJobs.includes(x.id)).concat(jobs) + newJobs + .filter((x) => oldJobs.includes(x.id)) + .forEach((x) => (jobs![jobs?.findIndex((y) => y.id == x.id)!] = x)) + jobs = jobs + computeCompletedJobs() + } + loading = false } - loading = false } } @@ -204,10 +206,7 @@ path = $page.params.path user = $page.url.searchParams.get('user') folder = $page.url.searchParams.get('folder') - success = - $page.url.searchParams.get('success') != undefined - ? $page.url.searchParams.get('success') == 'true' - : undefined + success = ($page.url.searchParams.get('success') ?? undefined) as "success" | "failure" | "running" | undefined isSkipped = $page.url.searchParams.get('is_skipped') != undefined ? $page.url.searchParams.get('is_skipped') == 'true' From 8da819edbf3aca0fd58ec9f640816e71b5dcca62 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 22 Sep 2023 18:24:00 +0200 Subject: [PATCH 10/32] fix tests --- backend/tests/worker.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 85fdbdafa0..b41ca8245b 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2,10 +2,9 @@ use std::sync::Arc; #[cfg(feature = "enterprise")] use chrono::Timelike; -#[cfg(feature = "enterprise")] use futures::StreamExt; -use futures::{stream, Stream, StreamExt}; +use futures::{stream, Stream}; use serde::Deserialize; use serde_json::json; use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres}; @@ -15,14 +14,15 @@ use tokio::sync::RwLock; use tokio::time::{timeout, Duration}; use windmill_api_client::types::{ - CreateFlowBody, RawScript, NewScript, NewScriptLanguage + CreateFlowBody, RawScript }; -#[cfg(feature = "enterprise")] use sqlx::query; #[cfg(feature = "enterprise")] -use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs, NewScript, NewScriptLanguage}; +use windmill_api_client::types::{EditSchedule, NewSchedule, ScriptArgs}; + +use windmill_api_client::types::{NewScript, NewScriptLanguage}; use windmill_common::worker::WORKER_CONFIG; use windmill_common::{ @@ -2925,7 +2925,7 @@ async fn run_deployed_relative_imports(db: &Pool, script_content: Stri completed.next().await; // deployed script let script = - sqlx::query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string()) + query!("SELECT hash FROM script WHERE path = $1", "f/system/test_import".to_string()) .fetch_one(&db2) .await .unwrap(); From 4db934f39e736de9a63471568c68276318ac2290 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 22 Sep 2023 19:10:56 +0200 Subject: [PATCH 11/32] add meticulous to dev --- frontend/src/routes/+layout.svelte | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index 45fde8f3e6..909bd382dd 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -18,6 +18,13 @@ + {#if !import.meta.env.PROD} + + {/if} {$page.data?.stuff?.title ? `${$page.data?.stuff?.title} | ` : ''}Windmill From 304a2596fd29fbd9a79c5cf9fe4df7b44d5c5254 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 23 Sep 2023 14:40:07 +0200 Subject: [PATCH 12/32] feat: add license key as superadmin setting (#2321) * license key * all * feat: add license key as a superadmin setting * fix * fix --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 1 - backend/src/ee.rs | 78 ++++++++++--------- backend/src/main.rs | 21 ++--- backend/src/monitor.rs | 75 +++++++++++++++++- backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 26 +++++++ backend/windmill-api/src/ee.rs | 41 ++++++++++ backend/windmill-api/src/jobs.rs | 40 ++++++++++ backend/windmill-api/src/lib.rs | 13 ++-- backend/windmill-api/src/settings.rs | 18 +++++ .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/users.rs | 6 ++ .../lib/components/InstanceSettings.svelte | 51 ++++++++++++ .../src/lib/components/instanceSettings.ts | 1 + .../lib/components/sidebar/UserMenu.svelte | 2 +- 16 files changed, 317 insertions(+), 60 deletions(-) create mode 100644 backend/windmill-api/src/ee.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 88ec17a3f2..ad0498c176 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7098,7 +7098,6 @@ dependencies = [ "prometheus", "rand 0.8.5", "reqwest", - "rsa 0.7.2", "rsmq_async", "serde", "serde_json", @@ -7148,6 +7147,7 @@ dependencies = [ "regex", "reqwest", "retainer", + "rsa 0.7.2", "rsmq_async", "rust-embed", "samael", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 00f25adf3a..4f68e8bb33 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -54,7 +54,6 @@ tokio-metrics.workspace = true rand.workspace = true chrono.workspace = true git-version.workspace = true -rsa.workspace = true base64.workspace = true sha2.workspace = true rsmq_async.workspace = true diff --git a/backend/src/ee.rs b/backend/src/ee.rs index 865819a701..1d185b4638 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee.rs @@ -1,41 +1,49 @@ #[cfg(feature = "enterprise")] -use base64::Engine; -#[cfg(feature = "enterprise")] -use rsa::{pkcs8::DecodePublicKey, signature::Verifier}; -#[cfg(feature = "enterprise")] -use sha2::Sha256; +use windmill_common::error; -#[cfg(feature = "enterprise")] -pub fn verify_license_key(license_key: Option) -> anyhow::Result<()> { - if let Some(license_key) = license_key { - let mut splitted_lk = license_key.split("."); - if splitted_lk.clone().count() != 3 { - panic!("license_key can be splitted with 2 . (..)"); - } - let id = splitted_lk.next().unwrap(); - let expiry = splitted_lk.next().unwrap(); - let signature_b64 = splitted_lk.next().unwrap(); +pub async fn set_license_key(license_key: String) -> anyhow::Result<()> { + use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID}; - let expiry_nb = expiry.parse::()?; - if expiry_nb < chrono::Utc::now().timestamp() as u64 { - panic!( - "License key is expired (timestamp expiry: {expiry_nb}. Now: {}", - chrono::Utc::now().timestamp() - ); - } - const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB"; - let pub_key = rsa::RsaPublicKey::from_public_key_der( - &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?, - )?; - let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?; - rsa::pss::VerifyingKey::::new(pub_key) - .verify( - &format!("{id}{expiry}").as_bytes(), - &rsa::pss::Signature::from(signature), - ) - .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?; - } else { - panic!("License key is required for the enterprise edition"); + let id = validate_license_key(license_key.clone()).await?; + { + let mut l = LICENSE_KEY_ID.write().await; + *l = id.to_string() } + + { + let mut l = LICENSE_KEY.write().await; + *l = license_key + } + { + let mut l = LICENSE_KEY_VALID.write().await; + *l = true + } + + Ok(()) +} + +#[cfg(feature = "enterprise")] +pub async fn verify_license_key() -> error::Result<()> { + use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID}; + use windmill_common::error::to_anyhow; + + let expiry_nb = LICENSE_KEY + .read() + .await + .clone() + .split(".") + .nth(1) + .unwrap_or_else(|| "") + .parse::() + .map_err(to_anyhow)?; + if expiry_nb < chrono::Utc::now().timestamp() as u64 { + tracing::error!( + "License key expired: {} < {}", + expiry_nb, + chrono::Utc::now().timestamp() as u64 + ); + let mut l = LICENSE_KEY_VALID.write().await; + *l = false; + }; Ok(()) } diff --git a/backend/src/main.rs b/backend/src/main.rs index ec55eb9901..a45ff03d69 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -19,11 +19,10 @@ use tokio::{ fs::{metadata, DirBuilder}, sync::RwLock, }; -use windmill_api::LICENSE_KEY; use windmill_common::{ global_settings::{ BASE_URL_SETTING, CUSTOM_TAGS_SETTING, ENV_SETTINGS, OAUTH_SETTING, - REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, LICENSE_KEY_SETTING, }, utils::rd_string, worker::{reload_custom_tags_setting, WORKER_GROUP}, @@ -38,7 +37,7 @@ use windmill_worker::{ use crate::monitor::{ initial_load, monitor_db, reload_base_url_setting, reload_retention_period_setting, - reload_server_config, reload_worker_config, + reload_server_config, reload_worker_config, reload_license_key, }; const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -153,9 +152,11 @@ Windmill Community Edition {GIT_VERSION} // since it's only on server mode, the port is statically defined let base_internal_url: String = format!("http://localhost:{}", port.to_string()); + initial_load(&db, tx.clone(), worker_mode, server_mode).await; + + monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode).await; - initial_load(&db, tx.clone(), worker_mode, server_mode).await; if std::env::var("BASE_INTERNAL_URL").is_ok() { tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored, you can remove it."); @@ -271,6 +272,12 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload custom tags setting"); } }, + LICENSE_KEY_SETTING => { + tracing::info!("License Key setting change detected"); + if let Err(e) = reload_license_key(&db).await { + tracing::error!(error = %e, "Could not reload license key setting"); + } + }, RETENTION_PERIOD_SECS_SETTING => { tracing::info!("Retention period setting change detected"); reload_retention_period_setting(&db).await @@ -355,13 +362,7 @@ pub async fn run_workers, ) -> anyhow::Result<()> { - #[cfg(feature = "enterprise")] - ee::verify_license_key(LICENSE_KEY.clone())?; - #[cfg(not(feature = "enterprise"))] - if LICENSE_KEY.as_ref().is_some_and(|x| !x.is_empty()) { - panic!("License key is required ONLY for the enterprise edition"); - } let instance_name = gethostname() .to_str() diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index b2ba7d2b13..e47e04bf4d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -15,10 +15,12 @@ use windmill_api::{ use windmill_common::{ error, global_settings::{ - BASE_URL_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, RETENTION_PERIOD_SECS_SETTING, + BASE_URL_SETTING, LICENSE_KEY_SETTING, OAUTH_SETTING, REQUEST_SIZE_LIMIT_SETTING, + RETENTION_PERIOD_SECS_SETTING, }, jobs::{JobKind, QueuedJob}, server::load_server_config, + users::truncate_token, worker::{load_worker_config, reload_custom_tags_setting, SERVER_CONFIG, WORKER_CONFIG}, BASE_URL, DB, METRICS_ENABLED, }; @@ -26,6 +28,14 @@ use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY, }; +#[cfg(feature = "enterprise")] +use crate::ee::verify_license_key; + +#[cfg(feature = "enterprise")] +use windmill_api::LICENSE_KEY_VALID; + +use crate::ee::set_license_key; + lazy_static::lazy_static! { static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT") .ok() @@ -63,7 +73,7 @@ pub async fn initial_load( db: &Pool, tx: tokio::sync::broadcast::Sender<()>, worker_mode: bool, - server_mode: bool + server_mode: bool, ) { let reload_worker_config_f = async { if worker_mode { @@ -102,13 +112,24 @@ pub async fn initial_load( reload_request_size(&db).await; } }; + + let reload_license_key_f = async { + if server_mode { + #[cfg(feature = "enterprise")] + if let Err(e) = reload_license_key(&db).await { + tracing::error!("Error reloading license key: {:?}", e) + } + } + }; + join!( reload_worker_config_f, reload_server_config_f, reload_custom_tags_f, reload_request_size_f, reload_base_url_f, - reload_retention_period_f + reload_retention_period_f, + reload_license_key_f ); } @@ -214,6 +235,36 @@ pub async fn reload_request_size(db: &DB) { } } +pub async fn reload_license_key(db: &DB) -> error::Result<()> { + let q = sqlx::query!( + "SELECT value FROM global_settings WHERE name = $1", + LICENSE_KEY_SETTING + ) + .fetch_optional(db) + .await?; + + let mut value = std::env::var("LICENSE_KEY") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(String::new()); + + if let Some(q) = q { + if let Ok(v) = serde_json::from_value::(q.value.clone()) { + tracing::info!( + "Loaded setting LICENSE_KEY from db config: {}", + truncate_token(&v) + ); + value = v; + } else { + tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q.value); + } + }; + + set_license_key(value).await?; + + Ok(()) +} + pub async fn reload_setting( db: &DB, setting_name: &str, @@ -271,12 +322,28 @@ pub async fn monitor_db) { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index cd423b5e1d..486602025e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -74,3 +74,4 @@ bytes.workspace = true mail-send.workspace = true samael = { workspace = true, optional = true } async-recursion.workspace = true +rsa.workspace = true \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2ea6c4f1bf..5597bc1212 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -624,6 +624,32 @@ paths: schema: type: string + /settings/test_license_key: + post: + summary: test license key + operationId: testLicenseKey + tags: + - setting + requestBody: + description: test license key + required: true + content: + application/json: + schema: + type: object + properties: + license_key: + type: string + required: + - license_key + responses: + "200": + description: status + content: + text/plain:: + schema: + type: string + /users/email: get: summary: get current user email (if logged in) diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee.rs new file mode 100644 index 0000000000..8eda5eeafd --- /dev/null +++ b/backend/windmill-api/src/ee.rs @@ -0,0 +1,41 @@ +use anyhow::anyhow; +use base64::Engine; +use rsa::{pkcs8::DecodePublicKey, signature::Verifier}; +use sha2::Sha256; + +pub async fn validate_license_key(license_key: String) -> anyhow::Result { + let mut splitted_lk = license_key.split("."); + if splitted_lk.clone().count() != 3 { + return Err(anyhow!( + "license_key can be splitted with 2 . (..)" + )); + } + + let id = splitted_lk.next().unwrap(); + let expiry = splitted_lk.next().unwrap(); + let signature_b64 = splitted_lk.next().unwrap(); + + const PUBLIC_KEY: &str = "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgVShzcLSPiOi+8ET8fggob1kmi47/cE12JaidPkwfGnScZItghkqtiLsct0U4kJhlp5gO89DYTBmIKadvxwY7kMsLlZzmi2emVH7c27cByGASY8QmWDNdG4Ggy/NDflGGBdAtN6gHawZAg4zHv3qpbPQGHH1/6sXIohcXhOnouwIDAQAB"; + let pub_key = rsa::RsaPublicKey::from_public_key_der( + &base64::engine::general_purpose::STANDARD.decode(PUBLIC_KEY)?, + )?; + let signature = base64::engine::general_purpose::STANDARD.decode(signature_b64)?; + rsa::pss::VerifyingKey::::new(pub_key) + .verify( + &format!("{id}{expiry}").as_bytes(), + &rsa::pss::Signature::from(signature), + ) + .map_err(|_| anyhow::anyhow!("Invalid license key".to_string()))?; + + let expiry_nb = expiry.parse::()?; + if expiry_nb < chrono::Utc::now().timestamp() as u64 { + tracing::error!( + "License key expired: {} < {}", + expiry_nb, + chrono::Utc::now().timestamp() as u64 + ); + return Err(anyhow!("License key expired".to_string())); + }; + + Ok(id.to_string()) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 3e60a2acd6..d2f213a8f1 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1650,6 +1650,19 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option) -> } } +#[cfg(feature = "enterprise")] +pub async fn check_license_key_valid() -> error::Result<()> { + use crate::LICENSE_KEY_VALID; + + let valid = *LICENSE_KEY_VALID.read().await; + if !valid { + return Err(error::Error::BadRequest(format!( + "License key is not valid. Go to your superadmin settings to update your license key.", + ))); + } + Ok(()) +} + pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -1660,6 +1673,8 @@ pub async fn run_flow_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; @@ -1713,7 +1728,11 @@ pub async fn run_job_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + let script_path = script_path.to_path(); + check_scopes(&authed, || format!("run:script/{script_path}"))?; let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?; @@ -1901,6 +1920,9 @@ pub async fn run_wait_result_job_by_path_get( Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + if method == http::Method::HEAD { return Ok(Json(serde_json::json!(""))); } @@ -1961,6 +1983,9 @@ pub async fn run_wait_result_flow_by_path_get( headers: HeaderMap, Query(run_query): Query, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + if method == http::Method::HEAD { return Ok(Json(serde_json::json!(""))); } @@ -2000,6 +2025,9 @@ pub async fn run_wait_result_script_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + run_wait_result_script_by_path_internal( db, run_query, @@ -2125,6 +2153,9 @@ pub async fn run_wait_result_script_by_hash( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + check_queue_too_long(&db, run_query.queue_limit).await?; let hash = script_hash.0; @@ -2215,6 +2246,9 @@ pub async fn run_wait_result_flow_by_path( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::JsonResult { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + run_wait_result_flow_by_path_internal( db, run_query, flow_path, authed, rsmq, user_db, headers, args, raw_string, w_id, ) @@ -2290,6 +2324,9 @@ async fn run_preview_job( headers: HeaderMap, Json(preview): Json, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + check_scopes(&authed, || format!("runscript"))?; if authed.is_operator { return Err(error::Error::NotAuthorized( @@ -2532,6 +2569,9 @@ pub async fn run_job_by_hash( headers: HeaderMap, JsonOrForm(args, raw_string): JsonOrForm, ) -> error::Result<(StatusCode, String)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + let hash = script_hash.0; let ( path, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 960715ab9c..fa1b2bb24e 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -47,6 +47,7 @@ mod capture; mod configs; mod db; mod drafts; +pub mod ee; mod favorite; mod flows; mod folders; @@ -101,7 +102,9 @@ lazy_static::lazy_static! { slack: None })); - pub static ref LICENSE_KEY: Option = std::env::var("LICENSE_KEY").ok(); + pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); + pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); + pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); } pub async fn run_server( @@ -317,13 +320,7 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - LICENSE_KEY - .as_ref() - .unwrap() - .split(".") - .next() - .unwrap() - .to_string() + LICENSE_KEY_ID.read().await.clone() } async fn openapi() -> &'static str { diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 69864a6b80..99da96777c 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -10,6 +10,7 @@ use std::time::Duration; use crate::{ db::{ApiAuthed, DB}, + ee::validate_license_key, utils::require_super_admin, }; @@ -36,6 +37,7 @@ pub fn global_service() -> Router { post(set_global_setting).get(get_global_setting), ) .route("/test_smtp", post(test_email)) + .route("/test_license_key", post(test_license_key)) } #[derive(Deserialize)] @@ -72,6 +74,22 @@ pub async fn test_email( Ok("Sent test email".to_string()) } +#[derive(Deserialize)] +pub struct TestKey { + pub license_key: String, +} + + +pub async fn test_license_key( + Extension(db): Extension, + authed: ApiAuthed, + Json(TestKey { license_key }): Json, +) -> error::Result { + require_super_admin(&db, &authed.email).await?; + validate_license_key(license_key).await?; + Ok("Sent test email".to_string()) +} + pub async fn get_local_settings( Extension(db): Extension, authed: ApiAuthed, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 83837c55a1..8119f783bd 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -4,6 +4,7 @@ pub const BASE_URL_SETTING: &str = "base_url"; pub const OAUTH_SETTING: &str = "oauths"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb"; +pub const LICENSE_KEY_SETTING: &str = "license_key"; pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index 772670af74..688e0ba8e2 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -15,3 +15,9 @@ pub fn username_to_permissioned_as(user: &str) -> String { format!("u/{}", user) } } + +pub fn truncate_token(token: &str) -> String { + let mut s = token[..10].to_owned(); + s.push_str("*****"); + s +} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index b81aefd925..c63e13cb0e 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -50,6 +50,14 @@ fieldType: 'seconds', placeholder: '60', storage: 'config' + }, + { + label: 'License Key', + description: 'License Key required to use the EE (switch image for windmill-ee)', + key: 'license_key', + fieldType: 'license_key', + placeholder: 'only needed to prepare upgrade to EE', + storage: 'setting' } ], SMTP: [ @@ -187,6 +195,18 @@ let resourceName = '' let tab: 'Core' | 'SMTP' | 'OAuth' = 'Core' + function parseDate(license_key: string): string | undefined { + let splitted = license_key.split('.') + if (splitted.length >= 3) { + try { + let i = parseInt(splitted[1]) + let date = new Date(i * 1000) + return date.toDateString() + } catch {} + } + return undefined + } + let to: string = '' @@ -226,6 +246,37 @@ placeholder={setting.placeholder} bind:value={values[setting.key]} /> + {:else if setting.fieldType == 'textarea'} +