From 1c70c0c2f10858fbd0f22880ff18c8ec92b3ff69 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Oct 2024 18:30:32 +0100 Subject: [PATCH 1/8] fix(cli): improve instance sync for CI/CD + --folder-per-instance --- cli/instance.ts | 74 ++++++++++++++++++++++++++++++----------- cli/local_encryption.ts | 1 - 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/cli/instance.ts b/cli/instance.ts index 32117f8e3f..3e21bc72e4 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -181,6 +181,7 @@ export type InstanceSyncOptions = { instance?: string; baseUrl?: string; token?: string; + folderPerInstance?: boolean; yes?: boolean; }; @@ -297,7 +298,11 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { if (opts.includeWorkspaces) { log.info("\nPulling all workspaces"); const rootDir = Deno.cwd(); - const localWorkspaces = await getLocalWorkspaces(rootDir, instance.prefix); + const localWorkspaces = await getLocalWorkspaces( + rootDir, + instance.prefix, + opts.folderPerInstance + ); const previousActiveWorkspace = await getActiveWorkspace(undefined); const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ @@ -306,7 +311,9 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { }); for (const remoteWorkspace of remoteWorkspaces) { log.info("\nPulling workspace " + remoteWorkspace.id); - const workspaceName = instance.prefix + "_" + remoteWorkspace.id; + const workspaceName = opts?.folderPerInstance + ? instance.prefix + "/" + remoteWorkspace.id + : instance.prefix + "_" + remoteWorkspace.id; await Deno.mkdir(path.join(rootDir, workspaceName), { recursive: true, }); @@ -425,16 +432,21 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { instances = await allInstances(); const rootDir = Deno.cwd(); - const localPrefix = (await Select.prompt({ - message: "What is the prefix of the local workspaces you want to sync?", - options: [ - ...instances.map((i) => ({ - name: `${i.prefix} (${i.name} - ${i.remote})`, - value: i.prefix, - })), - ], - default: instance.prefix as unknown, - })) as unknown as string; + let localPrefix; + if (opts.baseUrl && opts.token) { + localPrefix = "custom"; + } else { + localPrefix = (await Select.prompt({ + message: "What is the prefix of the local workspaces you want to sync?", + options: [ + ...instances.map((i) => ({ + name: `${i.prefix} (${i.name} - ${i.remote})`, + value: i.prefix, + })), + ], + default: instance.prefix as unknown, + })) as unknown as string; + } const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ page: 1, @@ -443,7 +455,11 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { const previousActiveWorkspace = await getActiveWorkspace(undefined); - const localWorkspaces = await getLocalWorkspaces(rootDir, localPrefix); + const localWorkspaces = await getLocalWorkspaces( + rootDir, + localPrefix, + opts.folderPerInstance + ); log.info( `\nPushing all workspaces: ${localWorkspaces.map((x) => x.id).join(", ")}` @@ -521,17 +537,35 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } } -async function getLocalWorkspaces(rootDir: string, localPrefix: string) { +async function getLocalWorkspaces( + rootDir: string, + localPrefix: string, + folderPerInstance?: boolean +) { const localWorkspaces: { dir: string; id: string }[] = []; - for await (const dir of Deno.readDir(rootDir)) { - const dirName = dir.name; - if (dirName.startsWith(localPrefix + "_")) { + if (!(await Deno.stat(localPrefix).catch(() => null))) { + await Deno.mkdir(localPrefix); + } + if (folderPerInstance) { + for await (const dir of Deno.readDir(rootDir + "/" + localPrefix)) { + const dirName = dir.name; localWorkspaces.push({ - dir: dirName, - id: dirName.substring(localPrefix.length + 1), + dir: localPrefix + "/" + dirName, + id: dirName, }); } + log.info(localWorkspaces); + } else { + for await (const dir of Deno.readDir(rootDir)) { + const dirName = dir.name; + if (dirName.startsWith(localPrefix + "_")) { + localWorkspaces.push({ + dir: dirName, + id: dirName.substring(localPrefix.length + 1), + }); + } + } } return localWorkspaces; } @@ -650,6 +684,7 @@ const command = new Command() .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") .option("--skip-groups", "Skip pulling instance groups") .option("--include-workspaces", "Also pull workspaces") + .option("--folder-per-instance", "Create a folder per instance") .option( "--instance ", "Name of the instance to push to, override the active instance" @@ -665,6 +700,7 @@ const command = new Command() .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") .option("--skip-groups", "Skip pushing instance groups") .option("--include-workspaces", "Also push workspaces") + .option("--folder-per-instance", "Create a folder per instance") .option( "--instance ", "Name of the instance to push to, override the active instance" diff --git a/cli/local_encryption.ts b/cli/local_encryption.ts index 8e7a241e52..4277093f8f 100644 --- a/cli/local_encryption.ts +++ b/cli/local_encryption.ts @@ -1,4 +1,3 @@ -import { log } from "./deps.ts"; import crypto from "node:crypto"; // Helper function to convert strings to Uint8Array (binary) From f2d6b47f8607b2f554e870f7b3608e77ff7f4585 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Oct 2024 18:35:47 +0100 Subject: [PATCH 2/8] chore(main): release 1.414.2 (#4603) * chore(main): release 1.414.2 * Apply automatic changes --------- Co-authored-by: rubenfiszel --- CHANGELOG.md | 7 +++ backend/Cargo.lock | 48 +++++++++---------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 4 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- python-client/wmill_pg/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 16 files changed, 48 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4dda9f297..347ae8ba4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.414.2](https://github.com/windmill-labs/windmill/compare/v1.414.1...v1.414.2) (2024-10-29) + + +### Bug Fixes + +* **cli:** improve instance sync for CI/CD + --folder-per-instance ([212579a](https://github.com/windmill-labs/windmill/commit/212579a514d070355fe0d9e0215593bacfa05e1f)) + ## [1.414.1](https://github.com/windmill-labs/windmill/compare/v1.414.0...v1.414.1) (2024-10-29) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ba671e1b77..d8c8ec64be 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10395,7 +10395,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "axum", @@ -10437,7 +10437,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "argon2", @@ -10522,7 +10522,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.414.1" +version = "1.414.2" dependencies = [ "base64 0.21.7", "chrono", @@ -10540,7 +10540,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.414.1" +version = "1.414.2" dependencies = [ "chrono", "serde", @@ -10553,7 +10553,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "rsmq_async", @@ -10568,7 +10568,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "async-stream", @@ -10614,7 +10614,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.414.1" +version = "1.414.2" dependencies = [ "regex", "rsmq_async", @@ -10629,7 +10629,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "bytes", @@ -10651,7 +10651,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.414.1" +version = "1.414.2" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10663,7 +10663,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.414.1" +version = "1.414.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -10672,7 +10672,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "lazy_static", @@ -10684,7 +10684,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "gosyn", @@ -10696,7 +10696,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "lazy_static", @@ -10708,7 +10708,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10719,7 +10719,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10730,7 +10730,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "async-recursion", @@ -10748,7 +10748,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10765,7 +10765,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "lazy_static", @@ -10777,7 +10777,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "lazy_static", @@ -10795,7 +10795,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10816,7 +10816,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "serde_json", @@ -10826,7 +10826,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "async-recursion", @@ -10859,7 +10859,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.414.1" +version = "1.414.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10869,7 +10869,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.414.1" +version = "1.414.2" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index cc23b4af6e..88d768cd6b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.414.1" +version = "1.414.2" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.414.1" +version = "1.414.2" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6175aa3572..7deeb96c81 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.414.1 + version: 1.414.2 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 0c58ed533d..2f3ae12ed5 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.414.1"; +export const VERSION = "v1.414.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index f23d96fcaa..2132a818b2 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.414.1"; +export const VERSION = "1.414.2"; const command = new Command() .name("wmill") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 89cb1179be..0eb1a2ec8c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.414.1", + "version": "1.414.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.414.1", + "version": "1.414.2", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index 3d83a6b5c4..f4dc37ce2f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.414.1", + "version": "1.414.2", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 33a5b690a0..419e796fbe 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.414.1" -wmill_pg = ">=1.414.1" +wmill = ">=1.414.2" +wmill_pg = ">=1.414.2" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index beae57dc15..79189c7af5 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.414.1 + version: 1.414.2 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index b2a5af5704..ebbf6572e6 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.414.1' +ModuleVersion = '1.414.2' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 5d49404f60..2aca3bdfa1 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.414.1" +version = "1.414.2" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 63a9e7b102..4cbd31bd02 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.414.1" +version = "1.414.2" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 5208ff11f2..395e7ad413 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.414.1", + "version": "1.414.2", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 5f9511d59d..8546939726 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.414.1", + "version": "1.414.2", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 3e8272a300..fd40ca5a31 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.414.1 +1.414.2 From 8b02cf1ae7be3ddd7c4487b3961e4cff76e0f432 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Oct 2024 19:40:10 +0100 Subject: [PATCH 3/8] chore: fix nsjail image build --- .github/workflows/docker-image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 6ad794e136..be0bfc771c 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -461,6 +461,8 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha,enable=true,priority=100,prefix=,suffix=,format=short + type=ref,event=branch + type=ref,event=pr - name: Login to registry uses: docker/login-action@v3 @@ -478,7 +480,6 @@ jobs: file: "./docker/DockerfileNsjail" tags: | ${{ steps.meta-ee-public.outputs.tags }} - ghcr.io/windmill-labs/windmill-ee-nsjail:main labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License From 8ba1cd6276c7da15698c235b4a8c92df607f014c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Oct 2024 14:44:32 +0100 Subject: [PATCH 4/8] nit confirmCallback edit script --- frontend/src/lib/components/ScriptBuilder.svelte | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e96fd824ee..8c2af724ff 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -279,7 +279,11 @@ // Handle through confirmation modal confirmCallback = async () => { open = false - await editScript(stay, actual_parent_hash, deployMsg) + if (actual_parent_hash) { + await editScript(stay, actual_parent_hash, deployMsg) + } else { + sendUserToast('Could not fetch latest version of the script', true) + } } // Open confirmation modal open = true From 921e3217c6fc819138dfce1c3a2b3850bcda7a46 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 30 Oct 2024 14:44:42 +0100 Subject: [PATCH 5/8] improve array params handling in postgres (#4605) --- backend/windmill-worker/src/pg_executor.rs | 259 +++++++++++++-------- frontend/src/lib/consts.ts | 8 +- 2 files changed, 163 insertions(+), 104 deletions(-) diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index f27eeee12b..55f5abea31 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -18,12 +18,8 @@ use serde_json::value::RawValue; use serde_json::Map; use serde_json::Value; use tokio::sync::Mutex; -use tokio_postgres::types::IsNull; use tokio_postgres::Client; -use tokio_postgres::{ - types::{to_sql_checked, ToSql}, - NoTls, Row, -}; +use tokio_postgres::{types::ToSql, NoTls, Row}; use tokio_postgres::{ types::{FromSql, Type}, Column, @@ -41,7 +37,7 @@ use windmill_queue::CanceledBy; use crate::common::{build_args_values, sizeof_val, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE}; -use bytes::{Buf, BytesMut}; +use bytes::Buf; use lazy_static::lazy_static; use urlencoding::encode; @@ -98,6 +94,11 @@ fn do_postgresql_inner<'a>( let mut res: Vec = vec![]; + let query_params = query_params + .iter() + .map(|p| &**p as &(dyn ToSql + Sync)) + .collect_vec(); + if skip_collect { client .execute_raw(&query, query_params) @@ -414,133 +415,176 @@ pub async fn do_postgresql( return Ok(raw_result); } -#[derive(Debug)] -enum PgType { - String(String), - Bool(bool), - I8(i8), - I16(i16), - I32(i32), - I64(i64), - U32(u32), - F32(f32), - F64(f64), - Uuid(Uuid), - Decimal(Decimal), - Date(chrono::NaiveDate), - Time(chrono::NaiveTime), - Timestamp(chrono::NaiveDateTime), - None(Option), - Array(Vec), - Json(serde_json::Value), - Bytea(Vec), +fn map_as_single_type( + vec: &Vec, + f: impl Fn(&Value) -> Option, +) -> anyhow::Result>> { + vec.into_iter() + .map(|v| { + // allow nulls in arrays + if matches!(v, Value::Null) { + Some(None) + } else { + f(v).map(Some) + } + }) + .collect::>>>() + .ok_or_else(|| anyhow::anyhow!("Mixed types in array")) } -impl ToSql for PgType { - fn to_sql( - &self, - ty: &Type, - out: &mut BytesMut, - ) -> Result> { - match *self { - PgType::String(ref val) => val.to_sql(ty, out), - PgType::Bool(ref val) => val.to_sql(ty, out), - PgType::I8(ref val) => val.to_sql(ty, out), - PgType::I16(ref val) => val.to_sql(ty, out), - PgType::I32(ref val) => val.to_sql(ty, out), - PgType::I64(ref val) => val.to_sql(ty, out), - PgType::U32(ref val) => val.to_sql(ty, out), - PgType::F32(ref val) => val.to_sql(ty, out), - PgType::F64(ref val) => val.to_sql(ty, out), - PgType::Uuid(ref val) => val.to_sql(ty, out), - PgType::Decimal(ref val) => val.to_sql(ty, out), - PgType::Date(ref val) => val.to_sql(ty, out), - PgType::Time(ref val) => val.to_sql(ty, out), - PgType::Timestamp(ref val) => val.to_sql(ty, out), - PgType::None(ref val) => val.to_sql(ty, out), - PgType::Array(ref val) => val.to_sql(ty, out), - PgType::Json(ref val) => val.to_sql(ty, out), - PgType::Bytea(ref val) => val.to_sql(ty, out), +fn convert_vec_val( + vec: &Vec, + arg_t: &String, +) -> windmill_common::error::Result> { + match arg_t.as_str() { + "bool" | "boolean" => Ok(Box::new(map_as_single_type(vec, |v| v.as_bool())?)), + "char" | "character" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i8) + })?)), + "smallint" | "smallserial" | "int2" | "serial2" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i16) + })?)) } + "int" | "integer" | "int4" | "serial" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i32) + })?)), + "numeric" | "decimal" => Ok(Box::new(map_as_single_type(vec, |v| { + if v.is_i64() { + Decimal::from_i64(v.as_i64().unwrap()) + } else if v.is_f64() { + Decimal::from_f64(v.as_f64().unwrap()) + } else { + None + } + })?)), + "oid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as u32) + })?)), + "bigint" | "bigserial" | "int8" | "serial8" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as i64) + })?)) + } + "real" | "float4" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_f64().map(|x| x as f32) + })?)), + "double" | "float8" => Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)), + "uuid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| Uuid::parse_str(x).ok()).flatten() + })?)), + "date" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDate::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "time" | "timetz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "timestamp" | "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDateTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ") + .unwrap_or_default() + }) + })?)), + "jsonb" | "json" => Ok(Box::new(vec.clone().into_iter().map(Some).collect_vec())), + "bytea" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + engine::general_purpose::STANDARD + .decode(x) + .unwrap_or(vec![]) + }) + })?)), + "text" | "varchar" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| x.to_string()) + })?)), + _ => Err(anyhow::anyhow!("Unsupported JSON array type"))?, } - - fn accepts(_: &Type) -> bool { - true - } - - to_sql_checked!(); } -fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::error::Result { +fn convert_val( + value: &Value, + arg_t: &String, + typ: &Typ, +) -> windmill_common::error::Result> { match value { Value::Array(vec) if arg_t.ends_with("[]") => { let arg_t = arg_t.trim_end_matches("[]").to_string(); - let mut result = vec![]; - for val in vec { - result.push(convert_val(val, &arg_t, typ)?); - } - Ok(PgType::Array(result)) + convert_vec_val(vec, &arg_t) } - Value::Null => Ok(PgType::None(None::)), - Value::Bool(b) => Ok(PgType::Bool(b.clone())), - Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(PgType::String(n.to_string())), - Value::Number(n) if n.is_i64() && arg_t == "char" => { - Ok(PgType::I8(n.as_i64().unwrap() as i8)) - } - Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => { - Ok(PgType::I16(n.as_i64().unwrap() as i16)) + Value::Null => Ok(Box::new(None::)), + Value::Bool(b) => Ok(Box::new(b.clone())), + Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(Box::new(n.to_string())), + Value::Number(n) if arg_t == "char" && n.is_i64() => { + Ok(Box::new(n.as_i64().unwrap() as i8)) } Value::Number(n) - if n.is_i64() - && (arg_t == "int" - || arg_t == "integer" - || arg_t == "int4" - || arg_t == "serial") => + if (arg_t == "smallint" + || arg_t == "smallserial" + || arg_t == "int2" + || arg_t == "serial2") + && n.is_i64() => { - Ok(PgType::I32(n.as_i64().unwrap() as i32)) + Ok(Box::new(n.as_i64().unwrap() as i16)) } - Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_i64(n.as_i64().unwrap()).unwrap()), + Value::Number(n) + if (arg_t == "int" || arg_t == "integer" || arg_t == "int4" || arg_t == "serial") + && n.is_i64() => + { + Ok(Box::new(n.as_i64().unwrap() as i32)) + } + Value::Number(n) if (arg_t == "real" || arg_t == "float4") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap() as f32)) + } + Value::Number(n) if (arg_t == "double" || arg_t == "float8") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap())) + } + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_i64() => Ok( + Box::new(Decimal::from_i64(n.as_i64().unwrap()).unwrap_or_default()), ), - Value::Number(n) if n.is_i64() => Ok(PgType::I64(n.as_i64().unwrap())), - Value::Number(n) if n.is_u64() && arg_t == "oid" => { - Ok(PgType::U32(n.as_u64().unwrap() as u32)) - } - Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => { - Ok(PgType::I64(n.as_u64().unwrap() as i64)) - } - Value::Number(n) if n.is_f64() && arg_t == "real" => { - Ok(PgType::F32(n.as_f64().unwrap() as f32)) - } - Value::Number(n) if n.is_f64() && arg_t == "double" => Ok(PgType::F64(n.as_f64().unwrap())), - Value::Number(n) if n.is_f64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_f64(n.as_f64().unwrap()).unwrap()), + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_f64() => Ok( + Box::new(Decimal::from_f64(n.as_f64().unwrap()).unwrap_or_default()), ), - Value::Number(n) => Ok(PgType::F64(n.as_f64().unwrap())), - Value::String(s) if arg_t == "uuid" => Ok(PgType::Uuid(Uuid::parse_str(s)?)), + Value::Number(n) if arg_t == "oid" && n.is_u64() => { + Ok(Box::new(n.as_u64().unwrap() as u32)) + } + Value::Number(n) + if (arg_t == "bigint" + || arg_t == "bigserial" + || arg_t == "int8" + || arg_t == "serial8") + && n.is_u64() => + { + Ok(Box::new(n.as_u64().unwrap() as i64)) + } + Value::Number(n) if n.is_i64() => Ok(Box::new(n.as_i64().unwrap())), + Value::Number(n) => Ok(Box::new(n.as_f64().unwrap())), + Value::String(s) if arg_t == "uuid" => Ok(Box::new(Uuid::parse_str(s)?)), Value::String(s) if arg_t == "date" => { let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Date(date)) + Ok(Box::new(date)) } Value::String(s) if arg_t == "time" || arg_t == "timetz" => { let time = chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Time(time)) + Ok(Box::new(time)) } Value::String(s) if arg_t == "timestamp" || arg_t == "timestamptz" => { let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ") .unwrap_or_default(); - Ok(PgType::Timestamp(datetime)) + Ok(Box::new(datetime)) } Value::String(s) if arg_t == "bytea" => { let bytes = engine::general_purpose::STANDARD .decode(s) .unwrap_or(vec![]); - Ok(PgType::Bytea(bytes)) + Ok(Box::new(bytes)) } - Value::Object(_) => Ok(PgType::Json(value.clone())), - Value::String(s) => Ok(PgType::String(s.clone())), + Value::Object(_) => Ok(Box::new(value.clone())), + Value::String(s) => Ok(Box::new(s.clone())), _ => Err(Error::ExecutionErr(format!( "Unsupported type in query: {:?} and signature {arg_t:?}", value @@ -624,6 +668,9 @@ pub fn pg_cell_to_json_value( Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) })?, + Type::OID => get_basic(row, column, column_i, |a: u32| { + Ok(JSONValue::Number(serde_json::Number::from(a))) + })?, // array types Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?, Type::BIT_ARRAY => get_array(row, column, column_i, |a: bit_vec::BitVec| match a.len() { @@ -655,6 +702,10 @@ pub fn pg_cell_to_json_value( Type::FLOAT8_ARRAY => { get_array(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))? } + Type::NUMERIC_ARRAY => get_array(row, column, column_i, |a: Decimal| { + Ok(serde_json::to_value(a) + .map_err(|_| anyhow::anyhow!("Cannot convert decimal to json"))?) + })?, // these types require a custom StringCollector struct as an intermediary (see struct at bottom) Type::TS_VECTOR_ARRAY => get_array(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) @@ -766,7 +817,7 @@ fn get_array<'a, T: FromSql<'a>>( val_to_json_val: impl Fn(T) -> Result, ) -> Result { let raw_val_array = row - .try_get::<_, Option>>(column_i) + .try_get::<_, Option>>>(column_i) .with_context(|| { format!( "conversion issue for array at column_name `{}`", @@ -777,7 +828,11 @@ fn get_array<'a, T: FromSql<'a>>( Some(val_array) => { let mut result = vec![]; for val in val_array { - result.push(val_to_json_val(val)?); + result.push( + val.map(|v| val_to_json_val(v)) + .transpose()? + .unwrap_or(Value::Null), + ); } JSONValue::Array(result) } diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index 2fa5fc30eb..45ccb51978 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -65,8 +65,8 @@ export const POSTGRES_TYPES = [ 'BIGSERIAL[]', 'REAL', 'REAL[]', - 'DOUBLE PRECISION', - 'DOUBLE PRECISION[]', + 'FLOAT8', + 'FLOAT8[]', 'NUMERIC', 'NUMERIC[]', 'DECIMAL', @@ -77,8 +77,12 @@ export const POSTGRES_TYPES = [ 'DATE[]', 'TIME', 'TIME[]', + 'TIMETZ', + 'TIMETZ[]', 'TIMESTAMP', 'TIMESTAMP[]', + 'TIMESTAMPTZ', + 'TIMESTAMPTZ[]', 'JSON', 'JSON[]', 'JSONB', From d8d2840650a6a4b8474c88c6bc72c331965d2fe6 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 30 Oct 2024 17:36:38 +0100 Subject: [PATCH 6/8] fix slack error handler when error is too long (#4606) --- frontend/src/lib/hubPaths.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 6fa5966973..c63f215a33 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,10 +1,10 @@ { "gitSync": "hub/9087/sync-script-to-git-repo-windmill", "gitSyncTest": "hub/9073/git-repo-test-read-write-windmill", - "slackErrorHandler": "hub/9079/workspace-or-schedule-error-handler-slack", + "slackErrorHandler": "hub/9206/workspace-or-schedule-error-handler-slack", "slackRecoveryHandler": "hub/9080/slack/schedule-recovery-handler-slack", "slackSuccessHandler": "hub/9072/slack/schedule-success-handler-slack", "slackReport": "hub/9084/slack", "discordReport": "hub/9085/discord", "smtpReport": "hub/9086/smtp" -} +} \ No newline at end of file From 29d4c9d53f5cfe72034a9caee839ce4ee4975330 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 30 Oct 2024 17:41:43 +0100 Subject: [PATCH 7/8] cache old slack error handler (#4607) --- frontend/src/lib/hubPaths.json | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index c63f215a33..06d3255db6 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -2,6 +2,7 @@ "gitSync": "hub/9087/sync-script-to-git-repo-windmill", "gitSyncTest": "hub/9073/git-repo-test-read-write-windmill", "slackErrorHandler": "hub/9206/workspace-or-schedule-error-handler-slack", + "slackErrorHandler_0": "hub/9079/workspace-or-schedule-error-handler-slack", "slackRecoveryHandler": "hub/9080/slack/schedule-recovery-handler-slack", "slackSuccessHandler": "hub/9072/slack/schedule-success-handler-slack", "slackReport": "hub/9084/slack", From f5b6050d5d787ac3527736f66133a69027bf12aa Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 30 Oct 2024 20:40:19 +0100 Subject: [PATCH 8/8] fix: improve express oauth setup --- .../windmill-parser-py-imports/src/lib.rs | 1 + .../lib/components/AppConnectDrawer.svelte | 8 +- .../src/lib/components/AppConnectInner.svelte | 495 +++++++++--------- ...AppConnectLightweightResourcePicker.svelte | 35 +- .../LightweightResourcePicker.svelte | 13 +- .../src/lib/components/ResourcePicker.svelte | 5 +- .../lib/components/SuperadminSettings.svelte | 1 - 7 files changed, 284 insertions(+), 274 deletions(-) diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 34487e04ef..841a763ec2 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -61,6 +61,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "opensearchpy" => "opensearch-py", "lokalise" => "python-lokalise-api", "msgraph" => "msgraph-sdk", + "pythonjsonlogger" => "python-json-logger", }; fn replace_import(x: String) -> String { diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 28cb801616..4f30737736 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -6,7 +6,7 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - let expressOAuthSetup = false + export let expressOAuthSetup = false let drawer: Drawer let resourceType = '' @@ -18,8 +18,7 @@ let appConnectInner: AppConnectInner | undefined = undefined let rtToLoad: string | undefined = '' - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { rtToLoad = rt drawer.openDrawer?.() } @@ -27,7 +26,7 @@ $: appConnectInner && onRtToLoadChange(rtToLoad) function onRtToLoadChange(rtToLoad: string | undefined) { - appConnectInner?.open(rtToLoad, expressOAuthSetup) + appConnectInner?.open(rtToLoad) } const dispatch = createEventDispatcher() @@ -60,6 +59,7 @@ bind:manual on:close={drawer?.closeDrawer} on:refresh + express={expressOAuthSetup} />
{#if step > 1} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index dbe162fd64..e568589b49 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -31,6 +31,7 @@ export let isGoogleSignin = false export let disabled = false export let manual = true + export let express = false let isValid = true @@ -81,10 +82,7 @@ let pathError = '' - let expressOAuthSetup = false - - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { if (!rt) { loadResourceTypes() } @@ -95,12 +93,12 @@ valueToken = undefined await loadConnects() manual = !connects?.includes(resourceType) - if (manual && expressOAuthSetup) { + if (manual && express) { dispatch('error', 'Express OAuth setup is not available for non OAuth resource types') return } if (rt) { - if (!manual && expressOAuthSetup) { + if (!manual && express) { await getScopesAndParams() step = 2 } @@ -171,7 +169,6 @@ } function popupListener(event) { - console.log('popupListener', event.data, event.origin, window.location.origin) let data = event.data if (event.origin == null || event.origin !== window.location.origin) { return @@ -187,7 +184,7 @@ value = data.res.access_token! valueToken = data.res step = 4 - if (expressOAuthSetup) { + if (express) { path = `u/${$userStore?.username}/${resourceType}_${new Date().getTime()}` next() } @@ -329,252 +326,254 @@ let editScopes = false - a.localeCompare(b)) - .map((key) => ({ - key - })) - : undefined} - bind:filteredItems={filteredConnects} - f={(x) => x.key} -/> - a[0].localeCompare(b[0]))} - bind:filteredItems={filteredConnectsManual} - f={(x) => x[0]} -/> -{#if step == 1} -
- -
- -

OAuth APIs

-
- {#if filteredConnects} - {#each filteredConnects as { key }} - - {/each} - {:else} - {#each new Array(3) as _} - - {/each} - {/if} -
- {#if connects && connects.length == 0} -
No OAuth APIs has been setup on the instance. To add oauth APIs, first sync the resource - types with the hub, then add oauth configuration. See documentation -
- {/if} - -

Others

- - {#if connectsManual && connectsManual?.length < 10} -
- Resource Types have not been synced with the hub. Go to the admins workspace to sync them (and - add a schedule to do daily): -

1. Go to the "admins" workspaces: - sync resource types -

-

- 2: Run the synchronization script: - sync resource types -

-
- {/if} - -
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if nativeLanguagesCategory.includes(key)} - - {/if} - {/each} - {/if} -
- -
-
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if !nativeLanguagesCategory.includes(key)} - - - {/if} - {/each} - {:else} - {#each new Array(9) as _} - - {/each} - {/if} -
-{:else if step == 2 && manual} - a.localeCompare(b)) + .map((key) => ({ + key + })) + : undefined} + bind:filteredItems={filteredConnects} + f={(x) => x.key} /> - - {#if apiTokenApps[resourceType]} -

Instructions

-
-
    - {#each apiTokenApps[resourceType].instructions as step} -
  1. - {@html step} -
    2. - {/each} -
-
- {#if apiTokenApps[resourceType].img} -
- connect -
- {/if} - {:else if !emptyString(resourceTypeInfo?.description)} -

{resourceTypeInfo?.name} description

-
- -
- {/if} - {#if resourceType == 'postgresql' || resourceType == 'mysql' || resourceType == 'mongodb'} - - {/if} - -

Resource description -
- - -
-

- {#if renderDescription} -
-
GH Markdown
-