diff --git a/backend/.sqlx/query-5c54f145e94dac117de02a94adf207684c52d8571b3507f4877c2cc151ff18b9.json b/backend/.sqlx/query-5c54f145e94dac117de02a94adf207684c52d8571b3507f4877c2cc151ff18b9.json new file mode 100644 index 0000000000..642f4a9592 --- /dev/null +++ b/backend/.sqlx/query-5c54f145e94dac117de02a94adf207684c52d8571b3507f4877c2cc151ff18b9.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Varchar", + "Timestamp", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5c54f145e94dac117de02a94adf207684c52d8571b3507f4877c2cc151ff18b9" +} diff --git a/backend/.sqlx/query-845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a.json b/backend/.sqlx/query-845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a.json new file mode 100644 index 0000000000..afede37f44 --- /dev/null +++ b/backend/.sqlx/query-845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a" +} diff --git a/backend/.sqlx/query-e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763.json b/backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json similarity index 69% rename from backend/.sqlx/query-e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763.json rename to backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json index 83ae3520d7..31501b4094 100644 --- a/backend/.sqlx/query-e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763.json +++ b/backend/.sqlx/query-958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5)", + "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763" + "hash": "958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c" } diff --git a/backend/.sqlx/query-dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9.json b/backend/.sqlx/query-dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9.json new file mode 100644 index 0000000000..bb8d339e55 --- /dev/null +++ b/backend/.sqlx/query-dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Timestamp" + ] + }, + "nullable": [ + null + ] + }, + "hash": "dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6337f452ae..d4c9c16960 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -588,7 +588,7 @@ dependencies = [ "futures-lite 2.3.0", "parking", "polling 3.7.3", - "rustix 0.38.35", + "rustix 0.38.34", "slab", "tracing", "windows-sys 0.59.0", @@ -1079,7 +1079,7 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.1", + "rustc_version 0.4.0", "tracing", ] @@ -1112,7 +1112,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -1363,7 +1363,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.3.1", + "constant_time_eq 0.3.0", ] [[package]] @@ -1508,9 +1508,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.17.1" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2" +checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" dependencies = [ "bytemuck_derive", ] @@ -1914,9 +1914,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "convert_case" @@ -2145,7 +2145,7 @@ dependencies = [ "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "rustc_version 0.4.1", + "rustc_version 0.4.0", "subtle", "zeroize", ] @@ -2683,14 +2683,14 @@ dependencies = [ "hyper-util", "ipnet", "percent-encoding", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.6", "serde", "serde_json", "tokio", "tokio-rustls 0.26.0", "tokio-socks", "tokio-util", - "tower 0.4.13", + "tower", "tower-http", "tower-service", ] @@ -2800,7 +2800,7 @@ dependencies = [ "rustls 0.23.12", "rustls-pemfile 2.1.3", "rustls-tokio-stream", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.6", "serde", "tokio", "webpki-roots 0.26.3", @@ -2926,7 +2926,7 @@ dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version 0.4.0", "syn 2.0.76", ] @@ -3311,9 +3311,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "filetime" -version = "0.2.25" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" dependencies = [ "cfg-if", "libc", @@ -3334,7 +3334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", - "rustc_version 0.4.1", + "rustc_version 0.4.0", ] [[package]] @@ -3464,7 +3464,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" dependencies = [ - "rustix 0.38.35", + "rustix 0.38.34", "windows-sys 0.52.0", ] @@ -4310,7 +4310,7 @@ dependencies = [ "pin-project-lite", "socket2 0.5.7", "tokio", - "tower 0.4.13", + "tower", "tower-service", "tracing", ] @@ -6052,7 +6052,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.35", + "rustix 0.38.34", "tracing", "windows-sys 0.59.0", ] @@ -6163,11 +6163,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.22.20", + "toml_edit 0.21.1", ] [[package]] @@ -7100,9 +7100,9 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ "semver 1.0.23", ] @@ -7136,9 +7136,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.6.0", "errno", @@ -7169,7 +7169,7 @@ dependencies = [ "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -7248,9 +7248,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -8270,15 +8270,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "95a5daa25ea337c85ed954c0496e3bdd2c7308cc3b24cf7b50d04876654c579f" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.36.1", ] [[package]] @@ -9094,7 +9094,7 @@ dependencies = [ "cfg-if", "fastrand 2.1.1", "once_cell", - "rustix 0.38.35", + "rustix 0.38.34", "windows-sys 0.59.0", ] @@ -9515,6 +9515,17 @@ dependencies = [ "winnow 0.5.40", ] +[[package]] +name = "toml_edit" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" +dependencies = [ + "indexmap 2.4.0", + "toml_datetime", + "winnow 0.5.40", +] + [[package]] name = "toml_edit" version = "0.22.20" @@ -9557,16 +9568,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b837f86b25d7c0d7988f00a54e74739be6477f2aac6201b8f429a7569991b7" -dependencies = [ - "tower-layer", - "tower-service", -] - [[package]] name = "tower-cookies" version = "0.10.0" @@ -9629,6 +9630,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.27" @@ -10403,7 +10416,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.35", + "rustix 0.38.34", ] [[package]] @@ -10414,7 +10427,7 @@ checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" dependencies = [ "either", "home", - "rustix 0.38.35", + "rustix 0.38.34", "winsafe", ] @@ -10480,6 +10493,7 @@ dependencies = [ "gethostname", "git-version", "lazy_static", + "object_store", "once_cell", "pg-embed", "prometheus", @@ -10538,7 +10552,7 @@ dependencies = [ "hmac", "http 1.1.0", "hyper 1.4.1", - "itertools 0.13.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "magic-crypt", @@ -10572,7 +10586,7 @@ dependencies = [ "tokio-native-tls", "tokio-tar", "tokio-util", - "tower 0.5.0", + "tower", "tower-cookies", "tower-http", "tracing", @@ -10636,12 +10650,13 @@ dependencies = [ "const_format", "cron", "futures-core", + "gethostname", "git-version", "hex", "hmac", "hyper 1.4.1", "indexmap 2.4.0", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "magic-crypt", "mail-send", @@ -10658,6 +10673,7 @@ dependencies = [ "tikv-jemalloc-ctl", "tokio", "tracing", + "tracing-appender", "tracing-flame", "tracing-loki", "tracing-subscriber", @@ -10727,7 +10743,7 @@ version = "1.388.0" dependencies = [ "anyhow", "gosyn", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "regex", "windmill-parser", @@ -10750,7 +10766,7 @@ name = "windmill-parser-php" version = "1.388.0" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "php-parser-rs", "serde_json", "windmill-parser", @@ -10761,7 +10777,7 @@ name = "windmill-parser-py" version = "1.388.0" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.10.5", "rustpython-parser", "serde_json", "windmill-parser", @@ -10773,7 +10789,7 @@ version = "1.388.0" dependencies = [ "anyhow", "async-recursion", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "phf", "regex", @@ -10866,7 +10882,7 @@ dependencies = [ "futures-core", "hex", "hmac", - "itertools 0.13.0", + "itertools 0.10.5", "lazy_static", "prometheus", "regex", @@ -10922,7 +10938,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", - "itertools 0.13.0", + "itertools 0.10.5", "jsonwebtoken", "lazy_static", "mappable-rc", @@ -11008,6 +11024,19 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc 0.36.1", + "windows_i686_gnu 0.36.1", + "windows_i686_msvc 0.36.1", + "windows_x86_64_gnu 0.36.1", + "windows_x86_64_msvc 0.36.1", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -11078,6 +11107,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -11090,6 +11125,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -11108,6 +11149,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -11120,6 +11167,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -11144,6 +11197,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -11207,7 +11266,7 @@ checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" dependencies = [ "libc", "linux-raw-sys 0.4.14", - "rustix 0.38.35", + "rustix 0.38.34", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0fff054078..64307eb624 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -48,7 +48,7 @@ flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"] loki = ["windmill-common/loki"] pg_embed = ["dep:pg-embed"] embedding = ["windmill-api/embedding"] -parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet"] +parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"] flow_testing = ["windmill-worker/flow_testing"] openidconnect = ["windmill-api/openidconnect"] @@ -85,6 +85,7 @@ gethostname.workspace = true serde_json.workspace = true serde.workspace = true deno_core.workspace = true +object_store = { workspace = true, optional = true } pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} quote.workspace = true @@ -139,6 +140,7 @@ chrono = { version = "0.4.35", features = ["serde"] } chrono-tz = "^0" tracing = "^0" tracing-subscriber = { version = "^0", features = ["env-filter", "json"] } +tracing-appender = "^0" prometheus = { version = "^0", default-features = false } cookie = { version = "0.17.0" } phf = { version = "0.11", features = ["macros"] } diff --git a/backend/migrations/20240613150524_add_job_perms.down.sql b/backend/migrations/20240613150524_add_job_perms.down.sql index d2f607c5b8..f489666b92 100644 --- a/backend/migrations/20240613150524_add_job_perms.down.sql +++ b/backend/migrations/20240613150524_add_job_perms.down.sql @@ -1 +1,2 @@ -- Add down migration script here +DROP TABLE public.job_perms; \ No newline at end of file diff --git a/backend/migrations/20240815085729_add_logfile.down.sql b/backend/migrations/20240815085729_add_logfile.down.sql new file mode 100644 index 0000000000..c2a9f2d62d --- /dev/null +++ b/backend/migrations/20240815085729_add_logfile.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TABLE log_file; +DROP TYPE LOG_MODE; + diff --git a/backend/migrations/20240815085729_add_logfile.up.sql b/backend/migrations/20240815085729_add_logfile.up.sql new file mode 100644 index 0000000000..1a79a9e7b1 --- /dev/null +++ b/backend/migrations/20240815085729_add_logfile.up.sql @@ -0,0 +1,16 @@ +-- Add up migration script here +-- Add up migration script here +CREATE TYPE LOG_MODE AS ENUM ('standalone', 'server', 'worker', 'agent', 'indexer'); +CREATE TABLE log_file ( + hostname VARCHAR(255) NOT NULL, + log_ts TIMESTAMP, + ok_lines BIGINT, + err_lines BIGINT, + mode LOG_MODE NOT NULL, + worker_group VARCHAR(255), + file_path VARCHAR(510) NOT NULL, + PRIMARY KEY (hostname, log_ts) +); + +CREATE INDEX log_file_log_ts_idx ON log_file (log_ts); +CREATE INDEX log_file_hostname_log_ts_idx ON log_file (hostname, log_ts); \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index e9ea129ab5..7100bf04f8 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -7,8 +7,8 @@ */ use anyhow::Context; -use gethostname::gethostname; use git_version::git_version; +use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store}; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; use std::{ @@ -39,7 +39,7 @@ use windmill_common::{ }, scripts::ScriptLang, stats_ee::schedule_stats, - utils::{rd_string, Mode}, + utils::{hostname, rd_string, Mode}, worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, WORKER_GROUP}, DB, METRICS_ENABLED, }; @@ -166,8 +166,10 @@ async fn windmill_main() -> anyhow::Result<()> { std::env::set_var("RUST_LOG", "info") } + let hostname = hostname(); + #[cfg(not(feature = "flamegraph"))] - windmill_common::tracing_init::initialize_tracing(); + let _guard = windmill_common::tracing_init::initialize_tracing(&hostname); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] tracing::info!("jemalloc enabled"); @@ -398,6 +400,8 @@ Windmill Community Edition {GIT_VERSION} monitor_pool(&db).await; + send_logs_to_object_store(&db, &hostname, &mode); + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] if !worker_mode { monitor_mem().await; @@ -483,6 +487,7 @@ Windmill Community Edition {GIT_VERSION} base_internal_url.clone(), rsmq.clone(), mode.clone() == Mode::Agent, + hostname.clone(), ) .await?; tracing::info!("All workers exited."); @@ -713,6 +718,8 @@ Windmill Community Edition {GIT_VERSION} } else { tracing::info!("Nothing to do, exiting."); } + send_current_log_file_to_object_store(&db, &hostname, &mode).await; + tracing::info!("Exiting connection pool"); tokio::select! { _ = db.close() => { @@ -784,6 +791,7 @@ pub async fn run_workers, agent_mode: bool, + hostname: String, ) -> anyhow::Result<()> { let mut killpill_rxs = vec![]; for _ in 0..num_workers { @@ -794,10 +802,6 @@ pub async fn run_workers (Option, Option) { + let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + let rd_dir = tokio::fs::read_dir(log_dir).await; + if let Ok(mut log_files) = rd_dir { + let mut highest_file: Option = None; + let mut second_highest_file: Option = None; + while let Ok(Some(file)) = log_files.next_entry().await { + let file_name = file + .file_name() + .to_str() + .map(|x| x.to_string()) + .unwrap_or_default(); + if file_name > highest_file.clone().unwrap_or_default() { + second_highest_file = highest_file; + highest_file = Some(file_name); + } + } + (highest_file, second_highest_file) + } else { + tracing::error!( + "Error reading log files: {TMP_WINDMILL_LOGS_SERVICE}, {:#?}", + rd_dir.unwrap_err() + ); + (None, None) + } +} + +fn get_worker_group(mode: &Mode) -> Option { + let worker_group = WORKER_GROUP.clone(); + if worker_group.is_empty() || mode == &Mode::Server || mode == &Mode::Indexer { + None + } else { + Some(worker_group) + } +} + +pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) { + let db = db.clone(); + let hostname = hostname.to_string(); + let mode = mode.clone(); + let worker_group = get_worker_group(&mode); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + sleep_until_next_minute_start_plus_one_s().await; + loop { + interval.tick().await; + let (_, snd_highest_file) = find_two_highest_files(&hostname).await; + send_log_file_to_object_store( + &hostname, + &mode, + &worker_group, + &db, + snd_highest_file, + false, + ) + .await; + } + }); +} + +pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) { + tracing::info!("Sending current log file to object store"); + let (highest_file, _) = find_two_highest_files(hostname).await; + let worker_group = get_worker_group(&mode); + send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await; +} + +fn get_now_and_str() -> (NaiveDateTime, String) { + let ts = Utc::now().naive_utc(); + ( + ts, + ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT) + .to_string(), + ) +} + +async fn send_log_file_to_object_store( + hostname: &str, + mode: &Mode, + worker_group: &Option, + db: &Pool, + snd_highest_file: Option, + use_now: bool, +) { + if let Some(highest_file) = snd_highest_file { + //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm + let (ts, ts_str) = if use_now { + get_now_and_str() + } else { + highest_file + .split(".") + .last() + .and_then(|x| { + NaiveDateTime::parse_from_str( + x, + windmill_common::tracing_init::LOG_TIMESTAMP_FMT, + ) + .ok() + .map(|y| (y, x.to_string())) + }) + .unwrap_or_else(get_now_and_str) + }; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)", + hostname, + ts + ) + .fetch_one(db) + .await; + + match exists { + Ok(Some(true)) => { + return; + } + Err(e) => { + tracing::error!("Error checking if log file exists: {:?}", e); + return; + } + _ => (), + } + + let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE) + .join(hostname) + .join(&highest_file); + + #[cfg(feature = "parquet")] + let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone(); + #[cfg(feature = "parquet")] + if let Some(s3_client) = s3_client { + //read file as byte stream + let bytes = tokio::fs::read(&path).await; + if let Err(e) = bytes { + tracing::error!("Error reading log file: {:?}", e); + return; + } + let path = object_store::path::Path::from_url_path(format!( + "{}{hostname}/{highest_file}", + windmill_common::tracing_init::LOGS_SERVICE + )); + if let Err(e) = path { + tracing::error!("Error creating log file path: {:?}", e); + return; + } + if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await { + tracing::error!("Error sending logs to object store: {:?}", e); + } + } + + let (ok_lines, err_lines) = read_log_counters(ts_str); + + if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7)", + hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64) + .execute(db) + .await { + tracing::error!("Error inserting log file: {:?}", e); + } + } +} + +fn read_log_counters(ts_str: String) -> (usize, usize) { + let counters = windmill_common::tracing_init::LOG_COUNTING_BY_MIN.read(); + let mut ok_lines = 0; + let mut err_lines = 0; + if let Ok(ref c) = counters { + let counter = c.get(&ts_str); + if let Some(counter) = counter { + ok_lines = counter.non_error_count; + err_lines = counter.error_count; + } else { + println!("no counter found for {ts_str}"); + } + } else { + println!("Error reading log counters 2"); + } + (ok_lines, err_lines) +} + pub async fn load_keep_job_dir(db: &DB) { let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await; match value { @@ -470,6 +661,15 @@ pub async fn delete_expired_items(db: &DB) -> () { { tracing::error!("Error deleting custom concurrency key: {:?}", e); } + if let Err(e) = sqlx::query!( + "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ", + job_retention_secs + ) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting log file: {:?}", e); + } } } Err(e) => { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 77cb1029af..5da17cded7 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -75,7 +75,9 @@ async fn initialize_tracing() { use std::sync::Once; static ONCE: Once = Once::new(); - ONCE.call_once(windmill_common::tracing_init::initialize_tracing); + ONCE.call_once(|| { + let _ = windmill_common::tracing_init::initialize_tracing("test"); + }); } /// it's important this is unique between tests as there is one prometheus registry and diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2e275e4f40..7bf1abd9af 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6137,7 +6137,7 @@ paths: /w/{workspace}/jobs_u/queue/cancel/{id}: post: - summary: cancel queued job + summary: cancel queued or running job operationId: cancelQueuedJob tags: - job @@ -8603,6 +8603,68 @@ paths: items: $ref: "#/components/schemas/TimeseriesMetric" + /service_logs/list_files: + get: + summary: list log files ordered by timestamp + operationId: listLogFiles + tags: + - service_logs + parameters: + - $ref: "#/components/parameters/Before" + - $ref: "#/components/parameters/After" + - name: with_error + in: query + required: false + schema: + type: boolean + responses: + "200": + description: time + content: + application/json: + schema: + type: array + items: + type: object + properties: + hostname: + type: string + mode: + type: string + worker_group: + type: string + log_ts: + type: string + format: date-time + file_path: + type: string + ok_lines: + type: integer + err_lines: + type: integer + required: + - hostname + - mode + - log_ts + - file_path + + /service_logs/get_log_file/{path}: + get: + summary: get log file by path + operationId: getLogFile + tags: + - service_logs + parameters: + - $ref: "#/components/parameters/Path" + responses: + "200": + description: log stream + content: + text/plain: + schema: + type: string + + /concurrency_groups/list: get: summary: List all concurrency groups @@ -8974,6 +9036,13 @@ components: schema: type: string format: date-time + Before: + name: before + description: filter on started before (inclusive) timestamp + in: query + schema: + type: string + format: date-time CreatedOrStartedAfter: name: created_or_started_after description: @@ -9050,13 +9119,6 @@ components: schema: type: string format: date-time - Before: - name: before - description: filter on created before (exclusive) timestamp - in: query - schema: - type: string - format: date-time Username: name: username description: filter on exact username of user diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index c7965b1581..784e9a06da 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -33,6 +33,7 @@ use crate::add_webhook_allowed_origin; use crate::concurrency_groups::join_concurrency_key; use crate::db::ApiAuthed; +use crate::utils::content_plain; use crate::{ db::DB, users::{check_scopes, require_owner_of_path, OptAuthed}, @@ -847,14 +848,6 @@ async fn get_logs_from_disk( return None; } -fn content_plain(body: Body) -> Response { - use axum::http::header; - Response::builder() - .header(header::CONTENT_TYPE, "text/plain") - .body(body) - .unwrap() -} - async fn get_job_logs( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index ed842aeb42..05202afd12 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -76,6 +76,7 @@ mod saml_ee; mod schedule; mod scim_ee; mod scripts; +mod service_logs; mod settings; pub mod smtp_server_ee; mod static_assets; @@ -291,6 +292,7 @@ pub async fn run_server( ) .nest("/settings", settings::global_service()) .nest("/workers", workers::global_service()) + .nest("/service_logs", service_logs::global_service()) .nest("/configs", configs::global_service()) .nest("/scripts", scripts::global_service()) .nest("/integrations", integration::global_service()) diff --git a/backend/windmill-api/src/service_logs.rs b/backend/windmill-api/src/service_logs.rs new file mode 100644 index 0000000000..f563f9be71 --- /dev/null +++ b/backend/windmill-api/src/service_logs.rs @@ -0,0 +1,139 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::utils::content_plain; +use axum::{body::Body, extract::Query, response::Response, routing::get, Extension, Json, Router}; +use serde::Serialize; + +use windmill_common::{ + error::{Error, JsonResult}, + utils::Pagination, +}; + +use crate::{ + db::{ApiAuthed, DB}, + utils::require_super_admin, +}; + +pub fn global_service() -> Router { + Router::new() + .route("/list_files", get(list_files)) + .route("/get_log_file/*path", get(get_log_file)) +} +use axum::extract::Path; + +#[derive(Debug, serde::Deserialize)] +pub struct LogFileQuery { + before: Option>, + after: Option>, + with_error: Option, +} + +#[derive(Debug, sqlx::FromRow, Serialize)] +pub struct LogFile { + pub hostname: String, + pub mode: String, + pub worker_group: Option, + pub log_ts: chrono::NaiveDateTime, + pub file_path: String, + pub ok_lines: Option, + pub err_lines: Option, +} +async fn list_files( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Query(pagination): Query, + Query(lq): Query, +) -> JsonResult> { + require_super_admin(&db, &email).await?; + let (per_page, offset) = windmill_common::utils::paginate(pagination); + + let mut sqlb = sql_builder::SqlBuilder::select_from("log_file") + .fields(&[ + "hostname", + "mode::text", + "worker_group", + "log_ts", + "file_path", + "ok_lines", + "err_lines", + ]) + .order_by("log_ts", true) + .offset(offset) + .limit(per_page) + .clone(); + + if let Some(dt) = &lq.before { + sqlb.and_where_le( + "log_ts", + format!("to_timestamp({} / 1000.0)", dt.timestamp_millis()), + ); + } + if let Some(dt) = &lq.after { + sqlb.and_where_ge( + "log_ts", + format!("to_timestamp({} / 1000.0)", dt.timestamp_millis()), + ); + } + + if let Some(true) = lq.with_error { + sqlb.and_where("err_lines > 0"); + } + let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?; + let rows = sqlx::query_as::<_, LogFile>(&sql).fetch_all(&db).await?; + Ok(Json(rows)) +} + +async fn get_log_file( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Path(path): Path, +) -> windmill_common::error::Result { + use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE; + + require_super_admin(&db, &email).await?; + let path = path.to_path(); + #[cfg(feature = "parquet")] + let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS + .read() + .await + .clone(); + #[cfg(feature = "parquet")] + if let Some(s3_client) = s3_client { + let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path); + let file = s3_client.get(&object_store::path::Path::from(path)).await; + match file { + Ok(file) => { + let bytes = file.bytes().await; + match bytes { + Ok(bytes) => { + return Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))); + } + Err(e) => { + return Err(Error::InternalErr(format!( + "Error pulling the bytes: {}", + e + ))); + } + } + } + Err(e) => { + return Err(Error::InternalErr(format!( + "Error fetching the file: {}", + e + ))); + } + } + } + let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await; + if let Ok(bytes) = file { + Ok(content_plain(Body::from(bytes::Bytes::from(bytes)))) + } else { + Err(Error::NotFound(format!("File {path} not found"))) + } +} diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 4836d0dfd5..d8a81ab741 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -6,6 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ +use axum::{body::Body, response::Response}; use regex::Regex; use serde::Deserialize; use sqlx::{Postgres, Transaction}; @@ -172,3 +173,11 @@ pub async fn get_and_delete_pending_username_or_generate<'c>( Ok(username) } } + +pub fn content_plain(body: Body) -> Response { + use axum::http::header; + Response::builder() + .header(header::CONTENT_TYPE, "text/plain") + .body(body) + .unwrap() +} diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index c324dcf728..5f45e051d4 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -37,6 +37,8 @@ reqwest = { workspace = true } tracing-subscriber = { workspace = true } lazy_static.workspace = true tracing-flame = { version = "^0", optional = true } +tracing-appender.workspace = true +gethostname.workspace = true itertools.workspace = true regex.workspace = true git-version.workspace = true diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index fe8366f399..d60f98a496 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -6,6 +6,8 @@ * LICENSE-AGPL for a copy of the license. */ +use const_format::concatcp; +use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard}; use tracing_subscriber::{ fmt::{format, Layer}, prelude::*, @@ -24,7 +26,11 @@ fn compact_layer() -> Layer WorkerGuard { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); let json_fmt = std::env::var("JSON_FMT") .map(|x| x == "true") @@ -38,6 +44,21 @@ pub fn initialize_tracing() { } let env_filter = EnvFilter::from_default_env(); + use tracing_appender::rolling::{RollingFileAppender, Rotation}; + + let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); + std::fs::create_dir_all(&log_dir).unwrap(); + let file_appender = RollingFileAppender::builder() + .rotation(Rotation::MINUTELY) + .filename_prefix(format!("{}.log", hostname)) + .max_log_files(20) + .build(log_dir) + .expect("Can build tracing file appender"); + + let (log_file_writer, _guard) = NonBlockingBuilder::default() + .lossy(false) + .finish(file_appender); + let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer); let ts_base = tracing_subscriber::registry().with(env_filter); @@ -51,17 +72,26 @@ pub fn initialize_tracing() { }; match json_fmt { - true => ts_base.with(json_layer().flatten_event(true)).init(), + true => ts_base + .with( + json_layer() + .with_writer(stdout_and_log_file_writer) + .flatten_event(true), + ) + .init(), false => ts_base .with( compact_layer() + .with_writer(stdout_and_log_file_writer) .with_ansi(style.to_lowercase() != "never") .with_file(true) .with_line_number(true) .with_target(false), ) + .with(CountingLayer::new()) .init(), } + _guard } #[cfg(feature = "flamegraph")] @@ -79,3 +109,75 @@ pub fn setup_flamegraph() -> impl Drop { .init(); _guard } + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; +use tracing::Event; +use tracing_subscriber::layer::Context; + +lazy_static::lazy_static! { + pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); +} + +#[derive(Debug)] +pub struct LogCounter { + pub non_error_count: usize, + pub error_count: usize, +} + +impl LogCounter { + fn new() -> Self { + LogCounter { non_error_count: 0, error_count: 0 } + } +} + +#[derive(Debug)] +struct CountingLayer {} + +impl CountingLayer { + pub fn new() -> Self { + CountingLayer {} + } +} + +// impl CountingLayer { +// pub fn new() -> Self { +// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) } +// } + +// pub fn get_counts(&self) -> (usize, usize) { +// let counter = self.counter.lock().unwrap(); +// (counter.non_error_count, counter.error_count) +// } + +// pub fn reset_counts(&self) { +// let mut counter = self.counter.lock().unwrap(); +// counter.reset(); +// } +// } + +pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M"; + +impl tracing_subscriber::Layer for CountingLayer +where + S: tracing::Subscriber, +{ + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + let level = *event.metadata().level(); + + let date_str = chrono::Utc::now().format(LOG_TIMESTAMP_FMT).to_string(); + let counters = LOG_COUNTING_BY_MIN.write(); + if let Ok(mut counters) = counters { + let counter = counters.entry(date_str).or_insert(LogCounter::new()); + if level == tracing::Level::ERROR { + counter.error_count += 1; + } else { + counter.non_error_count += 1; + } + } else { + println!("Error getting lock for log counting"); + } + } +} diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index fd14fc0fb0..4036a52418 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -14,6 +14,7 @@ use crate::global_settings::UNIQUE_ID_SETTING; use crate::server::Smtp; use crate::DB; use anyhow::Context; +use gethostname::gethostname; use git_version::git_version; use mail_send::mail_builder::MessageBuilder; use mail_send::SmtpClientBuilder; @@ -55,6 +56,13 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> { } } +pub fn hostname() -> String { + gethostname() + .to_str() + .map(|x| x.to_string()) + .unwrap_or_else(|| rd_string(5)) +} + pub fn paginate(pagination: Pagination) -> (usize, usize) { let per_page = pagination .per_page @@ -193,6 +201,18 @@ pub enum Mode { Indexer, } +impl std::fmt::Display for Mode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Mode::Worker => write!(f, "worker"), + Mode::Agent => write!(f, "agent"), + Mode::Server => write!(f, "server"), + Mode::Standalone => write!(f, "standalone"), + Mode::Indexer => write!(f, "indexer"), + } + } +} + pub async fn send_email( subject: &str, content: &str, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 5ea3210c82..93b80fd9d5 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -82,7 +82,7 @@ async fn add_relative_imports_to_dependency_map<'c>( for import in relative_imports { sqlx::query!( "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id) - VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5)", + VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING", w_id, script_path, import, diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 46b5693114..4427dc0f9d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -59,7 +59,7 @@ "svelte-carousel": "^1.0.25", "svelte-chartjs": "^3.1.5", "svelte-exmarkdown": "^3.0.5", - "svelte-infinite-loading": "^1.3.8", + "svelte-infinite-loading": "^1.4.0", "svelte-tiny-virtual-list": "^2.0.5", "tailwind-merge": "^1.13.2", "vscode": "npm:@codingame/monaco-vscode-api@~8.0.2", @@ -12461,9 +12461,10 @@ } }, "node_modules/svelte-infinite-loading": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/svelte-infinite-loading/-/svelte-infinite-loading-1.3.8.tgz", - "integrity": "sha512-hn4o848LKd2Q+M11hiMWnfFxM1GHKVDi92HPZ1FYvfed4bEeRZL+QvFAQzhy1SACq6Si0CAJcQFUZpIYmAEnpQ==" + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/svelte-infinite-loading/-/svelte-infinite-loading-1.4.0.tgz", + "integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg==", + "license": "MIT" }, "node_modules/svelte-multiselect": { "version": "10.2.0", diff --git a/frontend/package.json b/frontend/package.json index 800c69c313..544393e40c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -132,7 +132,7 @@ "svelte-carousel": "^1.0.25", "svelte-chartjs": "^3.1.5", "svelte-exmarkdown": "^3.0.5", - "svelte-infinite-loading": "^1.3.8", + "svelte-infinite-loading": "^1.4.0", "svelte-tiny-virtual-list": "^2.0.5", "tailwind-merge": "^1.13.2", "vscode": "npm:@codingame/monaco-vscode-api@~8.0.2", diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte index 522f4075f8..4cff07768c 100644 --- a/frontend/src/lib/components/LogViewer.svelte +++ b/frontend/src/lib/components/LogViewer.svelte @@ -26,6 +26,8 @@ export let tag: string | undefined export let small = false export let drawerOpen = false + export let noMaxH = false + export let noAutoScroll = false // @ts-ignore const ansi_up = new AnsiUp() @@ -144,17 +146,19 @@ - + {#if jobId} + + {/if} -
- Auto scroll - -
+ {#if !noAutoScroll} +
+ Auto scroll + +
+ {/if} {#if isLoading} diff --git a/frontend/src/lib/components/QueueMetricsDrawer.svelte b/frontend/src/lib/components/QueueMetricsDrawer.svelte index a3ae75e5ab..613a75e716 100644 --- a/frontend/src/lib/components/QueueMetricsDrawer.svelte +++ b/frontend/src/lib/components/QueueMetricsDrawer.svelte @@ -1,278 +1,15 @@ - - - + - {#if loading} - - {:else if noMetrics} -

No jobs delayed by more than 3 seconds in the last 14 days

- {:else} -
- {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { - title: { - display: true, - text: 'count' - } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' - }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } - } - } - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', - title: { - display: true, - text: 'delay (s)' - }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) - } - } - } - }} - /> - {/if} - - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. - -
- {/if} + diff --git a/frontend/src/lib/components/QueueMetricsDrawerInner.svelte b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte new file mode 100644 index 0000000000..28d223ef30 --- /dev/null +++ b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte @@ -0,0 +1,265 @@ + + + + +{#if loading} + +{:else if noMetrics} +

No jobs delayed by more than 3 seconds in the last 14 days

+{:else} +
+ {#if countData} + 3s)' + } + }, + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + y: { + title: { + display: true, + text: 'count' + } + } + } + }} + /> + {/if} + {#if delayData} + 3s)' + }, + tooltip: { + callbacks: { + label: function (context) { + // @ts-ignore + if (context.raw.y === 1) { + return context.dataset.label + ': 0' + } else { + // @ts-ignore + return context.dataset.label + ': ' + context.raw.y + } + } + } + } + }, + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + + y: { + type: 'logarithmic', + title: { + display: true, + text: 'delay (s)' + }, + ticks: { + callback: (value, _) => (value === 1 ? '0' : value) + } + } + } + }} + /> + {/if} + + Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are + included in the graph. + +
+{/if} diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte new file mode 100644 index 0000000000..f4350961ba --- /dev/null +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -0,0 +1,495 @@ + + +
+ + +
+
+
+ + +
+ { + minTs = undefined + maxTs = undefined + allLogs = undefined + getAllLogs(minTsManual, maxTsManual) + }} + serviceLogsChoices + loadText="Last 1000 logfiles" + /> +
+ + +
+
+
0' }} + on:change={() => { + allLogs = undefined + getAllLogs(minTs, maxTs) + }} + /> + { + if (e.detail) { + getAllLogs(maxTs, undefined) + } else { + timeout && clearTimeout(timeout) + } + }} + options={{ right: 'auto-refresh' }} + />
+ {#if allLogs == undefined} +
+ {:else if Object.keys(allLogs).length == 0} +
No logs
+ {:else if minTs && maxTs} + {@const minTsN = new Date(minTs).getTime()} + {@const maxTsN = new Date(maxTs).getTime()} + {@const diff = maxTsN - minTsN} +
+
+ +
{new Date(minTs).toLocaleTimeString([], { + day: '2-digit', + month: '2-digit', + hour: '2-digit', + minute: '2-digit' + })}
{new Date(maxTs).toLocaleTimeString([], { + day: '2-digit', + month: '2-digit', + hour: '2-digit', + minute: '2-digit' + })}
+
+ {#each Object.entries(allLogs) as [mode, o1]} +
+

{mode}s

+ {#each Object.entries(o1) as [wg, o2]} +
+ {#if wg && wg != ''} +

{wg}

+ {/if} +
+ {#each Object.entries(o2) as [hn, files]} + + +
{ + selected = [mode, wg, hn] + upToIsLatest = true + upTo = getLatestUpTo(selected) + scrollToBottom() + }} + > +
{hn}
+
+ {#each files as file} + {@const okHeight = 100.0 * ((file.ok_lines * 1.0) / (max_lines ?? 1))} + {@const errHeight = 100.0 * ((file.err_lines * 1.0) / (max_lines ?? 1))} +
+
+ {/each} +
+
+ {/each} +
+
+ {/each} +
+ {/each} + {/if} +
+ +
1 min delay: logs are compacted before being available
+ {#if selected} +
+ {#each getLogs(selected, upTo) as file} +
+
{new Date(file.ts).toLocaleTimeString([], { + day: '2-digit', + month: '2-digit', + hour: '2-digit', + minute: '2-digit' + })}
+ {#if logsContent[file.file_path] == undefined} +
+ {:else if logsContent[file.file_path]} + {#if logsContent[file.file_path].error} + {#if logsContent[file.file_path].error?.startsWith('Not Found')} +
Log file is missing. Log files require a shared log volume to be mounted + across servers and workers or to use the EE S3/object storage integration + for logs. To avoid mounting a shared volume, set the EE object store logs in + the instance settings
+ {:else} +
{logsContent[file.file_path].error}
+ {/if} + {:else if logsContent[file.file_path].content} + + +
+ {:else} +
No logs
+ {/if} + {/if} +
+ {/each} +
+
+
Last 5 log files up to:
+
+ {#if upTo} + + {:else} +
+ {/if} + +
+
+ {#if upTo} + + {:else} +
+ {/if} +
+
+ +
+
+ {:else} +
Select a host to see its logs
+ {/if}
+ +
diff --git a/frontend/src/lib/components/WorkspaceGroup.svelte b/frontend/src/lib/components/WorkspaceGroup.svelte index 7ada518172..63d2db50a8 100644 --- a/frontend/src/lib/components/WorkspaceGroup.svelte +++ b/frontend/src/lib/components/WorkspaceGroup.svelte @@ -870,7 +870,7 @@
{pluralize(activeWorkers, 'worker')} {#if vcpus_memory?.vcpus} - - {vcpus_memory?.vcpus} vCPUs{/if}{#if vcpus_memory?.memory} + - {vcpus_memory?.vcpus / 1000} vCPUs{/if}{#if vcpus_memory?.memory} - {vcpus_memory?.memory} MB{/if}
diff --git a/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte b/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte index 5ff9b63eb4..85b056bfcf 100644 --- a/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte +++ b/frontend/src/lib/components/apps/components/display/table/AppAggridExplorerTable.svelte @@ -150,7 +150,7 @@ outputs?.inputs.set(inputs, true) }, onRemove: (id, rowIndex) => { - if (inputs?.[id] == undefined) { + if (inputs?.[id] == undefined) { return } delete inputs[id][rowIndex] diff --git a/frontend/src/lib/components/common/button/Button.svelte b/frontend/src/lib/components/common/button/Button.svelte index c023f64b2c..a3695acf43 100644 --- a/frontend/src/lib/components/common/button/Button.svelte +++ b/frontend/src/lib/components/common/button/Button.svelte @@ -39,7 +39,7 @@ type MenuItem = { label: string - onClick?: () => void + onClick?: (e?: Event) => void href?: string icon?: any } diff --git a/frontend/src/lib/components/common/calendarPicker/CalendarPicker.svelte b/frontend/src/lib/components/common/calendarPicker/CalendarPicker.svelte index 1b6492f21a..4e1e7cadb5 100644 --- a/frontend/src/lib/components/common/calendarPicker/CalendarPicker.svelte +++ b/frontend/src/lib/components/common/calendarPicker/CalendarPicker.svelte @@ -9,6 +9,7 @@ export let label: string export let useDropdown: boolean = false export let clearable: boolean = false + export let target: string | HTMLElement | undefined = undefined const dispatch = createEventDispatcher() let input: HTMLInputElement @@ -16,7 +17,7 @@ export let placement: Placement = 'top-end' - +