feat: service logs (#4244)

2026-08-22 16:02:24 +00:00 · 2024-08-31 14:57:35 +02:00
parent 8e30928a78
commit 2fe48df720
36 changed files with 1636 additions and 432 deletions
@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Text",
+        "Varchar",
+        "Timestamp",
+        "Varchar",
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "5c54f145e94dac117de02a94adf207684c52d8571b3507f4877c2cc151ff18b9"
+}
@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "845120388af12c2b2f57fedf95c8cce4a406b74c9a8c8590a3bae323d59d046a"
+}
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n                 VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5)",
+  "query": "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)\n                 VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING",
  "describe": {
    "columns": [],
    "parameters": {
@@ -14,5 +14,5 @@
    },
    "nullable": []
  },
-  "hash": "e87c317550af671e4b65752a9c0b659cedf430fb7242faee4379b4a13e5fd763"
+  "hash": "958ed17dafffdd37e636ccd244dc4ca60cbf562e6f6a371d5f9a9943fb30254c"
 }
@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Timestamp"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "dee32ce9c4010ae407b4d1ba2ecf2062c36ee5dab3d4894ad4b2aa8b2f5a0db9"
+}
@@ -588,7 +588,7 @@ dependencies = [
 "futures-lite 2.3.0",
 "parking",
 "polling 3.7.3",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 "slab",
 "tracing",
 "windows-sys 0.59.0",
@@ -1079,7 +1079,7 @@ dependencies = [
 "aws-smithy-async",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
- "rustc_version 0.4.1",
+ "rustc_version 0.4.0",
 "tracing",
 ]

@@ -1112,7 +1112,7 @@ dependencies = [
 "serde_urlencoded",
 "sync_wrapper 1.0.1",
 "tokio",
- "tower 0.4.13",
+ "tower",
 "tower-layer",
 "tower-service",
 "tracing",
@@ -1363,7 +1363,7 @@ dependencies = [
 "arrayvec",
 "cc",
 "cfg-if",
- "constant_time_eq 0.3.1",
+ "constant_time_eq 0.3.0",
 ]

 [[package]]
@@ -1508,9 +1508,9 @@ dependencies = [

 [[package]]
 name = "bytemuck"
-version = "1.17.1"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2"
+checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31"
 dependencies = [
 "bytemuck_derive",
 ]
@@ -1914,9 +1914,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"

 [[package]]
 name = "constant_time_eq"
-version = "0.3.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"

 [[package]]
 name = "convert_case"
@@ -2145,7 +2145,7 @@ dependencies = [
 "curve25519-dalek-derive",
 "digest 0.10.7",
 "fiat-crypto",
- "rustc_version 0.4.1",
+ "rustc_version 0.4.0",
 "subtle",
 "zeroize",
 ]
@@ -2683,14 +2683,14 @@ dependencies = [
 "hyper-util",
 "ipnet",
 "percent-encoding",
- "rustls-webpki 0.102.7",
+ "rustls-webpki 0.102.6",
 "serde",
 "serde_json",
 "tokio",
 "tokio-rustls 0.26.0",
 "tokio-socks",
 "tokio-util",
- "tower 0.4.13",
+ "tower",
 "tower-http",
 "tower-service",
 ]
@@ -2800,7 +2800,7 @@ dependencies = [
 "rustls 0.23.12",
 "rustls-pemfile 2.1.3",
 "rustls-tokio-stream",
- "rustls-webpki 0.102.7",
+ "rustls-webpki 0.102.6",
 "serde",
 "tokio",
 "webpki-roots 0.26.3",
@@ -2926,7 +2926,7 @@ dependencies = [
 "convert_case 0.4.0",
 "proc-macro2",
 "quote",
- "rustc_version 0.4.1",
+ "rustc_version 0.4.0",
 "syn 2.0.76",
 ]

@@ -3311,9 +3311,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"

 [[package]]
 name = "filetime"
-version = "0.2.25"
+version = "0.2.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
+checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"
 dependencies = [
 "cfg-if",
 "libc",
@@ -3334,7 +3334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f"
 dependencies = [
 "bitflags 1.3.2",
- "rustc_version 0.4.1",
+ "rustc_version 0.4.0",
 ]

 [[package]]
@@ -3464,7 +3464,7 @@ version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
 dependencies = [
- "rustix 0.38.35",
+ "rustix 0.38.34",
 "windows-sys 0.52.0",
 ]

@@ -4310,7 +4310,7 @@ dependencies = [
 "pin-project-lite",
 "socket2 0.5.7",
 "tokio",
- "tower 0.4.13",
+ "tower",
 "tower-service",
 "tracing",
 ]
@@ -6052,7 +6052,7 @@ dependencies = [
 "concurrent-queue",
 "hermit-abi 0.4.0",
 "pin-project-lite",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 "tracing",
 "windows-sys 0.59.0",
 ]
@@ -6163,11 +6163,11 @@ dependencies = [

 [[package]]
 name = "proc-macro-crate"
-version = "3.2.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
+checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
 dependencies = [
- "toml_edit 0.22.20",
+ "toml_edit 0.21.1",
 ]

 [[package]]
@@ -7100,9 +7100,9 @@ dependencies = [

 [[package]]
 name = "rustc_version"
-version = "0.4.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
 dependencies = [
 "semver 1.0.23",
 ]
@@ -7136,9 +7136,9 @@ dependencies = [

 [[package]]
 name = "rustix"
-version = "0.38.35"
+version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
 "bitflags 2.6.0",
 "errno",
@@ -7169,7 +7169,7 @@ dependencies = [
 "once_cell",
 "ring 0.17.8",
 "rustls-pki-types",
- "rustls-webpki 0.102.7",
+ "rustls-webpki 0.102.6",
 "subtle",
 "zeroize",
 ]
@@ -7248,9 +7248,9 @@ dependencies = [

 [[package]]
 name = "rustls-webpki"
-version = "0.102.7"
+version = "0.102.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56"
+checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
 dependencies = [
 "ring 0.17.8",
 "rustls-pki-types",
@@ -8270,15 +8270,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"

 [[package]]
 name = "stacker"
-version = "0.1.17"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b"
+checksum = "95a5daa25ea337c85ed954c0496e3bdd2c7308cc3b24cf7b50d04876654c579f"
 dependencies = [
 "cc",
 "cfg-if",
 "libc",
 "psm",
- "windows-sys 0.59.0",
+ "windows-sys 0.36.1",
 ]

 [[package]]
@@ -9094,7 +9094,7 @@ dependencies = [
 "cfg-if",
 "fastrand 2.1.1",
 "once_cell",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 "windows-sys 0.59.0",
 ]

@@ -9515,6 +9515,17 @@ dependencies = [
 "winnow 0.5.40",
 ]

+[[package]]
+name = "toml_edit"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
+dependencies = [
+ "indexmap 2.4.0",
+ "toml_datetime",
+ "winnow 0.5.40",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.22.20"
@@ -9557,16 +9568,6 @@ dependencies = [
 "tracing",
 ]

-[[package]]
-name = "tower"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36b837f86b25d7c0d7988f00a54e74739be6477f2aac6201b8f429a7569991b7"
-dependencies = [
- "tower-layer",
- "tower-service",
-]
-
 [[package]]
 name = "tower-cookies"
 version = "0.10.0"
@@ -9629,6 +9630,18 @@ dependencies = [
 "tracing-core",
 ]

+[[package]]
+name = "tracing-appender"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
+dependencies = [
+ "crossbeam-channel",
+ "thiserror",
+ "time",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-attributes"
 version = "0.1.27"
@@ -10403,7 +10416,7 @@ dependencies = [
 "either",
 "home",
 "once_cell",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 ]

 [[package]]
@@ -10414,7 +10427,7 @@ checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f"
 dependencies = [
 "either",
 "home",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 "winsafe",
 ]

@@ -10480,6 +10493,7 @@ dependencies = [
 "gethostname",
 "git-version",
 "lazy_static",
+ "object_store",
 "once_cell",
 "pg-embed",
 "prometheus",
@@ -10538,7 +10552,7 @@ dependencies = [
 "hmac",
 "http 1.1.0",
 "hyper 1.4.1",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "jsonwebtoken",
 "lazy_static",
 "magic-crypt",
@@ -10572,7 +10586,7 @@ dependencies = [
 "tokio-native-tls",
 "tokio-tar",
 "tokio-util",
- "tower 0.5.0",
+ "tower",
 "tower-cookies",
 "tower-http",
 "tracing",
@@ -10636,12 +10650,13 @@ dependencies = [
 "const_format",
 "cron",
 "futures-core",
+ "gethostname",
 "git-version",
 "hex",
 "hmac",
 "hyper 1.4.1",
 "indexmap 2.4.0",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "lazy_static",
 "magic-crypt",
 "mail-send",
@@ -10658,6 +10673,7 @@ dependencies = [
 "tikv-jemalloc-ctl",
 "tokio",
 "tracing",
+ "tracing-appender",
 "tracing-flame",
 "tracing-loki",
 "tracing-subscriber",
@@ -10727,7 +10743,7 @@ version = "1.388.0"
 dependencies = [
 "anyhow",
 "gosyn",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "lazy_static",
 "regex",
 "windmill-parser",
@@ -10750,7 +10766,7 @@ name = "windmill-parser-php"
 version = "1.388.0"
 dependencies = [
 "anyhow",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "php-parser-rs",
 "serde_json",
 "windmill-parser",
@@ -10761,7 +10777,7 @@ name = "windmill-parser-py"
 version = "1.388.0"
 dependencies = [
 "anyhow",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "rustpython-parser",
 "serde_json",
 "windmill-parser",
@@ -10773,7 +10789,7 @@ version = "1.388.0"
 dependencies = [
 "anyhow",
 "async-recursion",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "lazy_static",
 "phf",
 "regex",
@@ -10866,7 +10882,7 @@ dependencies = [
 "futures-core",
 "hex",
 "hmac",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "lazy_static",
 "prometheus",
 "regex",
@@ -10922,7 +10938,7 @@ dependencies = [
 "gcp_auth",
 "git-version",
 "hex",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 "jsonwebtoken",
 "lazy_static",
 "mappable-rc",
@@ -11008,6 +11024,19 @@ dependencies = [
 "windows-targets 0.52.6",
 ]

+[[package]]
+name = "windows-sys"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+dependencies = [
+ "windows_aarch64_msvc 0.36.1",
+ "windows_i686_gnu 0.36.1",
+ "windows_i686_msvc 0.36.1",
+ "windows_x86_64_gnu 0.36.1",
+ "windows_x86_64_msvc 0.36.1",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -11078,6 +11107,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"

+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.48.5"
@@ -11090,6 +11125,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"

+[[package]]
+name = "windows_i686_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.48.5"
@@ -11108,6 +11149,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"

+[[package]]
+name = "windows_i686_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.48.5"
@@ -11120,6 +11167,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"

+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.48.5"
@@ -11144,6 +11197,12 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"

+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.48.5"
@@ -11207,7 +11266,7 @@ checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f"
 dependencies = [
 "libc",
 "linux-raw-sys 0.4.14",
- "rustix 0.38.35",
+ "rustix 0.38.34",
 ]

 [[package]]
@@ -48,7 +48,7 @@ flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
 pg_embed = ["dep:pg-embed"]
 embedding = ["windmill-api/embedding"]
-parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet"]
+parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"]
 prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
 flow_testing = ["windmill-worker/flow_testing"]
 openidconnect = ["windmill-api/openidconnect"]
@@ -85,6 +85,7 @@ gethostname.workspace = true
 serde_json.workspace = true
 serde.workspace = true
 deno_core.workspace = true
+object_store = { workspace = true, optional = true }
 pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']}
 quote.workspace = true

@@ -139,6 +140,7 @@ chrono = { version = "0.4.35", features = ["serde"] }
 chrono-tz = "^0"
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"] }
+tracing-appender = "^0"
 prometheus = { version = "^0", default-features = false }
 cookie = { version = "0.17.0" }
 phf = { version = "0.11", features = ["macros"] }
@@ -1 +1,2 @@
 -- Add down migration script here
+DROP TABLE public.job_perms;
@@ -0,0 +1,4 @@
+-- Add down migration script here
+DROP TABLE log_file;
+DROP TYPE LOG_MODE;
+
@@ -0,0 +1,16 @@
+-- Add up migration script here
+-- Add up migration script here
+CREATE TYPE LOG_MODE AS ENUM ('standalone', 'server', 'worker', 'agent', 'indexer');
+CREATE TABLE log_file (
+    hostname VARCHAR(255) NOT NULL,
+    log_ts TIMESTAMP,
+    ok_lines BIGINT,
+    err_lines BIGINT,
+    mode LOG_MODE NOT NULL,
+    worker_group VARCHAR(255),
+    file_path VARCHAR(510) NOT NULL,
+    PRIMARY KEY (hostname, log_ts)
+);
+
+CREATE INDEX log_file_log_ts_idx ON log_file (log_ts);
+CREATE INDEX log_file_hostname_log_ts_idx ON log_file (hostname, log_ts);
@@ -7,8 +7,8 @@
 */

 use anyhow::Context;
-use gethostname::gethostname;
 use git_version::git_version;
+use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store};
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
 use std::{
@@ -39,7 +39,7 @@ use windmill_common::{
    },
    scripts::ScriptLang,
    stats_ee::schedule_stats,
-    utils::{rd_string, Mode},
+    utils::{hostname, rd_string, Mode},
    worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, WORKER_GROUP},
    DB, METRICS_ENABLED,
 };
@@ -166,8 +166,10 @@ async fn windmill_main() -> anyhow::Result<()> {
        std::env::set_var("RUST_LOG", "info")
    }

+    let hostname = hostname();
+
    #[cfg(not(feature = "flamegraph"))]
-    windmill_common::tracing_init::initialize_tracing();
+    let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);

    #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
    tracing::info!("jemalloc enabled");
@@ -398,6 +400,8 @@ Windmill Community Edition {GIT_VERSION}

        monitor_pool(&db).await;

+        send_logs_to_object_store(&db, &hostname, &mode);
+
        #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
        if !worker_mode {
            monitor_mem().await;
@@ -483,6 +487,7 @@ Windmill Community Edition {GIT_VERSION}
                        base_internal_url.clone(),
                        rsmq.clone(),
                        mode.clone() == Mode::Agent,
+                        hostname.clone(),
                    )
                    .await?;
                    tracing::info!("All workers exited.");
@@ -713,6 +718,8 @@ Windmill Community Edition {GIT_VERSION}
    } else {
        tracing::info!("Nothing to do, exiting.");
    }
+    send_current_log_file_to_object_store(&db, &hostname, &mode).await;
+
    tracing::info!("Exiting connection pool");
    tokio::select! {
        _ = db.close() => {
@@ -784,6 +791,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
    base_internal_url: String,
    rsmq: Option<R>,
    agent_mode: bool,
+    hostname: String,
 ) -> anyhow::Result<()> {
    let mut killpill_rxs = vec![];
    for _ in 0..num_workers {
@@ -794,10 +802,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
        tracing::info!("Received killpill, exiting");
        return Ok(());
    }
-    let hostname = gethostname()
-        .to_str()
-        .map(|x| x.to_string())
-        .unwrap_or_else(|| rd_string(5));
    let instance_name = hostname
        .clone()
        .replace(" ", "")
@@ -7,6 +7,7 @@ use std::{
    time::Duration,
 };

+use chrono::{NaiveDateTime, Utc};
 use rsmq_async::MultiplexedRsmq;
 use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
@@ -42,10 +43,11 @@ use windmill_common::{
    oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
    server::load_server_config,
    users::truncate_token,
-    utils::{now_from_db, rd_string, report_critical_error},
+    utils::{now_from_db, rd_string, report_critical_error, Mode},
    worker::{
        load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
        DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
+        WORKER_GROUP,
    },
    BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
    METRICS_DEBUG_ENABLED, METRICS_ENABLED,
@@ -336,6 +338,195 @@ pub async fn monitor_mem() {
    });
 }

+async fn sleep_until_next_minute_start_plus_one_s() {
+    let now = Utc::now();
+    let next_minute = now + Duration::from_secs(60 - now.timestamp() as u64 % 60 + 1);
+    tokio::time::sleep(tokio::time::Duration::from_secs(
+        next_minute.timestamp() as u64 - now.timestamp() as u64,
+    ))
+    .await;
+}
+
+use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE;
+async fn find_two_highest_files(hostname: &str) -> (Option<String>, Option<String>) {
+    let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
+    let rd_dir = tokio::fs::read_dir(log_dir).await;
+    if let Ok(mut log_files) = rd_dir {
+        let mut highest_file: Option<String> = None;
+        let mut second_highest_file: Option<String> = None;
+        while let Ok(Some(file)) = log_files.next_entry().await {
+            let file_name = file
+                .file_name()
+                .to_str()
+                .map(|x| x.to_string())
+                .unwrap_or_default();
+            if file_name > highest_file.clone().unwrap_or_default() {
+                second_highest_file = highest_file;
+                highest_file = Some(file_name);
+            }
+        }
+        (highest_file, second_highest_file)
+    } else {
+        tracing::error!(
+            "Error reading log files: {TMP_WINDMILL_LOGS_SERVICE}, {:#?}",
+            rd_dir.unwrap_err()
+        );
+        (None, None)
+    }
+}
+
+fn get_worker_group(mode: &Mode) -> Option<String> {
+    let worker_group = WORKER_GROUP.clone();
+    if worker_group.is_empty() || mode == &Mode::Server || mode == &Mode::Indexer {
+        None
+    } else {
+        Some(worker_group)
+    }
+}
+
+pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
+    let db = db.clone();
+    let hostname = hostname.to_string();
+    let mode = mode.clone();
+    let worker_group = get_worker_group(&mode);
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(10));
+        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+        sleep_until_next_minute_start_plus_one_s().await;
+        loop {
+            interval.tick().await;
+            let (_, snd_highest_file) = find_two_highest_files(&hostname).await;
+            send_log_file_to_object_store(
+                &hostname,
+                &mode,
+                &worker_group,
+                &db,
+                snd_highest_file,
+                false,
+            )
+            .await;
+        }
+    });
+}
+
+pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
+    tracing::info!("Sending current log file to object store");
+    let (highest_file, _) = find_two_highest_files(hostname).await;
+    let worker_group = get_worker_group(&mode);
+    send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await;
+}
+
+fn get_now_and_str() -> (NaiveDateTime, String) {
+    let ts = Utc::now().naive_utc();
+    (
+        ts,
+        ts.format(windmill_common::tracing_init::LOG_TIMESTAMP_FMT)
+            .to_string(),
+    )
+}
+
+async fn send_log_file_to_object_store(
+    hostname: &str,
+    mode: &Mode,
+    worker_group: &Option<String>,
+    db: &Pool<Postgres>,
+    snd_highest_file: Option<String>,
+    use_now: bool,
+) {
+    if let Some(highest_file) = snd_highest_file {
+        //parse datetime frome file xxxx.yyyy-MM-dd-HH-mm
+        let (ts, ts_str) = if use_now {
+            get_now_and_str()
+        } else {
+            highest_file
+                .split(".")
+                .last()
+                .and_then(|x| {
+                    NaiveDateTime::parse_from_str(
+                        x,
+                        windmill_common::tracing_init::LOG_TIMESTAMP_FMT,
+                    )
+                    .ok()
+                    .map(|y| (y, x.to_string()))
+                })
+                .unwrap_or_else(get_now_and_str)
+        };
+
+        let exists = sqlx::query_scalar!(
+            "SELECT EXISTS(SELECT 1 FROM log_file WHERE hostname = $1 AND log_ts = $2)",
+            hostname,
+            ts
+        )
+        .fetch_one(db)
+        .await;
+
+        match exists {
+            Ok(Some(true)) => {
+                return;
+            }
+            Err(e) => {
+                tracing::error!("Error checking if log file exists: {:?}", e);
+                return;
+            }
+            _ => (),
+        }
+
+        let path = std::path::Path::new(TMP_WINDMILL_LOGS_SERVICE)
+            .join(hostname)
+            .join(&highest_file);
+
+        #[cfg(feature = "parquet")]
+        let s3_client = OBJECT_STORE_CACHE_SETTINGS.read().await.clone();
+        #[cfg(feature = "parquet")]
+        if let Some(s3_client) = s3_client {
+            //read file as byte stream
+            let bytes = tokio::fs::read(&path).await;
+            if let Err(e) = bytes {
+                tracing::error!("Error reading log file: {:?}", e);
+                return;
+            }
+            let path = object_store::path::Path::from_url_path(format!(
+                "{}{hostname}/{highest_file}",
+                windmill_common::tracing_init::LOGS_SERVICE
+            ));
+            if let Err(e) = path {
+                tracing::error!("Error creating log file path: {:?}", e);
+                return;
+            }
+            if let Err(e) = s3_client.put(&path.unwrap(), bytes.unwrap().into()).await {
+                tracing::error!("Error sending logs to object store: {:?}", e);
+            }
+        }
+
+        let (ok_lines, err_lines) = read_log_counters(ts_str);
+
+        if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7)", 
+            hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64)
+            .execute(db)
+            .await {
+            tracing::error!("Error inserting log file: {:?}", e);
+        }
+    }
+}
+
+fn read_log_counters(ts_str: String) -> (usize, usize) {
+    let counters = windmill_common::tracing_init::LOG_COUNTING_BY_MIN.read();
+    let mut ok_lines = 0;
+    let mut err_lines = 0;
+    if let Ok(ref c) = counters {
+        let counter = c.get(&ts_str);
+        if let Some(counter) = counter {
+            ok_lines = counter.non_error_count;
+            err_lines = counter.error_count;
+        } else {
+            println!("no counter found for {ts_str}");
+        }
+    } else {
+        println!("Error reading log counters 2");
+    }
+    (ok_lines, err_lines)
+}
+
 pub async fn load_keep_job_dir(db: &DB) {
    let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await;
    match value {
@@ -470,6 +661,15 @@ pub async fn delete_expired_items(db: &DB) -> () {
                            {
                                tracing::error!("Error deleting  custom concurrency key: {:?}", e);
                            }
+                            if let Err(e) = sqlx::query!(
+                                "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval ",
+                                job_retention_secs
+                            )
+                            .execute(&mut *tx)
+                            .await
+                            {
+                                tracing::error!("Error deleting log file: {:?}", e);
+                            }
                        }
                    }
                    Err(e) => {
@@ -75,7 +75,9 @@ async fn initialize_tracing() {
    use std::sync::Once;

    static ONCE: Once = Once::new();
-    ONCE.call_once(windmill_common::tracing_init::initialize_tracing);
+    ONCE.call_once(|| {
+        let _ = windmill_common::tracing_init::initialize_tracing("test");
+    });
 }

 /// it's important this is unique between tests as there is one prometheus registry and
@@ -6137,7 +6137,7 @@ paths:

  /w/{workspace}/jobs_u/queue/cancel/{id}:
    post:
-      summary: cancel queued job
+      summary: cancel queued or running job
      operationId: cancelQueuedJob
      tags:
        - job
@@ -8603,6 +8603,68 @@ paths:
                    items:
                      $ref: "#/components/schemas/TimeseriesMetric"

+  /service_logs/list_files:
+    get:
+      summary: list log files ordered by timestamp
+      operationId: listLogFiles
+      tags:
+        - service_logs
+      parameters:
+        - $ref: "#/components/parameters/Before"
+        - $ref: "#/components/parameters/After"
+        - name: with_error
+          in: query
+          required: false
+          schema:
+            type: boolean
+      responses:
+        "200":
+          description: time
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    hostname:
+                      type: string
+                    mode:
+                      type: string
+                    worker_group:
+                      type: string
+                    log_ts:
+                      type: string
+                      format: date-time
+                    file_path:
+                      type: string
+                    ok_lines:
+                      type: integer
+                    err_lines:
+                      type: integer
+                  required:
+                    - hostname
+                    - mode
+                    - log_ts
+                    - file_path
+
+  /service_logs/get_log_file/{path}:
+    get:
+      summary: get log file by path
+      operationId: getLogFile
+      tags:
+        - service_logs
+      parameters:
+        - $ref: "#/components/parameters/Path"            
+      responses:
+        "200":
+          description: log stream
+          content:
+            text/plain:
+              schema:
+                type: string
+
+
  /concurrency_groups/list:
    get:
      summary: List all concurrency groups
@@ -8974,6 +9036,13 @@ components:
      schema:
        type: string
        format: date-time
+    Before:
+      name: before
+      description: filter on started before (inclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
    CreatedOrStartedAfter:
      name: created_or_started_after
      description:
@@ -9050,13 +9119,6 @@ components:
      schema:
        type: string
        format: date-time
-    Before:
-      name: before
-      description: filter on created before (exclusive) timestamp
-      in: query
-      schema:
-        type: string
-        format: date-time
    Username:
      name: username
      description: filter on exact username of user
@@ -33,6 +33,7 @@ use crate::add_webhook_allowed_origin;
 use crate::concurrency_groups::join_concurrency_key;
 use crate::db::ApiAuthed;

+use crate::utils::content_plain;
 use crate::{
    db::DB,
    users::{check_scopes, require_owner_of_path, OptAuthed},
@@ -847,14 +848,6 @@ async fn get_logs_from_disk(
    return None;
 }

-fn content_plain(body: Body) -> Response {
-    use axum::http::header;
-    Response::builder()
-        .header(header::CONTENT_TYPE, "text/plain")
-        .body(body)
-        .unwrap()
-}
-
 async fn get_job_logs(
    OptAuthed(opt_authed): OptAuthed,
    Extension(db): Extension<DB>,
@@ -76,6 +76,7 @@ mod saml_ee;
 mod schedule;
 mod scim_ee;
 mod scripts;
+mod service_logs;
 mod settings;
 pub mod smtp_server_ee;
 mod static_assets;
@@ -291,6 +292,7 @@ pub async fn run_server(
                )
                .nest("/settings", settings::global_service())
                .nest("/workers", workers::global_service())
+                .nest("/service_logs", service_logs::global_service())
                .nest("/configs", configs::global_service())
                .nest("/scripts", scripts::global_service())
                .nest("/integrations", integration::global_service())
@@ -0,0 +1,139 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::utils::content_plain;
+use axum::{body::Body, extract::Query, response::Response, routing::get, Extension, Json, Router};
+use serde::Serialize;
+
+use windmill_common::{
+    error::{Error, JsonResult},
+    utils::Pagination,
+};
+
+use crate::{
+    db::{ApiAuthed, DB},
+    utils::require_super_admin,
+};
+
+pub fn global_service() -> Router {
+    Router::new()
+        .route("/list_files", get(list_files))
+        .route("/get_log_file/*path", get(get_log_file))
+}
+use axum::extract::Path;
+
+#[derive(Debug, serde::Deserialize)]
+pub struct LogFileQuery {
+    before: Option<chrono::DateTime<chrono::Utc>>,
+    after: Option<chrono::DateTime<chrono::Utc>>,
+    with_error: Option<bool>,
+}
+
+#[derive(Debug, sqlx::FromRow, Serialize)]
+pub struct LogFile {
+    pub hostname: String,
+    pub mode: String,
+    pub worker_group: Option<String>,
+    pub log_ts: chrono::NaiveDateTime,
+    pub file_path: String,
+    pub ok_lines: Option<i64>,
+    pub err_lines: Option<i64>,
+}
+async fn list_files(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Query(pagination): Query<Pagination>,
+    Query(lq): Query<LogFileQuery>,
+) -> JsonResult<Vec<LogFile>> {
+    require_super_admin(&db, &email).await?;
+    let (per_page, offset) = windmill_common::utils::paginate(pagination);
+
+    let mut sqlb = sql_builder::SqlBuilder::select_from("log_file")
+        .fields(&[
+            "hostname",
+            "mode::text",
+            "worker_group",
+            "log_ts",
+            "file_path",
+            "ok_lines",
+            "err_lines",
+        ])
+        .order_by("log_ts", true)
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+
+    if let Some(dt) = &lq.before {
+        sqlb.and_where_le(
+            "log_ts",
+            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
+        );
+    }
+    if let Some(dt) = &lq.after {
+        sqlb.and_where_ge(
+            "log_ts",
+            format!("to_timestamp({}  / 1000.0)", dt.timestamp_millis()),
+        );
+    }
+
+    if let Some(true) = lq.with_error {
+        sqlb.and_where("err_lines > 0");
+    }
+    let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+    let rows = sqlx::query_as::<_, LogFile>(&sql).fetch_all(&db).await?;
+    Ok(Json(rows))
+}
+
+async fn get_log_file(
+    ApiAuthed { email, .. }: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(path): Path<windmill_common::utils::StripPath>,
+) -> windmill_common::error::Result<Response> {
+    use windmill_common::tracing_init::TMP_WINDMILL_LOGS_SERVICE;
+
+    require_super_admin(&db, &email).await?;
+    let path = path.to_path();
+    #[cfg(feature = "parquet")]
+    let s3_client = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
+        .read()
+        .await
+        .clone();
+    #[cfg(feature = "parquet")]
+    if let Some(s3_client) = s3_client {
+        let path = format!("{}{}", windmill_common::tracing_init::LOGS_SERVICE, path);
+        let file = s3_client.get(&object_store::path::Path::from(path)).await;
+        match file {
+            Ok(file) => {
+                let bytes = file.bytes().await;
+                match bytes {
+                    Ok(bytes) => {
+                        return Ok(content_plain(Body::from(bytes::Bytes::from(bytes))));
+                    }
+                    Err(e) => {
+                        return Err(Error::InternalErr(format!(
+                            "Error pulling the bytes: {}",
+                            e
+                        )));
+                    }
+                }
+            }
+            Err(e) => {
+                return Err(Error::InternalErr(format!(
+                    "Error fetching the file: {}",
+                    e
+                )));
+            }
+        }
+    }
+    let file = tokio::fs::read(format!("{}{}", TMP_WINDMILL_LOGS_SERVICE, path)).await;
+    if let Ok(bytes) = file {
+        Ok(content_plain(Body::from(bytes::Bytes::from(bytes))))
+    } else {
+        Err(Error::NotFound(format!("File {path} not found")))
+    }
+}
@@ -6,6 +6,7 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use axum::{body::Body, response::Response};
 use regex::Regex;
 use serde::Deserialize;
 use sqlx::{Postgres, Transaction};
@@ -172,3 +173,11 @@ pub async fn get_and_delete_pending_username_or_generate<'c>(
        Ok(username)
    }
 }
+
+pub fn content_plain(body: Body) -> Response {
+    use axum::http::header;
+    Response::builder()
+        .header(header::CONTENT_TYPE, "text/plain")
+        .body(body)
+        .unwrap()
+}
@@ -37,6 +37,8 @@ reqwest = { workspace = true }
 tracing-subscriber = { workspace = true }
 lazy_static.workspace = true
 tracing-flame = { version = "^0", optional = true }
+tracing-appender.workspace = true
+gethostname.workspace = true
 itertools.workspace = true
 regex.workspace = true
 git-version.workspace = true
@@ -6,6 +6,8 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use const_format::concatcp;
+use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
 use tracing_subscriber::{
    fmt::{format, Layer},
    prelude::*,
@@ -24,7 +26,11 @@ fn compact_layer<S>() -> Layer<S, format::DefaultFields, format::Format<format::
    tracing_subscriber::fmt::layer().compact()
 }

-pub fn initialize_tracing() {
+pub const LOGS_SERVICE: &str = "logs/services/";
+
+pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE);
+
+pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
    let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
    let json_fmt = std::env::var("JSON_FMT")
        .map(|x| x == "true")
@@ -38,6 +44,21 @@ pub fn initialize_tracing() {
    }

    let env_filter = EnvFilter::from_default_env();
+    use tracing_appender::rolling::{RollingFileAppender, Rotation};
+
+    let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
+    std::fs::create_dir_all(&log_dir).unwrap();
+    let file_appender = RollingFileAppender::builder()
+        .rotation(Rotation::MINUTELY)
+        .filename_prefix(format!("{}.log", hostname))
+        .max_log_files(20)
+        .build(log_dir)
+        .expect("Can build tracing file appender");
+
+    let (log_file_writer, _guard) = NonBlockingBuilder::default()
+        .lossy(false)
+        .finish(file_appender);
+    let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer);

    let ts_base = tracing_subscriber::registry().with(env_filter);

@@ -51,17 +72,26 @@ pub fn initialize_tracing() {
    };

    match json_fmt {
-        true => ts_base.with(json_layer().flatten_event(true)).init(),
+        true => ts_base
+            .with(
+                json_layer()
+                    .with_writer(stdout_and_log_file_writer)
+                    .flatten_event(true),
+            )
+            .init(),
        false => ts_base
            .with(
                compact_layer()
+                    .with_writer(stdout_and_log_file_writer)
                    .with_ansi(style.to_lowercase() != "never")
                    .with_file(true)
                    .with_line_number(true)
                    .with_target(false),
            )
+            .with(CountingLayer::new())
            .init(),
    }
+    _guard
 }

 #[cfg(feature = "flamegraph")]
@@ -79,3 +109,75 @@ pub fn setup_flamegraph() -> impl Drop {
        .init();
    _guard
 }
+
+use std::{
+    collections::HashMap,
+    sync::{Arc, RwLock},
+};
+use tracing::Event;
+use tracing_subscriber::layer::Context;
+
+lazy_static::lazy_static! {
+    pub static ref LOG_COUNTING_BY_MIN: Arc<RwLock<HashMap<String, LogCounter>>> = Arc::new(RwLock::new(HashMap::new()));
+}
+
+#[derive(Debug)]
+pub struct LogCounter {
+    pub non_error_count: usize,
+    pub error_count: usize,
+}
+
+impl LogCounter {
+    fn new() -> Self {
+        LogCounter { non_error_count: 0, error_count: 0 }
+    }
+}
+
+#[derive(Debug)]
+struct CountingLayer {}
+
+impl CountingLayer {
+    pub fn new() -> Self {
+        CountingLayer {}
+    }
+}
+
+// impl CountingLayer {
+//     pub fn new() -> Self {
+//         CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) }
+//     }
+
+//     pub fn get_counts(&self) -> (usize, usize) {
+//         let counter = self.counter.lock().unwrap();
+//         (counter.non_error_count, counter.error_count)
+//     }
+
+//     pub fn reset_counts(&self) {
+//         let mut counter = self.counter.lock().unwrap();
+//         counter.reset();
+//     }
+// }
+
+pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M";
+
+impl<S> tracing_subscriber::Layer<S> for CountingLayer
+where
+    S: tracing::Subscriber,
+{
+    fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
+        let level = *event.metadata().level();
+
+        let date_str = chrono::Utc::now().format(LOG_TIMESTAMP_FMT).to_string();
+        let counters = LOG_COUNTING_BY_MIN.write();
+        if let Ok(mut counters) = counters {
+            let counter = counters.entry(date_str).or_insert(LogCounter::new());
+            if level == tracing::Level::ERROR {
+                counter.error_count += 1;
+            } else {
+                counter.non_error_count += 1;
+            }
+        } else {
+            println!("Error getting lock for log counting");
+        }
+    }
+}
@@ -14,6 +14,7 @@ use crate::global_settings::UNIQUE_ID_SETTING;
 use crate::server::Smtp;
 use crate::DB;
 use anyhow::Context;
+use gethostname::gethostname;
 use git_version::git_version;
 use mail_send::mail_builder::MessageBuilder;
 use mail_send::SmtpClientBuilder;
@@ -55,6 +56,13 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> {
    }
 }

+pub fn hostname() -> String {
+    gethostname()
+        .to_str()
+        .map(|x| x.to_string())
+        .unwrap_or_else(|| rd_string(5))
+}
+
 pub fn paginate(pagination: Pagination) -> (usize, usize) {
    let per_page = pagination
        .per_page
@@ -193,6 +201,18 @@ pub enum Mode {
    Indexer,
 }

+impl std::fmt::Display for Mode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Mode::Worker => write!(f, "worker"),
+            Mode::Agent => write!(f, "agent"),
+            Mode::Server => write!(f, "server"),
+            Mode::Standalone => write!(f, "standalone"),
+            Mode::Indexer => write!(f, "indexer"),
+        }
+    }
+}
+
 pub async fn send_email(
    subject: &str,
    content: &str,
@@ -82,7 +82,7 @@ async fn add_relative_imports_to_dependency_map<'c>(
    for import in relative_imports {
        sqlx::query!(
            "INSERT INTO dependency_map (workspace_id, importer_path, importer_kind, imported_path, importer_node_id)
-                 VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5)",
+                 VALUES ($1, $2, $4::text::IMPORTER_KIND, $3, $5) ON CONFLICT DO NOTHING",
            w_id,
            script_path,
            import,
@@ -59,7 +59,7 @@
 				"svelte-carousel": "^1.0.25",
 				"svelte-chartjs": "^3.1.5",
 				"svelte-exmarkdown": "^3.0.5",
-				"svelte-infinite-loading": "^1.3.8",
+				"svelte-infinite-loading": "^1.4.0",
 				"svelte-tiny-virtual-list": "^2.0.5",
 				"tailwind-merge": "^1.13.2",
 				"vscode": "npm:@codingame/monaco-vscode-api@~8.0.2",
@@ -12461,9 +12461,10 @@
 			}
 		},
 		"node_modules/svelte-infinite-loading": {
-			"version": "1.3.8",
-			"resolved": "https://registry.npmjs.org/svelte-infinite-loading/-/svelte-infinite-loading-1.3.8.tgz",
-			"integrity": "sha512-hn4o848LKd2Q+M11hiMWnfFxM1GHKVDi92HPZ1FYvfed4bEeRZL+QvFAQzhy1SACq6Si0CAJcQFUZpIYmAEnpQ=="
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/svelte-infinite-loading/-/svelte-infinite-loading-1.4.0.tgz",
+			"integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg==",
+			"license": "MIT"
 		},
 		"node_modules/svelte-multiselect": {
 			"version": "10.2.0",
@@ -132,7 +132,7 @@
 		"svelte-carousel": "^1.0.25",
 		"svelte-chartjs": "^3.1.5",
 		"svelte-exmarkdown": "^3.0.5",
-		"svelte-infinite-loading": "^1.3.8",
+		"svelte-infinite-loading": "^1.4.0",
 		"svelte-tiny-virtual-list": "^2.0.5",
 		"tailwind-merge": "^1.13.2",
 		"vscode": "npm:@codingame/monaco-vscode-api@~8.0.2",
@@ -26,6 +26,8 @@
 	export let tag: string | undefined
 	export let small = false
 	export let drawerOpen = false
+	export let noMaxH = false
+	export let noAutoScroll = false

 	// @ts-ignore
 	const ansi_up = new AnsiUp()
@@ -144,17 +146,19 @@
 <Drawer bind:this={logViewer} bind:open={drawerOpen} size="900px">
 	<DrawerContent title="Expanded Logs" on:close={logViewer.closeDrawer}>
 		<svelte:fragment slot="actions">
-			<Button
-				href="{base}/api/w/{$workspaceStore}/jobs_u/get_logs/{jobId}"
-				download="windmill_logs_{jobId}.txt"
-				color="light"
-				size="xs"
-				startIcon={{
-					icon: Download
-				}}
-			>
-				Download
-			</Button>
+			{#if jobId}
+				<Button
+					href="{base}/api/w/{$workspaceStore}/jobs_u/get_logs/{jobId}"
+					download="windmill_logs_{jobId}.txt"
+					color="light"
+					size="xs"
+					startIcon={{
+						icon: Download
+					}}
+				>
+					Download
+				</Button>
+			{/if}

 			<Button
 				on:click={() => copyToClipboard(content)}
@@ -186,28 +190,32 @@
 <div class="relative w-full h-full {wrapperClass}">
 	<div
 		bind:this={div}
-		class="w-full h-full overflow-auto relative bg-surface-secondary max-h-screen"
+		class="w-full h-full overflow-auto relative bg-surface-secondary {noMaxH ? '' : 'max-h-screen'}"
 	>
 		<div class="sticky z-10 top-0 right-0 w-full flex flex-row-reverse justify-between text-sm">
 			<div class="flex gap-2 pl-0.5 bg-surface-secondary">
-				<div class="flex items-center">
-					<a
-						class="text-primary pb-0.5"
-						target="_blank"
-						href="{base}/api/w/{$workspaceStore}/jobs_u/get_logs/{jobId}"
-						download="windmill_logs_{jobId}.txt"
-						><Download size="14" />
-					</a>
-				</div>
+				{#if jobId}
+					<div class="flex items-center">
+						<a
+							class="text-primary pb-0.5"
+							target="_blank"
+							href="{base}/api/w/{$workspaceStore}/jobs_u/get_logs/{jobId}"
+							download="windmill_logs_{jobId}.txt"
+							><Download size="14" />
+						</a>
+					</div>
+				{/if}
 				<button on:click={logViewer.openDrawer}><Expand size="12" /></button>
-				<div
-					class="{small ? '' : 'py-2'} pr-2 {small
-						? '!text-2xs'
-						: '!text-xs'} flex gap-2 text-tertiary items-center"
-				>
-					Auto scroll
-					<input class="windmillapp" type="checkbox" bind:checked={scroll} />
-				</div>
+				{#if !noAutoScroll}
+					<div
+						class="{small ? '' : 'py-2'} pr-2 {small
+							? '!text-2xs'
+							: '!text-xs'} flex gap-2 text-tertiary items-center"
+					>
+						Auto scroll
+						<input class="windmillapp" type="checkbox" bind:checked={scroll} />
+					</div>
+				{/if}
 			</div>
 		</div>
 		{#if isLoading}
@@ -1,278 +1,15 @@
 <script lang="ts">
 	import { Drawer, DrawerContent } from './common'
-	import 'chartjs-adapter-date-fns'
-	import { Line } from 'svelte-chartjs'

-	import {
-		Chart as ChartJS,
-		Title,
-		Tooltip,
-		Legend,
-		LineElement,
-		CategoryScale,
-		LinearScale,
-		PointElement,
-		LogarithmicScale,
-		TimeScale,
-		type ChartData,
-		type Point
-	} from 'chart.js'
-	import { WorkerService } from '$lib/gen'
-	import { superadmin } from '$lib/stores'
-	import Skeleton from './common/skeleton/Skeleton.svelte'
-	import DarkModeObserver from './DarkModeObserver.svelte'
-	import Alert from './common/alert/Alert.svelte'
-	export let drawer: Drawer
-
-	let isOpen: boolean = false
-	let loading: boolean = true
-
-	const colorTuples = [
-		['#7EB26D', 'rgba(126, 178, 109, 0.2)'],
-		['#EAB839', 'rgba(234, 184, 57, 0.2)'],
-		['#6ED0E0', 'rgba(110, 208, 224, 0.2)'],
-		['#EF843C', 'rgba(239, 132, 60, 0.2)'],
-		['#E24D42', 'rgba(226, 77, 66, 0.2)'],
-		['#1F78C1', 'rgba(31, 120, 193, 0.2)'],
-		['#BA43A9', 'rgba(186, 67, 169, 0.2)'],
-		['#705DA0', 'rgba(112, 93, 160, 0.2)'],
-		['#508642', 'rgba(80, 134, 66, 0.2)'],
-		['#CCA300', 'rgba(204, 163, 0, 0.2)'],
-		['#447EBC', 'rgba(68, 126, 188, 0.2)'],
-		['#C15C17', 'rgba(193, 92, 23, 0.2)'],
-		['#890F02', 'rgba(137, 15, 2, 0.2)'],
-		['#666666', 'rgba(102, 102, 102, 0.2)'],
-		['#44AA99', 'rgba(68, 170, 153, 0.2)'],
-		['#6D8764', 'rgba(109, 135, 100, 0.2)'],
-		['#555555', 'rgba(85, 85, 85, 0.2)'],
-		['#B3B3B3', 'rgba(179, 179, 179, 0.2)'],
-		['#008C9E', 'rgba(0, 140, 158, 0.2)'],
-		['#6BBA70', 'rgba(107, 186, 112, 0.2)']
-	]
-
-	function getColors(labels: string[]) {
-		const colors = labels.map((_, i) => colorTuples[i % colorTuples.length])
-		return Object.fromEntries(colors.map((c, i) => [labels[i], c]))
+	import QueueMetricsDrawerInner from './QueueMetricsDrawerInner.svelte'
+	let drawer: Drawer
+	export function openDrawer() {
+		drawer?.openDrawer()
 	}
-
-	ChartJS.register(
-		Title,
-		Tooltip,
-		Legend,
-		LineElement,
-		LinearScale,
-		PointElement,
-		CategoryScale,
-		TimeScale,
-		LogarithmicScale
-	)
-
-	let countData: ChartData<'line', Point[], undefined> | undefined = undefined
-	let delayData: ChartData<'line', Point[], undefined> | undefined = undefined
-
-	let minDate = new Date()
-
-	let noMetrics = false
-
-	function fillData(
-		data: {
-			created_at: string
-			value: number
-		}[],
-		zero = 0
-	) {
-		// fill holes with 0
-		const sorted: typeof data = []
-		for (const el of [
-			...data,
-			{
-				created_at: new Date().toISOString(),
-				value: zero
-			}
-		]) {
-			const last =
-				sorted.length > 0 ? new Date(sorted[sorted.length - 1].created_at).getTime() : undefined
-			const currentTs = new Date(el.created_at).getTime()
-			if (last && currentTs - last > 1000 * 60 * 2) {
-				const numElements = Math.floor((currentTs - last) / (1000 * 30))
-				for (let i = 1; i < numElements; i++) {
-					sorted.push({
-						created_at: new Date(last + i * (1000 * 30)).toISOString(),
-						value: zero
-					})
-				}
-			}
-			sorted.push(el)
-		}
-
-		// remove high frequency data points for similar values
-		const light: typeof sorted = []
-		for (const el of sorted) {
-			const last = light.length > 0 ? light[light.length - 1] : undefined
-			if (
-				!last ||
-				Math.abs((el.value - last.value) / last.value) > 0.1 ||
-				new Date(el.created_at).getTime() - new Date(last.created_at).getTime() > 1000 * 60 * 15
-			) {
-				light.push(el)
-			}
-		}
-
-		return light
-	}
-
-	async function loadMetrics() {
-		loading = true
-		let metrics = await WorkerService.getQueueMetrics()
-
-		if (metrics.length == 0) {
-			noMetrics = true
-			loading = false
-			return
-		}
-
-		const labels = metrics
-			.map((m) => m.id.slice(12))
-			.filter((v, i, a) => a.indexOf(v) === i)
-			.sort()
-		const labelColors = getColors(labels)
-
-		countData = {
-			datasets: metrics
-				.filter((m) => m.id.startsWith('queue_count_'))
-				.map((m) => {
-					const [color, bgColor] = labelColors[m.id.slice(12)]
-					return {
-						label: m.id.slice(12),
-						backgroundColor: bgColor,
-						borderColor: color,
-						data: fillData(m.values).map((v) => ({ x: v.created_at as any, y: v.value }))
-					}
-				})
-		}
-
-		delayData = {
-			datasets: metrics
-				.filter((m) => m.id.startsWith('queue_delay_'))
-				.map((m) => {
-					const [color, bgColor] = labelColors[m.id.slice(12)]
-					return {
-						label: m.id.slice(12),
-						borderColor: color,
-						backgroundColor: bgColor,
-						data: fillData(m.values, 1).map((v) => ({
-							x: v.created_at as any,
-							y: v.value
-						}))
-					}
-				})
-		}
-
-		minDate = new Date(Math.min(...countData.datasets.map((d) => new Date(d.data[0].x).getTime())))
-
-		loading = false
-	}
-
-	$: if ($superadmin && isOpen) {
-		loadMetrics()
-	} else {
-		countData = undefined
-		delayData = undefined
-	}
-
-	let darkMode = false
-
-	$: ChartJS.defaults.color = darkMode ? '#ccc' : '#666'
-	$: ChartJS.defaults.borderColor = darkMode ? 'rgba(255, 255, 255, 0.1)' : 'rgba(0, 0, 0, 0.1)'
 </script>

-<DarkModeObserver bind:darkMode />
-
-<Drawer bind:this={drawer} bind:open={isOpen}>
+<Drawer bind:this={drawer}>
 	<DrawerContent title="Queue Metrics" on:close={drawer.closeDrawer}>
-		{#if loading}
-			<Skeleton layout={[[20]]} />
-		{:else if noMetrics}
-			<p class="text-secondary">No jobs delayed by more than 3 seconds in the last 14 days</p>
-		{:else}
-			<div class="flex flex-col gap-4">
-				{#if countData}
-					<Line
-						data={countData}
-						options={{
-							animation: false,
-							plugins: {
-								title: {
-									display: true,
-									text: 'Number of delayed jobs per tag (> 3s)'
-								}
-							},
-							scales: {
-								x: {
-									type: 'time',
-									min: minDate.toISOString(),
-									max: new Date().toISOString()
-								},
-								y: {
-									title: {
-										display: true,
-										text: 'count'
-									}
-								}
-							}
-						}}
-					/>
-				{/if}
-				{#if delayData}
-					<Line
-						data={delayData}
-						options={{
-							animation: false,
-							plugins: {
-								title: {
-									display: true,
-									text: 'Queue delay per tag (> 3s)'
-								},
-								tooltip: {
-									callbacks: {
-										label: function (context) {
-											// @ts-ignore
-											if (context.raw.y === 1) {
-												return context.dataset.label + ': 0'
-											} else {
-												// @ts-ignore
-												return context.dataset.label + ': ' + context.raw.y
-											}
-										}
-									}
-								}
-							},
-							scales: {
-								x: {
-									type: 'time',
-									min: minDate.toISOString(),
-									max: new Date().toISOString()
-								},
-
-								y: {
-									type: 'logarithmic',
-									title: {
-										display: true,
-										text: 'delay (s)'
-									},
-									ticks: {
-										callback: (value, _) => (value === 1 ? '0' : value)
-									}
-								}
-							}
-						}}
-					/>
-				{/if}
-				<Alert title="Info">
-					Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are
-					included in the graph.
-				</Alert>
-			</div>
-		{/if}
+		<QueueMetricsDrawerInner />
 	</DrawerContent>
 </Drawer>
@@ -0,0 +1,265 @@
+<script lang="ts">
+	import 'chartjs-adapter-date-fns'
+	import { Line } from 'svelte-chartjs'
+
+	import {
+		Chart as ChartJS,
+		Title,
+		Tooltip,
+		Legend,
+		LineElement,
+		CategoryScale,
+		LinearScale,
+		PointElement,
+		LogarithmicScale,
+		TimeScale,
+		type ChartData,
+		type Point
+	} from 'chart.js'
+	import { WorkerService } from '$lib/gen'
+	import Skeleton from './common/skeleton/Skeleton.svelte'
+	import DarkModeObserver from './DarkModeObserver.svelte'
+	import Alert from './common/alert/Alert.svelte'
+
+	let loading: boolean = true
+
+	const colorTuples = [
+		['#7EB26D', 'rgba(126, 178, 109, 0.2)'],
+		['#EAB839', 'rgba(234, 184, 57, 0.2)'],
+		['#6ED0E0', 'rgba(110, 208, 224, 0.2)'],
+		['#EF843C', 'rgba(239, 132, 60, 0.2)'],
+		['#E24D42', 'rgba(226, 77, 66, 0.2)'],
+		['#1F78C1', 'rgba(31, 120, 193, 0.2)'],
+		['#BA43A9', 'rgba(186, 67, 169, 0.2)'],
+		['#705DA0', 'rgba(112, 93, 160, 0.2)'],
+		['#508642', 'rgba(80, 134, 66, 0.2)'],
+		['#CCA300', 'rgba(204, 163, 0, 0.2)'],
+		['#447EBC', 'rgba(68, 126, 188, 0.2)'],
+		['#C15C17', 'rgba(193, 92, 23, 0.2)'],
+		['#890F02', 'rgba(137, 15, 2, 0.2)'],
+		['#666666', 'rgba(102, 102, 102, 0.2)'],
+		['#44AA99', 'rgba(68, 170, 153, 0.2)'],
+		['#6D8764', 'rgba(109, 135, 100, 0.2)'],
+		['#555555', 'rgba(85, 85, 85, 0.2)'],
+		['#B3B3B3', 'rgba(179, 179, 179, 0.2)'],
+		['#008C9E', 'rgba(0, 140, 158, 0.2)'],
+		['#6BBA70', 'rgba(107, 186, 112, 0.2)']
+	]
+
+	function getColors(labels: string[]) {
+		const colors = labels.map((_, i) => colorTuples[i % colorTuples.length])
+		return Object.fromEntries(colors.map((c, i) => [labels[i], c]))
+	}
+
+	ChartJS.register(
+		Title,
+		Tooltip,
+		Legend,
+		LineElement,
+		LinearScale,
+		PointElement,
+		CategoryScale,
+		TimeScale,
+		LogarithmicScale
+	)
+
+	let countData: ChartData<'line', Point[], undefined> | undefined = undefined
+	let delayData: ChartData<'line', Point[], undefined> | undefined = undefined
+
+	let minDate = new Date()
+
+	let noMetrics = false
+
+	function fillData(
+		data: {
+			created_at: string
+			value: number
+		}[],
+		zero = 0
+	) {
+		// fill holes with 0
+		const sorted: typeof data = []
+		for (const el of [
+			...data,
+			{
+				created_at: new Date().toISOString(),
+				value: zero
+			}
+		]) {
+			const last =
+				sorted.length > 0 ? new Date(sorted[sorted.length - 1].created_at).getTime() : undefined
+			const currentTs = new Date(el.created_at).getTime()
+			if (last && currentTs - last > 1000 * 60 * 2) {
+				const numElements = Math.floor((currentTs - last) / (1000 * 30))
+				for (let i = 1; i < numElements; i++) {
+					sorted.push({
+						created_at: new Date(last + i * (1000 * 30)).toISOString(),
+						value: zero
+					})
+				}
+			}
+			sorted.push(el)
+		}
+
+		// remove high frequency data points for similar values
+		const light: typeof sorted = []
+		for (const el of sorted) {
+			const last = light.length > 0 ? light[light.length - 1] : undefined
+			if (
+				!last ||
+				Math.abs((el.value - last.value) / last.value) > 0.1 ||
+				new Date(el.created_at).getTime() - new Date(last.created_at).getTime() > 1000 * 60 * 15
+			) {
+				light.push(el)
+			}
+		}
+
+		return light
+	}
+
+	async function loadMetrics() {
+		loading = true
+		let metrics = await WorkerService.getQueueMetrics()
+
+		if (metrics.length == 0) {
+			noMetrics = true
+			loading = false
+			return
+		}
+
+		const labels = metrics
+			.map((m) => m.id.slice(12))
+			.filter((v, i, a) => a.indexOf(v) === i)
+			.sort()
+		const labelColors = getColors(labels)
+
+		countData = {
+			datasets: metrics
+				.filter((m) => m.id.startsWith('queue_count_'))
+				.map((m) => {
+					const [color, bgColor] = labelColors[m.id.slice(12)]
+					return {
+						label: m.id.slice(12),
+						backgroundColor: bgColor,
+						borderColor: color,
+						data: fillData(m.values).map((v) => ({ x: v.created_at as any, y: v.value }))
+					}
+				})
+		}
+
+		delayData = {
+			datasets: metrics
+				.filter((m) => m.id.startsWith('queue_delay_'))
+				.map((m) => {
+					const [color, bgColor] = labelColors[m.id.slice(12)]
+					return {
+						label: m.id.slice(12),
+						borderColor: color,
+						backgroundColor: bgColor,
+						data: fillData(m.values, 1).map((v) => ({
+							x: v.created_at as any,
+							y: v.value
+						}))
+					}
+				})
+		}
+
+		minDate = new Date(Math.min(...countData.datasets.map((d) => new Date(d.data[0].x).getTime())))
+
+		loading = false
+	}
+
+	loadMetrics()
+
+	let darkMode = false
+
+	$: ChartJS.defaults.color = darkMode ? '#ccc' : '#666'
+	$: ChartJS.defaults.borderColor = darkMode ? 'rgba(255, 255, 255, 0.1)' : 'rgba(0, 0, 0, 0.1)'
+</script>
+
+<DarkModeObserver bind:darkMode />
+
+{#if loading}
+	<Skeleton layout={[[20]]} />
+{:else if noMetrics}
+	<p class="text-secondary">No jobs delayed by more than 3 seconds in the last 14 days</p>
+{:else}
+	<div class="flex flex-col gap-4">
+		{#if countData}
+			<Line
+				data={countData}
+				options={{
+					animation: false,
+					plugins: {
+						title: {
+							display: true,
+							text: 'Number of delayed jobs per tag (> 3s)'
+						}
+					},
+					scales: {
+						x: {
+							type: 'time',
+							min: minDate.toISOString(),
+							max: new Date().toISOString()
+						},
+						y: {
+							title: {
+								display: true,
+								text: 'count'
+							}
+						}
+					}
+				}}
+			/>
+		{/if}
+		{#if delayData}
+			<Line
+				data={delayData}
+				options={{
+					animation: false,
+					plugins: {
+						title: {
+							display: true,
+							text: 'Queue delay per tag (> 3s)'
+						},
+						tooltip: {
+							callbacks: {
+								label: function (context) {
+									// @ts-ignore
+									if (context.raw.y === 1) {
+										return context.dataset.label + ': 0'
+									} else {
+										// @ts-ignore
+										return context.dataset.label + ': ' + context.raw.y
+									}
+								}
+							}
+						}
+					},
+					scales: {
+						x: {
+							type: 'time',
+							min: minDate.toISOString(),
+							max: new Date().toISOString()
+						},
+
+						y: {
+							type: 'logarithmic',
+							title: {
+								display: true,
+								text: 'delay (s)'
+							},
+							ticks: {
+								callback: (value, _) => (value === 1 ? '0' : value)
+							}
+						}
+					}
+				}}
+			/>
+		{/if}
+		<Alert title="Info">
+			Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are
+			included in the graph.
+		</Alert>
+	</div>
+{/if}
@@ -0,0 +1,495 @@
+<script lang="ts">
+	import { ServiceLogsService } from '$lib/gen'
+	import { Pane, Splitpanes } from 'svelte-splitpanes'
+
+	import ManuelDatePicker from './runs/ManuelDatePicker.svelte'
+	import CalendarPicker from './common/calendarPicker/CalendarPicker.svelte'
+	import LogViewer from './LogViewer.svelte'
+	import Toggle from './Toggle.svelte'
+	import { sendUserToast } from '$lib/toast'
+	import { onDestroy } from 'svelte'
+	import { Loader2 } from 'lucide-svelte'
+
+	let minTs: undefined | string = undefined
+	let maxTs: undefined | string = undefined
+
+	let minTsManual: undefined | string = undefined
+	let maxTsManual: undefined | string = undefined
+
+	let max_lines: undefined | number = undefined
+
+	// let lastSeen: undefined | string = undefined
+
+	let withError = false
+	let autoRefresh = true
+	let loading = false
+
+	type LogFile = {
+		ts: number
+		file_path: string
+		ok_lines: number
+		err_lines: number
+	}
+
+	type ByHostname = Record<string, LogFile[]>
+	type ByWorkerGroup = Record<string, ByHostname>
+	type ByMode = Record<string, ByWorkerGroup>
+
+	let timeout: NodeJS.Timeout | undefined = undefined
+
+	let allLogs: ByMode | undefined = undefined
+	let manualPicker: ManuelDatePicker | undefined = undefined
+
+	let upTo: undefined | string = undefined
+	let upToIsLatest = true
+
+	$: minTsManual || maxTsManual || onManualChanges()
+
+	function onManualChanges() {
+		getAllLogs(minTsManual ?? maxTs, maxTsManual)
+	}
+
+	function getAllLogs(queryMinTs: string | undefined, queryMaxTs: string | undefined) {
+		timeout && clearTimeout(timeout)
+		loading = true
+		allLogs = allLogs ?? {}
+		ServiceLogsService.listLogFiles({ withError, before: queryMaxTs, after: queryMinTs })
+			.then((res) => {
+				loading = false
+
+				let minTsN: number | undefined = undefined
+				let maxTsN: number | undefined = undefined
+				if (minTsManual) {
+					minTsN = new Date(minTsManual).getTime()
+					Object.values(allLogs ?? {}).forEach((mode) => {
+						Object.values(mode).forEach((wg) => {
+							Object.keys(wg).forEach((key) => {
+								wg[key] = wg[key].filter(
+									(x) => !minTsManual || x.ts >= new Date(minTsManual).getTime()
+								)
+							})
+						})
+					})
+				}
+
+				res.reverse().forEach((log) => {
+					let ts = new Date(log.log_ts + 'Z').getTime()
+					if (minTsN == undefined || ts < minTsN) {
+						minTsN = ts
+					}
+					if (maxTsN == undefined || ts > maxTsN) {
+						maxTsN = ts
+					}
+					if (allLogs == undefined) {
+						allLogs = {}
+					}
+					if (!allLogs[log.mode]) {
+						allLogs[log.mode] = {}
+					}
+					const wg = log.worker_group ?? ''
+					if (!allLogs[log.mode][wg]) {
+						allLogs[log.mode][wg] = {}
+					}
+					const hn = log.hostname ?? ''
+					if (!allLogs[log.mode][wg][hn]) {
+						allLogs[log.mode][wg][hn] = []
+					}
+					allLogs[log.mode][wg][hn].push({
+						ts: ts,
+						file_path: log.file_path,
+						ok_lines: log.ok_lines ?? 1,
+						err_lines: log.err_lines ?? 0
+					})
+					if (
+						log.ok_lines != undefined &&
+						log.err_lines != undefined &&
+						(max_lines == undefined || log.ok_lines + log.err_lines > max_lines)
+					) {
+						max_lines = log.ok_lines + log.err_lines
+					}
+				})
+
+				Object.values(allLogs ?? {}).forEach((mode) => {
+					Object.values(mode).forEach((wg) => {
+						Object.keys(wg).forEach((key) => {
+							wg[key] = wg[key].filter(
+								(x) => !minTsManual || x.ts >= new Date(minTsManual).getTime()
+							)
+						})
+					})
+				})
+
+				loading = false
+				if (minTs == undefined) {
+					minTs = minTsN ? new Date(minTsN).toISOString() : undefined
+				}
+				if (maxTsN) {
+					maxTs = new Date(maxTsN).toISOString()
+				}
+				if (upToIsLatest && selected) {
+					upTo = getLatestUpTo(selected)
+				}
+				if (autoRefresh && !maxTsManual) {
+					timeout = setTimeout(() => {
+						let minMax = manualPicker?.computeMinMax()
+						if (minMax) {
+							maxTsManual = minMax?.maxTs
+							minTsManual = minMax?.minTs
+						}
+						let maxTsPlus1 = maxTs ? new Date(new Date(maxTs).getTime() + 1000) : undefined
+						getAllLogs(maxTsPlus1?.toISOString(), undefined)
+					}, 5000)
+				}
+			})
+			.catch((e) => {
+				sendUserToast('Failed to load service logs: ' + e.body, true)
+				console.error(e)
+				loading = false
+				autoRefresh = false
+			})
+	}
+
+	let selected: [string, string, string] | undefined = undefined
+
+	let logsContent: Record<string, { content?: string; error?: string }> = {}
+	export async function getLogFile(hostname: string, path: string) {
+		if (logsContent[path]) {
+			return
+		}
+		try {
+			const res = await ServiceLogsService.getLogFile({ path: `${hostname}/${path}` })
+			logsContent[path] = { content: res }
+		} catch (e) {
+			logsContent[path] = { error: `${e.message}: ${e.body}` }
+		}
+	}
+
+	getAllLogs(undefined, undefined)
+
+	function getLogs(selected: [string, string, string], upTo: string | undefined) {
+		if (!selected) {
+			return []
+		}
+		let logs = allLogs?.[selected[0]]?.[selected[1]]?.[selected[2]]
+		if (!logs) {
+			return []
+		}
+		if (upTo) {
+			let upToN = new Date(upTo).getTime()
+			let nlogs = logs.filter((x) => x.ts <= upToN)
+			logs = nlogs.slice(nlogs.length - 5, undefined)
+
+			getFiles(
+				selected[2],
+				logs.map((x) => x.file_path)
+			)
+		}
+		return logs
+	}
+
+	async function getFiles(hostname: string, logs: string[]) {
+		await Promise.all(logs.map((x) => getLogFile(hostname, x)))
+		scrollToBottom()
+	}
+
+	function getLatestUpTo(selected: [string, string, string]): any {
+		if (!selected) {
+			return undefined
+		}
+		let logs = allLogs?.[selected[0]]?.[selected[1]]?.[selected[2]]
+		if (!logs) {
+			return undefined
+		}
+		return logs[logs.length - 1]?.ts
+	}
+
+	function scrollToBottom() {
+		const el = document.querySelector('#logviewer')
+		if (el) {
+			el.scrollTop = el.scrollHeight
+		}
+	}
+
+	onDestroy(() => {
+		timeout && clearTimeout(timeout)
+	})
+</script>
+
+<div class="w-full h-[70vh]" on:scroll|preventDefault>
+	<Splitpanes>
+		<Pane size={40} minSize={20}>
+			<div class="p-1">
+				<div
+					class="flex flex-col lg:flex-row gap-y-1 justify-between w-full relative pb-4 gap-x-0.5"
+					id="service-logs-date-pickers"
+				>
+					<div class="flex relative">
+						<input
+							type="text"
+							value={minTsManual
+								? new Date(minTsManual).toLocaleTimeString([], {
+										day: '2-digit',
+										month: '2-digit',
+										hour: '2-digit',
+										minute: '2-digit'
+								  })
+								: 'min datetime'}
+							disabled
+						/>
+						<CalendarPicker label="min datetime" date={minTs} />
+					</div>
+					<ManuelDatePicker
+						bind:minTs={minTsManual}
+						bind:maxTs={maxTsManual}
+						bind:this={manualPicker}
+						{loading}
+						on:loadJobs={() => {
+							minTs = undefined
+							maxTs = undefined
+							allLogs = undefined
+							getAllLogs(minTsManual, maxTsManual)
+						}}
+						serviceLogsChoices
+						loadText="Last 1000 logfiles"
+					/>
+					<div class="flex relative">
+						<input
+							type="text"
+							value={maxTsManual
+								? new Date(maxTsManual).toLocaleTimeString([], {
+										day: '2-digit',
+										month: '2-digit',
+										hour: '2-digit',
+										minute: '2-digit'
+								  })
+								: 'max datetime'}
+							disabled
+						/>
+						<CalendarPicker label="max datetime" date={maxTs} />
+					</div>
+				</div>
+				<div class="flex w-full flex-row-reverse pb-4 -mt-2 gap-2"
+					><Toggle
+						size="xs"
+						bind:checked={withError}
+						options={{ right: 'errors > 0' }}
+						on:change={() => {
+							allLogs = undefined
+							getAllLogs(minTs, maxTs)
+						}}
+					/>
+					<Toggle
+						size="xs"
+						bind:checked={autoRefresh}
+						on:change={(e) => {
+							if (e.detail) {
+								getAllLogs(maxTs, undefined)
+							} else {
+								timeout && clearTimeout(timeout)
+							}
+						}}
+						options={{ right: 'auto-refresh' }}
+					/></div
+				>
+				{#if allLogs == undefined}
+					<div class="text-center pb-2"><Loader2 class="animate-spin" /></div>
+				{:else if Object.keys(allLogs).length == 0}
+					<div class="flex justify-center items-center h-full">No logs</div>
+				{:else if minTs && maxTs}
+					{@const minTsN = new Date(minTs).getTime()}
+					{@const maxTsN = new Date(maxTs).getTime()}
+					{@const diff = maxTsN - minTsN}
+					<div class="flex w-full text-2xs text-tertiary pb-6">
+						<div style="width: 60px;" />
+
+						<div class="flex justify-between w-full"
+							><div
+								>{new Date(minTs).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+								})}</div
+							><div
+								>{new Date(maxTs).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+								})}</div
+							></div
+						>
+					</div>
+					{#each Object.entries(allLogs) as [mode, o1]}
+						<div class="w-full pb-8">
+							<h2 class="pb-2 text-2xl">{mode}s</h2>
+							{#each Object.entries(o1) as [wg, o2]}
+								<div class="w-full px-1">
+									{#if wg && wg != ''}
+										<h4 class="pt-4">{wg}</h4>
+									{/if}
+									<div class="divide-y flex flex-col">
+										{#each Object.entries(o2) as [hn, files]}
+											<!-- svelte-ignore a11y-click-events-have-key-events -->
+											<!-- svelte-ignore a11y-no-static-element-interactions -->
+											<div
+												class="w-full flex items-baseline rounded px-1 hover:bg-surface-hover cursor-pointer {selected &&
+												selected[0] == mode &&
+												selected[1] == wg &&
+												selected[2] == hn
+													? 'bg-surface-secondary'
+													: ''}"
+												on:click={() => {
+													selected = [mode, wg, hn]
+													upToIsLatest = true
+													upTo = getLatestUpTo(selected)
+													scrollToBottom()
+												}}
+											>
+												<div class="text-sm pt-2 pl-0.5" style="width: 90px;">{hn}</div>
+												<div class="relative grow h-8 mr-2">
+													{#each files as file}
+														{@const okHeight = 100.0 * ((file.ok_lines * 1.0) / (max_lines ?? 1))}
+														{@const errHeight = 100.0 * ((file.err_lines * 1.0) / (max_lines ?? 1))}
+														<div
+															class=" w-2 bg-red-400 absolute"
+															style="left: {((file.ts - minTsN) / diff) *
+																100}%; height: {errHeight}%; bottom: {okHeight}%;"
+														/>
+														<div
+															class="w-2 bg-surface-secondary-inverse absolute bottom-0"
+															style="left: {((file.ts - minTsN) / diff) *
+																100}%; height: {okHeight}%"
+														/>
+													{/each}
+												</div>
+											</div>
+										{/each}
+									</div>
+								</div>
+							{/each}
+						</div>
+					{/each}
+				{/if}
+			</div>
+		</Pane>
+		<Pane size={60} minSize={20}
+			><div class="relative h-full flex flex-col gap-1"
+				><div class="w-full bg-surface-primary-inverse text-tertiary text-xs text-center"
+					>1 min delay: logs are compacted before being available</div
+				>
+				{#if selected}
+					<div class="grow overflow-auto" id="logviewer">
+						{#each getLogs(selected, upTo) as file}
+							<div
+								style="min-height: {logsContent[file.file_path]
+									? 10
+									: (file.ok_lines + file.err_lines) / 20}px;"
+							>
+								<div class="bg-surface-primary-inverse text-sm font-semibold px-1"
+									>{new Date(file.ts).toLocaleTimeString([], {
+										day: '2-digit',
+										month: '2-digit',
+										hour: '2-digit',
+										minute: '2-digit'
+									})}</div
+								>
+								{#if logsContent[file.file_path] == undefined}
+									<div
+										class="animate-skeleton dark:bg-frost-900/50 [animation-delay:1000ms] h-full w-full"
+									/>
+								{:else if logsContent[file.file_path]}
+									{#if logsContent[file.file_path].error}
+										{#if logsContent[file.file_path].error?.startsWith('Not Found')}
+											<div class="text-xs pb-4 pt-2 text-secondary"
+												>Log file is missing. Log files require a shared log volume to be mounted
+												across servers and workers or to use the EE S3/object storage integration
+												for logs. To avoid mounting a shared volume, set the EE object store logs in
+												the instance settings</div
+											>
+										{:else}
+											<div class="text-xs text-red-400 pb-4"
+												>{logsContent[file.file_path].error}</div
+											>
+										{/if}
+									{:else if logsContent[file.file_path].content}
+										<!-- svelte-ignore a11y-click-events-have-key-events -->
+										<!-- svelte-ignore a11y-no-static-element-interactions -->
+										<div on:click|preventDefault
+											><LogViewer
+												noAutoScroll
+												noMaxH
+												isLoading={false}
+												tag={undefined}
+												content={logsContent[file.file_path].content}
+											/></div
+										>
+									{:else}
+										<div>No logs</div>
+									{/if}
+								{/if}
+							</div>
+						{/each}
+					</div>
+					<div class="flex w-full items-center gap-4">
+						<div class="text-tertiary px-1 text-2xs">Last 5 log files up to:</div>
+						<div class="flex grow text-xs justify-center px-2 items-center gap-2">
+							{#if upTo}
+								<button
+									on:click={() => {
+										if (upTo) {
+											upToIsLatest = false
+											upTo = new Date(new Date(upTo).getTime() - 5 * 60 * 1000).toISOString()
+										}
+									}}>{'<'} 5m</button
+								>
+							{:else}
+								<div />
+							{/if}
+
+							<div class="flex gap-1 relative items-center"
+								><div class="flex gap-1 relative">
+									<input
+										type="text"
+										value={upTo
+											? new Date(upTo).toLocaleTimeString([], {
+													day: '2-digit',
+													month: '2-digit',
+													hour: '2-digit',
+													minute: '2-digit'
+											  })
+											: ''}
+										disabled
+									/><CalendarPicker bind:date={upTo} label="Logs up to" /></div
+								></div
+							>
+							{#if upTo}
+								<button
+									on:click={() => {
+										if (upTo) {
+											upToIsLatest = false
+											upTo = new Date(new Date(upTo).getTime() + 5 * 60 * 1000).toISOString()
+										}
+									}}>5m {'>'}</button
+								>
+							{:else}
+								<div />
+							{/if}
+						</div>
+						<div>
+							<button
+								class="text-xs"
+								on:click={() => {
+									upTo = new Date().toISOString()
+									upToIsLatest = true
+								}}>now</button
+							>
+						</div>
+					</div>
+				{:else}
+					<div class="flex justify-center items-center pt-8">Select a host to see its logs</div>
+				{/if}</div
+			></Pane
+		>
+	</Splitpanes>
+</div>
@@ -870,7 +870,7 @@
 	<div class="text-xs"
 		>{pluralize(activeWorkers, 'worker')}
 		{#if vcpus_memory?.vcpus}
-			- {vcpus_memory?.vcpus} vCPUs{/if}{#if vcpus_memory?.memory}
+			- {vcpus_memory?.vcpus / 1000} vCPUs{/if}{#if vcpus_memory?.memory}
 			- {vcpus_memory?.memory} MB{/if}</div
 	>
 	<div class="flex gap-2 items-center justify-end flex-row my-2">
@@ -150,7 +150,7 @@
 					outputs?.inputs.set(inputs, true)
 				},
 				onRemove: (id, rowIndex) => {
-					if (inputs?.[id] == undefined) {
+				if (inputs?.[id] == undefined) {
 						return
 					}
 					delete inputs[id][rowIndex]
@@ -39,7 +39,7 @@

 	type MenuItem = {
 		label: string
-		onClick?: () => void
+		onClick?: (e?: Event) => void
 		href?: string
 		icon?: any
 	}
@@ -9,6 +9,7 @@
 	export let label: string
 	export let useDropdown: boolean = false
 	export let clearable: boolean = false
+	export let target: string | HTMLElement | undefined = undefined

 	const dispatch = createEventDispatcher()
 	let input: HTMLInputElement
@@ -16,7 +17,7 @@
 	export let placement: Placement = 'top-end'
 </script>

-<Popup floatingConfig={{ placement: placement, strategy: 'absolute' }}>
+<Popup floatingConfig={{ placement: placement, strategy: 'absolute' }} {target}>
 	<svelte:fragment slot="button">
 		<button
 			title="Open calendar picker"
@@ -31,9 +32,12 @@
 	</svelte:fragment>

 	<!-- svelte-ignore a11y-label-has-associated-control -->
+	<!-- svelte-ignore a11y-click-events-have-key-events -->
 	<label class="block text-primary">
 		<div class="pb-1 text-sm text-secondary">{label}</div>
-		<div class="flex w-full">
+		<!-- svelte-ignore a11y-click-events-have-key-events -->
+		<!-- svelte-ignore a11y-no-static-element-interactions -->
+		<div on:click|stopPropagation class="flex w-full">
 			<DateTimeInput
 				{clearable}
 				{useDropdown}
@@ -2,10 +2,11 @@
 	import Portal from '$lib/components/Portal.svelte'

 	export let condition = false
+	export let target: string | HTMLElement | undefined = undefined
 </script>

 {#if condition}
-	<Portal><slot /></Portal>
+	<Portal {target}><slot /></Portal>
 {:else}
 	<slot />
 {/if}
@@ -14,6 +14,7 @@

 	export let blockOpen = false
 	export let shouldUsePortal: boolean = true
+	export let target: string | HTMLElement | undefined = undefined
 </script>

 <Popover on:close class="leading-none">
@@ -22,7 +23,7 @@
 			<slot name="button" />
 		</div>
 	</PopoverButton>
-	<ConditionalPortal condition={shouldUsePortal}>
+	<ConditionalPortal condition={shouldUsePortal} {target}>
 		<div use:floatingContent class="z5000">
 			<Transition
 				show={blockOpen || undefined}
@@ -7,6 +7,8 @@
 	export let maxTs: string | undefined
 	export let loading: boolean = false
 	export let selectedManualDate = 0
+	export let loadText: string | undefined = undefined
+	export let serviceLogsChoices: boolean = false

 	export function computeMinMax(): { minTs: string; maxTs: string | undefined } | undefined {
 		return manualDates[selectedManualDate].computeMinMax()
@@ -27,19 +29,23 @@
 		computeMinMax: () => { minTs: string; maxTs: string | undefined } | undefined
 	}[] = [
 		{
-			label: 'Last 1000 runs',
+			label: loadText ?? 'Last 1000 runs',
 			computeMinMax: () => {
 				return undefined
 			}
 		},
-		{
-			label: 'Within 30 seconds',
-			computeMinMax: () => computeMinMaxInc(30 * 1000)
-		},
-		{
-			label: 'Within last minute',
-			computeMinMax: () => computeMinMaxInc(1 * 60 * 1000)
-		},
+		...(!serviceLogsChoices
+			? [
+					{
+						label: 'Within 30 seconds',
+						computeMinMax: () => computeMinMaxInc(30 * 1000)
+					},
+					{
+						label: 'Within last minute',
+						computeMinMax: () => computeMinMaxInc(60 * 1000)
+					}
+			  ]
+			: []),
 		{
 			label: 'Within last 5 minutes',
 			computeMinMax: () => computeMinMaxInc(5 * 60 * 1000)
@@ -75,12 +81,13 @@
 			minTs = ts.minTs
 			maxTs = ts.maxTs
 		}
-		dispatch('loadJobs')
+		dispatch('loadJobs', { minTs, maxTs })
 	}}
 	dropdownItems={[
 		...manualDates.map((d, i) => ({
 			label: d.label,
-			onClick: () => {
+			onClick: (e) => {
+				e.preventDefault()
 				selectedManualDate = i
 				const ts = d.computeMinMax()
 				if (ts) {
@@ -33,12 +33,13 @@
 	import ContentSearchInner from '../ContentSearchInner.svelte'
 	import { goto } from '$app/navigation'
 	import QuickMenuItem from '../search/QuickMenuItem.svelte'
-	import { enterpriseLicense, workspaceStore } from '$lib/stores'
+	import { enterpriseLicense, superadmin, workspaceStore } from '$lib/stores'
 	import uFuzzy from '@leeoniya/ufuzzy'
 	import BarsStaggered from '../icons/BarsStaggered.svelte'
 	import { scroll_into_view_if_needed_polyfill } from '../multiselect/utils'
 	import { Alert } from '../common'
 	import Popover from '../Popover.svelte'
+	import ServiceLogsInner from '../ServiceLogsInner.svelte'

 	let open: boolean = false

@@ -273,23 +274,25 @@
 					open = false
 				}
 			}
-			if (event.key === 'ArrowDown') {
-				event.preventDefault()
-				let idx = itemMap[tab].indexOf(selectedItem)
-				if (idx != -1) {
-					idx = (idx + 1) % itemMap[tab].length
-					selectedItem = selectItem(idx)
-					let el = document.getElementById(selectedItem.search_id)
-					if (el) scroll_into_view_if_needed_polyfill(el, false)
-				}
-			} else if (event.key === 'ArrowUp') {
-				event.preventDefault()
-				let idx = itemMap[tab].indexOf(selectedItem)
-				if (idx != -1) {
-					idx = (idx - 1 + itemMap[tab].length) % itemMap[tab].length
-					selectedItem = selectItem(idx)
-					let el = document.getElementById(selectedItem.search_id)
-					if (el) scroll_into_view_if_needed_polyfill(el, false)
+			if (tab != 'logs') {
+				if (event.key === 'ArrowDown') {
+					event.preventDefault()
+					let idx = itemMap[tab].indexOf(selectedItem)
+					if (idx != -1) {
+						idx = (idx + 1) % itemMap[tab].length
+						selectedItem = selectItem(idx)
+						let el = document.getElementById(selectedItem.search_id)
+						if (el) scroll_into_view_if_needed_polyfill(el, false)
+					}
+				} else if (event.key === 'ArrowUp') {
+					event.preventDefault()
+					let idx = itemMap[tab].indexOf(selectedItem)
+					if (idx != -1) {
+						idx = (idx - 1 + itemMap[tab].length) % itemMap[tab].length
+						selectedItem = selectItem(idx)
+						let el = document.getElementById(selectedItem.search_id)
+						if (el) scroll_into_view_if_needed_polyfill(el, false)
+					}
 				}
 			}
 		}
@@ -477,7 +480,7 @@
 	}

 	function maxModalWidth(tab: SearchMode) {
-		if (tab === 'runs') {
+		if (tab === 'runs' || tab === 'logs') {
 			return 'max-w-7xl'
 		} else {
 			return 'max-w-4xl'
@@ -485,7 +488,7 @@
 	}

 	function maxModalHeight(tab: SearchMode) {
-		if (tab === 'runs') {
+		if (tab === 'runs' || tab === 'logs') {
 			return ''
 		} else if (tab === 'content') {
 			return 'max-h-[70vh]'
@@ -603,9 +606,17 @@
 						/>
 					{:else if tab === 'logs'}
 						<div class="p-2">
-							<Alert title="Service log search is coming soon" type="info">
-								Full text search on windmill's service logs is coming soon
-							</Alert>
+							{#if !$superadmin}
+								<Alert title="Service logs are only available to superadmins" type="warning">
+									Service logs are only available to superadmins
+								</Alert>
+							{:else if searchTerm.length == 1}
+								<ServiceLogsInner />
+							{:else}
+								<Alert title="Not yet supported" type="info">
+									Full-text search across Windmill logs is not yet supported
+								</Alert>
+							{/if}
 						</div>
 					{:else if tab === 'runs'}
 						<div class="flex h-full p-2 divide-x">
@@ -186,7 +186,7 @@
 		await loadWorkerGroups()
 	}

-	let queueMetricsDrawer: Drawer
+	let queueMetricsDrawer: QueueMetricsDrawer
 	let selectedTab: string = 'default'

 	$: groupedWorkers && selectedTab == 'default' && updateSelectedTabIfDefaultDoesNotExist()
@@ -240,7 +240,7 @@
 </script>

 {#if $superadmin}
-	<QueueMetricsDrawer bind:drawer={queueMetricsDrawer} />
+	<QueueMetricsDrawer bind:this={queueMetricsDrawer} />
 {/if}

 <Drawer bind:this={importConfigDrawer} size="800px">