fix: should exit when backend init failed (#4138)

.github/workflows/playwright.yml

@@ -120,6 +120,7 @@ jobs:
           cd tests/ui-testing && npm ci && npx playwright test ./playwright-tests/${{ matrix.testfilename }}
 
       - name: Check OpenObserve logs
+        if: always()
         run: cat o2.log
 
       - uses: actions/upload-artifact@v4

src/main.rs

@@ -163,55 +163,76 @@ async fn main() -> Result<(), anyhow::Error> {
     let (job_stopped_tx, job_stopped_rx) = oneshot::channel();
     let job_rt_handle = std::thread::spawn(move || {
         let cfg = get_config();
-        let rt = tokio::runtime::Builder::new_multi_thread()
+        let Ok(rt) = tokio::runtime::Builder::new_multi_thread()
             .worker_threads(cfg.limit.job_runtime_worker_num)
             .enable_all()
             .thread_name("job_runtime")
             .max_blocking_threads(cfg.limit.job_runtime_blocking_worker_num)
             .build()
-            .expect("job runtime init failed");
+        else {
+            job_init_tx.send(false).ok();
+            panic!("job runtime init failed")
+        };
         let _guard = rt.enter();
         rt.block_on(async move {
             // it must be initialized before the server starts
-            cluster::register_and_keepalive()
-                .await
-                .expect("cluster init failed");
+            if let Err(e) = cluster::register_and_keepalive().await {
+                job_init_tx.send(false).ok();
+                panic!("cluster init failed: {}", e);
+            }
             // init config
-            config::init().await.expect("config init failed");
+            if let Err(e) = config::init().await {
+                job_init_tx.send(false).ok();
+                panic!("config init failed: {}", e);
+            }
             // init infra
-            infra::init().await.expect("infra init failed");
-            common_infra::init()
-                .await
-                .expect("common infra init failed");
+            if let Err(e) = infra::init().await {
+                job_init_tx.send(false).ok();
+                panic!("infra init failed: {}", e);
+            }
+            if let Err(e) = common_infra::init().await {
+                job_init_tx.send(false).ok();
+                panic!("common infra init failed: {}", e);
+            }
 
             // init enterprise
             #[cfg(feature = "enterprise")]
-            o2_enterprise::enterprise::init()
-                .await
-                .expect("enerprise init failed");
+            if let Err(e) = o2_enterprise::enterprise::init().await {
+                job_init_tx.send(false).ok();
+                panic!("enerprise init failed: {}", e);
+            }
 
             // check version upgrade
             let old_version = db::version::get().await.unwrap_or("v0.0.0".to_string());
-            migration::check_upgrade(&old_version, VERSION)
-                .await
-                .expect("check upgrade failed");
+            if let Err(e) = migration::check_upgrade(&old_version, VERSION).await {
+                job_init_tx.send(false).ok();
+                panic!("check upgrade failed: {}", e);
+            }
             // migrate dashboards
-            migration::dashboards::run()
-                .await
-                .expect("migrate dashboards failed");
+            if let Err(e) = migration::dashboards::run().await {
+                job_init_tx.send(false).ok();
+                panic!("migrate dashboards failed: {}", e);
+            }
 
             // ingester init
-            ingester::init().await.expect("ingester init failed");
+            if let Err(e) = ingester::init().await {
+                job_init_tx.send(false).ok();
+                panic!("ingester init failed: {}", e);
+            }
 
             // init job
-            job::init().await.expect("job init failed");
+            if let Err(e) = job::init().await {
+                job_init_tx.send(false).ok();
+                panic!("job init failed: {}", e);
+            }
 
             // init meter provider
-            let meter_provider = job::metrics::init_meter_provider()
-                .await
-                .expect("meter provider init failed");
+            let Ok(meter_provider) = job::metrics::init_meter_provider().await else {
+                job_init_tx.send(false).ok();
+                panic!("meter provider init failed");
+            };
 
-            job_init_tx.send(()).ok();
+            job_init_tx.send(true).ok();
             job_shutdown_rx.await.ok();
             job_stopped_tx.send(()).ok();
 
@@ -231,7 +252,15 @@ async fn main() -> Result<(), anyhow::Error> {
     });
 
     // wait for job init
-    job_init_rx.await.ok();
+    match job_init_rx.await {
+        Ok(true) => log::info!("backend job init success"),
+        Ok(false) => {
+            return Err(anyhow::anyhow!("backend job init failed, exiting"));
+        }
+        Err(e) => {
+            return Err(anyhow::anyhow!("backend job init failed: {}", e));
+        }
+    }
 
     // init gRPC server
     let (grpc_init_tx, grpc_init_rx) = oneshot::channel();

tests/ui-testing/playwright-tests/dashboard.spec.js

@@ -269,7 +269,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -308,9 +308,8 @@ test.describe("dashboard UI testcases", () => {
       .first()
       .click();
 
-    await page.getByRole("button", { name: "7", exact: true }).click();
-
-    await page.getByRole("button", { name: "8", exact: true }).click();
+    await page.getByRole("button", { name: "7" }).last().click();
+    await page.getByRole("button", { name: "8" }).last().click();
 
     await page.locator('[data-test="dashboard-apply"]').click();
   });
@@ -405,7 +404,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -479,8 +478,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
-
+    await page.getByRole("option", { name: "e2e_automate" }).click();
     await page
       .locator(
         '[data-test="field-list-item-logs-e2e_automate-kubernetes_labels_app_kubernetes_io_component"] [data-test="dashboard-add-b-data"]'
@@ -681,7 +679,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     //  await page.locator('[data-test="field-list-item-logs-e2e_automate-_timestamp"] [data-test="dashboard-add-x-data"]').click();
     await page
@@ -750,7 +748,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -918,7 +916,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1054,7 +1052,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     // Add panel fields
     await page
@@ -1076,9 +1074,9 @@ test.describe("dashboard UI testcases", () => {
       .first()
       .click();
 
-    await page.getByRole("button", { name: "7", exact: true }).click();
+    await page.getByRole("button", { name: "7" }).last().click();
     await page.waitForTimeout(100);
-    await page.getByRole("button", { name: "10", exact: true }).click();
+    await page.getByRole("button", { name: "10" }).last().click();
     await page.locator('[data-test="datetime-timezone-select"]').click();
     await page.locator('[data-test="dashboard-apply"]').click();
 
@@ -1271,7 +1269,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     // Add fields to the chart
     await page
@@ -1553,7 +1551,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1605,7 +1603,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1625,8 +1623,8 @@ test.describe("dashboard UI testcases", () => {
       .first()
       .click();
 
-    await page.getByRole("button", { name: "9", exact: true }).click();
-    await page.getByRole("button", { name: "16", exact: true }).click();
+    await page.getByRole("button", { name: "9" }).last().click();
+    await page.getByRole("button", { name: "16" }).last().click();
     await page.locator('[data-test="chart-renderer"] div').click();
     await page.locator('[data-test="dashboard-apply"]').click();
 
@@ -1662,7 +1660,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1696,8 +1694,8 @@ test.describe("dashboard UI testcases", () => {
       .first()
       .click();
 
-    await page.getByRole("button", { name: "8", exact: true }).click();
-    await page.getByRole("button", { name: "16", exact: true }).click();
+    await page.getByRole("button", { name: "8" }).last().click();
+    await page.getByRole("button", { name: "16" }).last().click();
 
     await page.locator("#date-time-menu").getByText("arrow_drop_down").click();
     await page.locator('[data-test="datetime-timezone-select"]').click();
@@ -1741,7 +1739,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1784,7 +1782,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(
@@ -1864,7 +1862,7 @@ test.describe("dashboard UI testcases", () => {
       .filter({ hasText: "Streamarrow_drop_down" })
       .locator("i")
       .click();
-    await page.getByText("e2e_automate").click();
+    await page.getByRole("option", { name: "e2e_automate" }).click();
 
     await page
       .locator(

tests/ui-testing/playwright-tests/logspage.spec.js

@@ -7,8 +7,9 @@ test.describe.configure({ mode: "parallel" });
 
 async function login(page) {
   await page.goto(process.env["ZO_BASE_URL"]);
+  // await page.getByText('Login as internal user').click();
+  console.log("ZO_BASE_URL", process.env["ZO_BASE_URL"]);
   await page.waitForTimeout(1000);
-// await page.getByText('Login as internal user').click();
   await page
     .locator('[data-cy="login-user-id"]')
     .fill(process.env["ZO_ROOT_USER_EMAIL"]);
@@ -21,6 +22,34 @@ async function login(page) {
   await page.goto(process.env["ZO_BASE_URL"]);
 }
 
+async function ingestion(page) {
+  const orgId = process.env["ORGNAME"];
+  const streamName = "e2e_automate";
+  const basicAuthCredentials = Buffer.from(
+    `${process.env["ZO_ROOT_USER_EMAIL"]}:${process.env["ZO_ROOT_USER_PASSWORD"]}`
+  ).toString('base64');
+
+  const headers = {
+    "Authorization": `Basic ${basicAuthCredentials}`,
+    "Content-Type": "application/json",
+  };
+  const response = await page.evaluate(async ({ url, headers, orgId, streamName, logsdata }) => {
+    const fetchResponse = await fetch(`${url}/api/${orgId}/${streamName}/_json`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(logsdata)
+    });
+    return await fetchResponse.json();
+  }, {
+    url: process.env.INGESTION_URL,
+    headers: headers,
+    orgId: orgId,
+    streamName: streamName,
+    logsdata: logsdata
+  });
+  console.log(response);
+}
+
 const selectStreamAndStreamTypeForLogs = async (page, stream) => {
   await page.waitForTimeout(4000);
   await page
@@ -60,48 +89,10 @@ test.describe("Logs UI testcases", () => {
   // });
   test.beforeEach(async ({ page }) => {
     await login(page);
-    await page.waitForTimeout(5000);
+    await page.waitForTimeout(1000)
+    await ingestion(page);
+    await page.waitForTimeout(2000)
 
-    // ("ingests logs via API", () => {
-    const orgId = process.env["ORGNAME"];
-    const streamName = "e2e_automate";
-    const basicAuthCredentials = Buffer.from(
-      `${process.env["ZO_ROOT_USER_EMAIL"]}:${process.env["ZO_ROOT_USER_PASSWORD"]}`
-    ).toString("base64");
-
-    const headers = {
-      Authorization: `Basic ${basicAuthCredentials}`,
-      "Content-Type": "application/json",
-    };
-
-    // const logsdata = {}; // Fill this with your actual data
-
-    // Making a POST request using fetch API
-    const response = await page.evaluate(
-      async ({ url, headers, orgId, streamName, logsdata }) => {
-        const fetchResponse = await fetch(
-          `${url}/api/${orgId}/${streamName}/_json`,
-          {
-            method: "POST",
-            headers: headers,
-            body: JSON.stringify(logsdata),
-          }
-        );
-        return await fetchResponse.json();
-      },
-      {
-        url: process.env.INGESTION_URL,
-        headers: headers,
-        orgId: orgId,
-        streamName: streamName,
-        logsdata: logsdata,
-      }
-    );
-
-    console.log(response);
-    //  });
-    // const allorgs = page.waitForResponse("**/api/default/organizations**");
-    // const functions = page.waitForResponse("**/api/default/functions**");
     await page.goto(
       `${logData.logsUrl}?org_identifier=${process.env["ORGNAME"]}`
     );
@@ -165,9 +156,9 @@ test.describe("Logs UI testcases", () => {
     await page
       .locator('[data-test="logs-search-bar-show-query-toggle-btn"]')
       .click({ force: true });
-       await page.locator('[data-test="logs-search-field-list-collapse-btn"]').click();
-       await page.waitForTimeout(1000)
-       await page.locator('[data-test="logs-search-field-list-collapse-btn"]').click();
+    await page.locator('[data-test="logs-search-field-list-collapse-btn"]').click();
+    await page.waitForTimeout(1000)
+    await page.locator('[data-test="logs-search-field-list-collapse-btn"]').click();
     // await page
     //   .locator(".bg-primary > .q-btn__content > .q-icon")
     //   .click({ force: true });
@@ -508,7 +499,7 @@ test.describe("Logs UI testcases", () => {
     // Assert that ".a=2" is visible
     const logsPage = await page.locator('.q-page-container');
     await expect(logsPage).toContainText(".a=2");
-   
+
   });
 
   test('should display bar chart when histogram toggle is on', async ({ page }) => {
@@ -538,7 +529,7 @@ test.describe("Logs UI testcases", () => {
     const element = await page.locator('[data-test="log-table-column-0-source"]');
     const isVisible = await element.isVisible();
     expect(isVisible).toBeTruthy();
-    
+
   });
 
 
@@ -554,9 +545,9 @@ test.describe("Logs UI testcases", () => {
     const element = await page.locator('[data-test="log-table-column-0-source"]');
     const isVisible = await element.isVisible();
     expect(isVisible).toBeTruthy();
-    
+
   });
-  
+
   test('should display search around in SQL mode', async ({ page }) => {
     await page.waitForTimeout(1000);
     await page.getByLabel('SQL Mode').locator('div').nth(2).click();
@@ -566,13 +557,13 @@ test.describe("Logs UI testcases", () => {
     const element = await page.locator('[data-test="log-table-column-0-source"]');
     const isVisible = await element.isVisible();
     expect(isVisible).toBeTruthy();
-    
+
   });
 
   test("should display results for search around with limit query", async ({ page }) => {
     await page.waitForTimeout(2000);
     await page.locator('[data-cy="date-time-button"]').click({ force: true });
-    await page.locator('[data-test="date-time-relative-15-m-btn"] > .q-btn__content > .block').click({force: true });
+    await page.locator('[data-test="date-time-relative-15-m-btn"] > .q-btn__content > .block').click({ force: true });
     await page.click('[data-test="logs-search-bar-query-editor"]')
     await page.keyboard.type("match_all('code') limit 5");
     await page.waitForTimeout(2000);

tests/ui-testing/playwright-tests/logsquickmode.spec.js

@@ -51,10 +51,9 @@ async function ingestion(page) {
 }
 
 const selectStreamAndStreamTypeForLogs = async (page, stream) => {
-  await page.waitForTimeout(
-    4000); await page.locator(
-      '[data-test="log-search-index-list-select-stream"]').click({ force: true }); await page.locator(
-        "div.q-item").getByText(`${stream}`).first().click({ force: true });
+  await page.waitForTimeout(4000);
+  await page.locator('[data-test="log-search-index-list-select-stream"]').click({ force: true });
+  await page.locator("div.q-item").getByText(`${stream}`).first().click({ force: true });
 };
 
 test.describe("Logs Quickmode testcases", () => {

tests/ui-testing/playwright-tests/sanity.spec.js

@@ -10,19 +10,47 @@ const dashboardName = `AutomatedDashboard${Date.now()}`;
 
 async function login(page) {
   await page.goto(process.env["ZO_BASE_URL"]);
-  // await page.getByText("Login as internal user").click();
+  // await page.getByText('Login as internal user').click();
+  console.log("ZO_BASE_URL", process.env["ZO_BASE_URL"]);
   await page.waitForTimeout(1000);
   await page
     .locator('[data-cy="login-user-id"]')
     .fill(process.env["ZO_ROOT_USER_EMAIL"]);
   //Enter Password
-  await page.locator("label").filter({ hasText: "Password *" }).click();
   await page
     .locator('[data-cy="login-password"]')
     .fill(process.env["ZO_ROOT_USER_PASSWORD"]);
   await page.locator('[data-cy="login-sign-in"]').click();
-  //     await page.waitForTimeout(4000);
-  // await page.goto(process.env["ZO_BASE_URL"]);
+  await page.waitForTimeout(4000);
+  await page.goto(process.env["ZO_BASE_URL"]);
+}
+
+async function ingestion(page) {
+  const orgId = process.env["ORGNAME"];
+  const streamName = "e2e_automate";
+  const basicAuthCredentials = Buffer.from(
+    `${process.env["ZO_ROOT_USER_EMAIL"]}:${process.env["ZO_ROOT_USER_PASSWORD"]}`
+  ).toString('base64');
+
+  const headers = {
+    "Authorization": `Basic ${basicAuthCredentials}`,
+    "Content-Type": "application/json",
+  };
+  const response = await page.evaluate(async ({ url, headers, orgId, streamName, logsdata }) => {
+    const fetchResponse = await fetch(`${url}/api/${orgId}/${streamName}/_json`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(logsdata)
+    });
+    return await fetchResponse.json();
+  }, {
+    url: process.env.INGESTION_URL,
+    headers: headers,
+    orgId: orgId,
+    streamName: streamName,
+    logsdata: logsdata
+  });
+  console.log(response);
 }
 
 const selectStreamAndStreamTypeForLogs = async (page, stream) => {
@@ -55,54 +83,12 @@ test.describe("Sanity testcases", () => {
     await expect.poll(async () => (await search).status()).toBe(200);
     // await search.hits.FIXME_should("be.an", "array");
   }
-  // tebefore(async function () {
-  //   // logData("log");
-  //   // const data = page;
-  //   // logData = data;
 
-  //   console.log("--logData--", logData);
-  // });
   test.beforeEach(async ({ page }) => {
     await login(page);
-    await page.waitForTimeout(5000);
-
-    // ("ingests logs via API", () => {
-    const orgId = process.env["ORGNAME"];
-    const streamName = "e2e_automate";
-    const basicAuthCredentials = Buffer.from(
-      `${process.env["ZO_ROOT_USER_EMAIL"]}:${process.env["ZO_ROOT_USER_PASSWORD"]}`
-    ).toString("base64");
-
-    const headers = {
-      Authorization: `Basic ${basicAuthCredentials}`,
-      "Content-Type": "application/json",
-    };
-
-    // const logsdata = {}; // Fill this with your actual data
-
-    // Making a POST request using fetch API
-    const response = await page.evaluate(
-      async ({ url, headers, orgId, streamName, logsdata }) => {
-        const fetchResponse = await fetch(
-          `${url}/api/${orgId}/${streamName}/_json`,
-          {
-            method: "POST",
-            headers: headers,
-            body: JSON.stringify(logsdata),
-          }
-        );
-        return await fetchResponse.json();
-      },
-      {
-        url: process.env.INGESTION_URL,
-        headers: headers,
-        orgId: orgId,
-        streamName: streamName,
-        logsdata: logsdata,
-      }
-    );
-
-    console.log(response);
+    await page.waitForTimeout(1000)
+    await ingestion(page);
+    await page.waitForTimeout(2000)
 
     await page.goto(
       `${logData.logsUrl}?org_identifier=${process.env["ORGNAME"]}`
@@ -110,7 +96,6 @@ test.describe("Sanity testcases", () => {
     const allsearch = page.waitForResponse("**/api/default/_search**");
     await selectStreamAndStreamTypeForLogs(page, logData.Stream);
     await applyQueryButton(page);
-
     // const streams = page.waitForResponse("**/api/default/streams**");
   });
 
@@ -851,8 +836,8 @@ test.describe("Sanity testcases", () => {
     await page
       .locator('[data-test="index-dropdown-stream"]')
       .fill("e2e_tabledashboard");
-      await page.waitForTimeout(4000);
-    await page.getByRole("option", { name: "e2e_tabledashboard" }).click({force: true});
+    await page.waitForTimeout(4000);
+    await page.getByRole("option", { name: "e2e_tabledashboard" }).click({ force: true });
     await page.waitForTimeout(6000);
 
     await page
@@ -878,8 +863,8 @@ test.describe("Sanity testcases", () => {
     await page
       .locator('[data-test="datetime-timezone-select"]')
       .fill("Asia/Calcutta");
-      await page.getByText("Asia/Calcutta", { exact: true }).click();
-      await page.waitForTimeout(200);
+    await page.getByText("Asia/Calcutta", { exact: true }).click();
+    await page.waitForTimeout(200);
 
     // NOTE: pass selected timezone
     const calcuttaTime = toZonedTime(new Date(timestamp), "Asia/Calcutta");