feat: scheduled app reports (#2714)

* feat: app preview discord

* feat: add custom, slack, email + worker groups

* fix: npm build

* fix: add assignable chromium tag

* feat: use workspace linked slack resource + hub scripts

* fix: nit
This commit is contained in:
HugoCasa
2023-11-29 16:35:42 +01:00
committed by GitHub
parent 3d6fb15a90
commit 3789b34dae
17 changed files with 596 additions and 34 deletions
@@ -0,0 +1 @@
-- Add down migration script here
@@ -0,0 +1,3 @@
-- Add up migration script here
INSERT INTO config (name, config) VALUES
('worker__reports', '{"init_bash": "apt-get update\napt-get install -y chromium", "worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "chromium"]}') ON CONFLICT DO NOTHING;
+2 -2
View File
@@ -8,12 +8,12 @@
use windmill_common::{
error::{self, Error},
users::SUPERADMIN_SECRET_EMAIL,
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
DB,
};
pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
if email == SUPERADMIN_SECRET_EMAIL {
if email == SUPERADMIN_SECRET_EMAIL || email == SUPERADMIN_NOTIFICATION_EMAIL {
return Ok(());
}
let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
+1
View File
@@ -7,6 +7,7 @@
*/
pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev";
pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev";
pub fn username_to_permissioned_as(user: &str) -> String {
if user.contains('@') {
+7 -3
View File
@@ -55,7 +55,7 @@ use windmill_common::{
oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH,
schedule::Schedule,
scripts::{ScriptHash, ScriptLang},
users::SUPERADMIN_SECRET_EMAIL,
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
worker::{to_raw_value, WORKER_CONFIG},
DB, METRICS_ENABLED,
};
@@ -2095,8 +2095,8 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
job_payload: JobPayload,
args: T,
user: &str,
email: &str,
permissioned_as: String,
mut email: &str,
mut permissioned_as: String,
scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
schedule_path: Option<String>,
parent_job: Option<Uuid>,
@@ -2249,6 +2249,10 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection
priority,
),
JobPayload::ScriptHub { path } => {
if path == "hub/7771/slack" {
permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
email = SUPERADMIN_NOTIFICATION_EMAIL;
}
(
None,
Some(path),
+3 -2
View File
@@ -32,7 +32,7 @@ use windmill_common::{
flows::{FlowModule, FlowModuleValue, FlowValue},
jobs::{JobKind, QueuedJob},
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
users::SUPERADMIN_SECRET_EMAIL,
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
utils::{rd_string, StripPath},
worker::{
to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG, WORKER_GROUP,
@@ -142,7 +142,8 @@ pub async fn create_token_for_owner(
.fetch_optional(db)
.await?
.unwrap_or(false)
|| email == SUPERADMIN_SECRET_EMAIL;
|| email == SUPERADMIN_SECRET_EMAIL
|| email == SUPERADMIN_NOTIFICATION_EMAIL;
sqlx::query_scalar!(
"INSERT INTO token
+4 -4
View File
@@ -30,7 +30,7 @@
"graphql": "^16.7.1",
"highlight.js": "^11.8.0",
"lodash": "^4.17.21",
"lucide-svelte": "^0.277.0",
"lucide-svelte": "^0.293.0",
"monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0",
"monaco-graphql": "^1.3.0",
"monaco-languageclient": "~7.0.1",
@@ -4982,9 +4982,9 @@
"optional": true
},
"node_modules/lucide-svelte": {
"version": "0.277.0",
"resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.277.0.tgz",
"integrity": "sha512-HrlMk0mTOd3MVSWA2ZHR79QrCycLRV4pey4HWGnGGoVFsh0/P7zGyDCJxy/MKe2t4SJtcStLdKIybMHc5pjFSQ==",
"version": "0.293.0",
"resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.293.0.tgz",
"integrity": "sha512-nQ6QO6anjjiVoipw3dNptzI6NWuiArlhysVtRpoAWICSQVMR3ybwVf6piUUqbjGe+RfGufLzCl93C/yhsS+p4w==",
"peerDependencies": {
"svelte": ">=3 <5"
}
+2 -2
View File
@@ -113,7 +113,7 @@
"graphql": "^16.7.1",
"highlight.js": "^11.8.0",
"lodash": "^4.17.21",
"lucide-svelte": "^0.277.0",
"lucide-svelte": "^0.293.0",
"monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0",
"monaco-graphql": "^1.3.0",
"monaco-languageclient": "~7.0.1",
@@ -402,4 +402,4 @@
"optionalDependencies": {
"fsevents": "^2.3.3"
}
}
}
@@ -39,7 +39,8 @@
MoreVertical,
RefreshCw,
Save,
Smartphone
Smartphone,
FileClock
} from 'lucide-svelte'
import { getContext } from 'svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
@@ -81,6 +82,7 @@
import AppTimeline from './AppTimeline.svelte'
import type DiffDrawer from '$lib/components/DiffDrawer.svelte'
import { cloneDeep } from 'lodash'
import AppReportsDrawer from './AppReportsDrawer.svelte'
async function hash(message) {
try {
@@ -511,7 +513,13 @@
inputsDrawerOpen = true
}
},
{
displayName: 'Schedule Reports',
icon: FileClock,
action: () => {
appReportingDrawerOpen = true
}
},
...(savedApp
? [
{
@@ -546,6 +554,8 @@
}
let rightColumnSelect: 'timeline' | 'detail' = 'timeline'
let appReportingDrawerOpen = false
</script>
<svelte:window on:keydown={onKeyDown} />
@@ -1012,6 +1022,8 @@
</DrawerContent>
</Drawer>
<AppReportsDrawer bind:open={appReportingDrawerOpen} {appPath} />
<div
class="border-b flex flex-row justify-between py-1 gap-2 gap-y-2 px-2 items-center overflow-y-visible overflow-x-auto"
>
@@ -170,6 +170,7 @@
class="{$$props.class} {lockedClasses} {width} h-full bg-surface {app.fullscreen
? ''
: 'max-w-7xl'} mx-auto"
id="app-content"
>
{#if $appStore.grid}
<div
@@ -0,0 +1,532 @@
<script lang="ts">
import { enterpriseLicense } from '$lib/stores'
import CronInput from '$lib/components/CronInput.svelte'
import ScriptPicker from '$lib/components/ScriptPicker.svelte'
import Section from '$lib/components/Section.svelte'
import Alert from '$lib/components/common/alert/Alert.svelte'
import Drawer from '$lib/components/common/drawer/Drawer.svelte'
import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte'
import Tab from '$lib/components/common/tabs/Tab.svelte'
import Tabs from '$lib/components/common/tabs/Tabs.svelte'
import {
FlowService,
RawScript,
ScheduleService,
SettingService,
WorkspaceService
} from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { emptyString, formatCron, sendUserToast } from '$lib/utils'
import SchemaForm from '$lib/components/SchemaForm.svelte'
import Button from '$lib/components/common/button/Button.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import { RotateCw, Save } from 'lucide-svelte'
import { CUSTOM_TAGS_SETTING, WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts'
import { loadSchemaFromPath } from '$lib/infer'
export let appPath: string
export let open = false
let appReportingEnabled = false
let appReportingStartupDuration = 5
let appReportingSchedule: {
cron: string
timezone: string
} = {
cron: '0 0 12 * *',
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone
}
let selectedTab: 'email' | 'slack' | 'discord' | 'custom' = 'custom'
let customPath: string | undefined = undefined
let customPathSchema: Record<string, any> = {}
let args: Record<string, any> = {}
let areArgsValid = true
$: customPath
? loadSchemaFromPath(customPath).then((schema) => {
customPathSchema = schema
? {
...schema,
properties: Object.fromEntries(
Object.entries(schema.properties ?? {}).filter(
([key, _]) => key !== 'pdf' && key !== 'app_path'
)
)
}
: {}
})
: (customPathSchema = {})
let isSlackConnectedWorkspace = false
async function getWorspaceSlackSetting() {
const settings = await WorkspaceService.getSettings({
workspace: $workspaceStore!
})
if (settings.slack_name) {
isSlackConnectedWorkspace = true
} else {
isSlackConnectedWorkspace = false
}
}
getWorspaceSlackSetting()
async function getAppReportingInfo() {
const flowPath = appPath + '_reports'
try {
const flow = await FlowService.getFlowByPath({
workspace: $workspaceStore!,
path: flowPath
})
const schedule = await ScheduleService.getSchedule({
workspace: $workspaceStore!,
path: flowPath
})
appReportingSchedule = {
cron: schedule.schedule,
timezone: schedule.timezone
}
appReportingStartupDuration ??= schedule.args?.startup_duration
args = schedule.args
? Object.fromEntries(
Object.entries(schedule.args).filter(
([key, _]) => key !== 'app_path' && key !== 'startup_duration'
)
)
: {}
selectedTab =
flow.value.modules[1]?.value.type === 'script'
? flow.value.modules[1].value.path === notificationScripts.email.path
? 'email'
: flow.value.modules[1].value.path === notificationScripts.slack.path
? 'slack'
: flow.value.modules[1].value.path === notificationScripts.discord.path
? 'discord'
: 'custom'
: 'custom'
customPath =
selectedTab === 'custom' &&
flow.value.modules[1]?.value.type === 'script' &&
!flow.value.modules[1].value.path.startsWith('hub/')
? flow.value.modules[1].value.path
: undefined
appReportingEnabled = true
} catch (err) {}
}
$: appPath && getAppReportingInfo()
async function disableAppReporting(skipToast = false) {
const flowPath = appPath + '_reports'
await ScheduleService.deleteSchedule({
workspace: $workspaceStore!,
path: flowPath
})
await FlowService.deleteFlowByPath({
workspace: $workspaceStore!,
path: flowPath
})
if (!skipToast) {
appReportingEnabled = false
sendUserToast('App reporting disabled')
}
}
const pdfPreviewScript = `import puppeteer from \'puppeteer-core\';
export async function main(app_path: string, startup_duration = 5) {
const browser = await puppeteer.launch({ headless: \'new\', executablePath: \'/usr/bin/chromium\', args: [\'--no-sandbox\'] });
const page = await browser.newPage();
await page.setCookie({
"name": "token",
"value": Bun.env["WM_TOKEN"],
"domain": Bun.env["WM_BASE_URL"]?.replace(/https?:\\/\\//, \'\')
})
await page.goto(Bun.env["WM_BASE_URL"] + \'/apps/get/\' + app_path + \'?workspace=\' + Bun.env["WM_WORKSPACE"]);
await new Promise((resolve, _) => {
setTimeout(resolve, startup_duration * 1000)
})
await page.$eval("#sidebar", el => el.remove())
await page.$eval("#content", el => el.classList.remove("md:pl-12"))
await page.$eval("#app-edit-btn", el => el.remove())
const elem = await page.$(\'#app-content\');
const { height } = await elem.boundingBox();
await page.setViewport({ width: 1200, height });
await new Promise((resolve, _) => {
setTimeout(resolve, 200)
})
const pdf = await page.pdf({
printBackground: true,
width: 1200,
height
});
await browser.close();
return Buffer.from(pdf).toString(\'base64\');
}`
const notificationScripts = {
discord: {
path: 'hub/7772/discord',
schema: {
type: 'object',
properties: {
discord_webhook: {
type: 'object',
format: 'resource-discord_webhook',
properties: {},
required: [],
description: ''
}
},
required: ['discord_webhook']
}
},
slack: {
path: 'hub/7771/slack', // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs
schema: {
type: 'object',
properties: {
channel: {
type: 'string',
default: ''
}
},
required: ['channel']
}
},
email: {
path: 'hub/7774/smtp',
schema: {
type: 'object',
properties: {
smtp: {
type: 'object',
format: 'resource-smtp',
properties: {},
required: [],
description: ''
},
from_email: {
type: 'string',
default: ''
},
to_email: {
type: 'string',
default: ''
}
},
required: ['smtp', 'from_email', 'to_email']
}
}
}
async function updateAppReporting() {
await disableAppReporting(true)
await enableAppReporting(true)
sendUserToast('App reporting updated')
}
async function enableAppReporting(skipToast = false) {
const flowPath = appPath + '_reports'
try {
// will only work if the user is super admin
const customTags = ((await SettingService.getGlobal({
key: CUSTOM_TAGS_SETTING
})) ?? []) as string[]
if (!customTags.includes('chromium')) {
await SettingService.setGlobal({
key: CUSTOM_TAGS_SETTING,
requestBody: {
value: [...customTags, 'chromium']
}
})
}
} catch (err) {}
const inputTransforms: {
[key: string]: {
expr: string
type: 'javascript'
}
} = {
app_path: {
type: 'javascript',
expr: 'flow_input.app_path'
},
pdf: {
type: 'javascript',
expr: 'results.a'
},
...Object.fromEntries(
Object.keys(args).map((key) => [
key,
{
type: 'javascript',
expr: `flow_input.${key}`
}
])
),
...(selectedTab === 'slack'
? {
slack: {
type: 'javascript',
expr: 'flow_input.slack'
}
}
: {})
}
await FlowService.createFlow({
workspace: $workspaceStore!,
requestBody: {
summary: appPath + ' - Reports flow',
value: {
modules: [
{
id: 'a',
value: {
type: 'rawscript',
tag: 'chromium',
content: pdfPreviewScript,
language: RawScript.language.BUN,
input_transforms: {
app_path: {
expr: 'flow_input.app_path',
type: 'javascript'
},
startup_duration: {
expr: 'flow_input.startup_duration',
type: 'javascript'
}
}
}
},
{
id: 'b',
value: {
type: 'script',
path:
selectedTab === 'custom'
? customPath || ''
: notificationScripts[selectedTab].path,
input_transforms: inputTransforms
}
}
]
},
schema: {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {
app_path: {
description: '',
type: 'string',
default: null,
format: ''
},
startup_duration: {
description: '',
type: 'integer',
default: 5,
format: ''
},
...(selectedTab === 'custom'
? customPathSchema.properties
: notificationScripts[selectedTab].schema.properties),
...(selectedTab === 'slack'
? {
slack: {
description: '',
type: 'object',
format: 'resource-slack',
properties: {},
required: []
}
}
: {})
},
required: [
'app_path',
'startup_duration',
...(selectedTab === 'custom'
? customPathSchema.required
: notificationScripts[selectedTab].schema.required),
...(selectedTab === 'slack' ? ['slack'] : [])
],
type: 'object'
},
path: flowPath
}
})
await ScheduleService.createSchedule({
workspace: $workspaceStore!,
requestBody: {
path: flowPath,
schedule: formatCron(appReportingSchedule.cron),
timezone: appReportingSchedule.timezone,
script_path: flowPath,
is_flow: true,
args: {
app_path: appPath,
startup_duration: appReportingStartupDuration,
...args,
...(selectedTab === 'slack'
? {
slack: '$res:' + WORKSPACE_SLACK_BOT_TOKEN_PATH
}
: {})
},
enabled: true
}
})
appReportingEnabled = true
if (!skipToast) {
sendUserToast('App reporting enabled')
}
}
let disabled = true
$: disabled =
emptyString(appReportingSchedule.cron) ||
(selectedTab === 'custom' && emptyString(customPath)) ||
(selectedTab === 'slack' && !isSlackConnectedWorkspace) ||
!areArgsValid
</script>
<Drawer bind:open>
<DrawerContent title="Schedule Reports" on:close={() => (open = false)}
><svelte:fragment slot="actions">
<div class="mr-4 center-center -mt-2">
<Toggle
checked={appReportingEnabled}
options={{ right: 'enable', left: 'disable' }}
on:change={async () => {
if (appReportingEnabled) {
disableAppReporting()
} else {
enableAppReporting()
}
}}
{disabled}
/>
</div>
<Button
color="dark"
startIcon={{ icon: Save }}
size="sm"
on:click={async () => {
appReportingEnabled ? await updateAppReporting() : await enableAppReporting()
open = false
}}
{disabled}
>
{appReportingEnabled ? 'Update' : 'Save and enable'}
</Button>
</svelte:fragment>
<div class="flex flex-col gap-8">
<Alert type="info" title="Scheduled PDF reports"
>Send a PDF preview of the app at a given schedule. Enabling this feature will create a flow
and a schedule in your workspace.
<br /><br />
For the flow to be executed, you need to set the WORKER_GROUP environment variable of one of
your workers to "reports" or add the tag "chromium" to one of your worker groups.
</Alert>
<Section label="Reporting schedule">
<CronInput
bind:schedule={appReportingSchedule.cron}
bind:timezone={appReportingSchedule.timezone}
/>
</Section>
<Section
label="Startup duration in seconds"
tooltip="The number of seconds to wait before capturing a preview to ensure that all startup scripts
have been executed."
>
<div class="w-full pt-2">
<input
type="number"
class="text-sm w-full font-semibold"
bind:value={appReportingStartupDuration}
/>
</div>
</Section>
<Section label="Notification">
<Tabs bind:selected={selectedTab}>
<Tab value="custom">Custom</Tab>
<Tab value="email" disabled={!$enterpriseLicense}>
<div class="flex flex-row gap-1 items-center"
>Email{!$enterpriseLicense ? ' (EE only)' : ''}</div
>
</Tab>
<Tab value="slack" disabled={!$enterpriseLicense}
>Slack{!$enterpriseLicense ? ' (EE only)' : ''}</Tab
>
<Tab value="discord" disabled={!$enterpriseLicense}
>Discord{!$enterpriseLicense ? ' (EE only)' : ''}</Tab
>
</Tabs>
{#if selectedTab === 'custom'}
<div class="pt-2">
<ScriptPicker
on:select={(ev) => {
customPath = ev.detail.path
}}
initialPath={customPath}
allowRefresh
/>
</div>
<div class="prose text-2xs text-tertiary mt-2">
Pick a script that does whatever with the PDF report.
<br />
The script chosen is passed the parameters `pdf: string` and `app_path: string` where
`pdf` is the base64 encoded PDF report and `app_path` is the path of the app being
reported.
</div>
{/if}
{#if selectedTab === 'slack'}
<div class="pt-4">
{#if isSlackConnectedWorkspace}
<Alert type="info" title="Will use the Slack resource linked to the workspace" />
{:else}
<Alert type="error" title="Workspace not connected to Slack">
<div class="flex flex-row gap-x-1 w-full items-center">
<p class="text-clip grow min-w-0">
The workspace needs to be connected to Slack to use this feature. You can <a
target="_blank"
href="/workspace_settings?tab=slack">configure it here</a
>.
</p>
<Button
variant="border"
color="light"
on:click={getWorspaceSlackSetting}
startIcon={{ icon: RotateCw }}
/>
</div>
</Alert>
{/if}
</div>
{/if}
<div class="w-full pt-4">
{#if selectedTab !== 'custom' || customPath !== undefined}
{#key selectedTab + JSON.stringify(customPathSchema ?? {})}
<SchemaForm
bind:isValid={areArgsValid}
bind:args
schema={selectedTab !== 'custom'
? notificationScripts[selectedTab].schema
: customPathSchema}
/>
{/key}
{/if}
</div>
</Section>
</div>
</DrawerContent>
</Drawer>
@@ -60,7 +60,7 @@
<svelte:component this={icons[type]} class={classes[type].iconClass} size={16} />
</div>
<div class="ml-2 w-full">
<div class="ml-2 pt-[0.15rem] w-full">
<span
class={classNames(
size === 'sm' ? 'text-sm' : 'text-xs ',
@@ -72,15 +72,17 @@
<Tooltip {documentationLink} scale={0.9}>{tooltip}</Tooltip>
{/if}
</span>
<div
class={classNames(
size === 'sm' ? 'text-sm' : 'text-xs ',
'mt-2',
classes[type].descriptionClass
)}
>
<slot />
</div>
{#if $$slots.default}
<div
class={classNames(
size === 'sm' ? 'text-sm' : 'text-xs ',
'mt-2',
classes[type].descriptionClass
)}
>
<slot />
</div>
{/if}
</div>
</div>
</div>
+5
View File
@@ -38,3 +38,8 @@ export const SCRIPT_VIEW_WEBHOOK_INFO_LINK = 'https://www.windmill.dev/docs/core
export const SCRIPT_EDITOR_SHOW_EXPLORE_OTHER_SCRIPTS = true
export const SCRIPT_CUSTOMISE_SHOW_KIND = true
export const WORKER_S3_BUCKET_SYNC_SETTING = 'worker_s3_bucket_sync'
export const CUSTOM_TAGS_SETTING = 'custom_tags'
export const WORKSPACE_SLACK_BOT_TOKEN_PATH = 'f/slack_bot/bot_token'
+5 -5
View File
@@ -210,12 +210,12 @@ function argSigToJsonSchemaType(
export async function loadSchemaFromPath(path: string, hash?: string): Promise<Schema> {
if (path.startsWith('hub/')) {
const { content, language, schema } = await ScriptService.getHubScriptByPath({ path })
if (language == 'deno' || language == 'nativets') {
const newSchema = emptySchema()
await inferArgs('deno' as SupportedLanguage, content ?? '', newSchema)
return newSchema
if (schema && 'properties' in schema) {
return schema
} else {
return schema ?? emptySchema()
const newSchema = emptySchema()
await inferArgs(language as SupportedLanguage, content ?? '', newSchema)
return newSchema
}
} else if (hash) {
const script = await ScriptService.getScriptByHash({
@@ -256,6 +256,7 @@
</div>
<div
id="sidebar"
class={classNames(
'hidden md:flex md:flex-col md:fixed md:inset-y-0 transition-all ease-in-out duration-200 shadow-md z-40 ',
isCollapsed ? 'md:w-12' : 'md:w-40',
@@ -307,6 +308,7 @@
</div>
</div>
<div
id="content"
class={classNames(
'w-full flex flex-col flex-1 h-full',
devOnly ? '!pl-0' : isCollapsed ? 'md:pl-12' : 'md:pl-40',
@@ -54,7 +54,7 @@
noBackend={false}
/>
{#if can_write}
<div class="absolute bottom-4 z-50 right-4">
<div id="app-edit-btn" class="absolute bottom-4 z-50 right-4">
<Button size="sm" startIcon={{ icon: Pen }} variant="border" href="/apps/edit/{app.path}"
>Edit</Button
>
@@ -9,6 +9,7 @@
import Toggle from '$lib/components/Toggle.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import WorkspaceGroup from '$lib/components/WorkspaceGroup.svelte'
import { CUSTOM_TAGS_SETTING, WORKER_S3_BUCKET_SYNC_SETTING } from '$lib/consts'
import { WorkerService, type WorkerPing, SettingService, ConfigService } from '$lib/gen'
import { enterpriseLicense, superadmin } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
@@ -36,9 +37,6 @@
(x) => x[0]?.split(splitter)?.[0]
)
const WORKER_S3_BUCKET_SYNC_SETTING = 'worker_s3_bucket_sync'
const CUSTOM_TAGS_SETTING = 'custom_tags'
let timeSinceLastPing = 0
async function loadWorkers(): Promise<void> {