diff --git a/backend/migrations/20231128105015_add_reports_worker_group.down.sql b/backend/migrations/20231128105015_add_reports_worker_group.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20231128105015_add_reports_worker_group.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20231128105015_add_reports_worker_group.up.sql b/backend/migrations/20231128105015_add_reports_worker_group.up.sql new file mode 100644 index 0000000000..ea970d95a1 --- /dev/null +++ b/backend/migrations/20231128105015_add_reports_worker_group.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +INSERT INTO config (name, config) VALUES + ('worker__reports', '{"init_bash": "apt-get update\napt-get install -y chromium", "worker_tags": ["deno", "python3", "go", "bash", "powershell", "dependency", "flow", "hub", "other", "bun", "chromium"]}') ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index 167072ff9f..dceaf012a1 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -8,12 +8,12 @@ use windmill_common::{ error::{self, Error}, - users::SUPERADMIN_SECRET_EMAIL, + users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, DB, }; pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { - if email == SUPERADMIN_SECRET_EMAIL { + if email == SUPERADMIN_SECRET_EMAIL || email == SUPERADMIN_NOTIFICATION_EMAIL { return Ok(()); } let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index de8a193b24..a349d928f6 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -7,6 +7,7 @@ */ pub const SUPERADMIN_SECRET_EMAIL: &str = "superadmin_secret@windmill.dev"; +pub const SUPERADMIN_NOTIFICATION_EMAIL: &str = "superadmin_notification@windmill.dev"; pub fn username_to_permissioned_as(user: &str) -> String { if user.contains('@') { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index fc9994c48e..c99191ff7a 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -55,7 +55,7 @@ use windmill_common::{ oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, schedule::Schedule, scripts::{ScriptHash, ScriptLang}, - users::SUPERADMIN_SECRET_EMAIL, + users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, worker::{to_raw_value, WORKER_CONFIG}, DB, METRICS_ENABLED, }; @@ -2095,8 +2095,8 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection job_payload: JobPayload, args: T, user: &str, - email: &str, - permissioned_as: String, + mut email: &str, + mut permissioned_as: String, scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, @@ -2249,6 +2249,10 @@ pub async fn push<'c, T: Serialize + Send + Sync, R: rsmq_async::RsmqConnection priority, ), JobPayload::ScriptHub { path } => { + if path == "hub/7771/slack" { + permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); + email = SUPERADMIN_NOTIFICATION_EMAIL; + } ( None, Some(path), diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0f1c5f13a5..7e9ad4b05c 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -32,7 +32,7 @@ use windmill_common::{ flows::{FlowModule, FlowModuleValue, FlowValue}, jobs::{JobKind, QueuedJob}, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, - users::SUPERADMIN_SECRET_EMAIL, + users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{rd_string, StripPath}, worker::{ to_raw_value, to_raw_value_owned, update_ping, CLOUD_HOSTED, WORKER_CONFIG, WORKER_GROUP, @@ -142,7 +142,8 @@ pub async fn create_token_for_owner( .fetch_optional(db) .await? .unwrap_or(false) - || email == SUPERADMIN_SECRET_EMAIL; + || email == SUPERADMIN_SECRET_EMAIL + || email == SUPERADMIN_NOTIFICATION_EMAIL; sqlx::query_scalar!( "INSERT INTO token diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 683447d8ca..cade7434fa 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -30,7 +30,7 @@ "graphql": "^16.7.1", "highlight.js": "^11.8.0", "lodash": "^4.17.21", - "lucide-svelte": "^0.277.0", + "lucide-svelte": "^0.293.0", "monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0", "monaco-graphql": "^1.3.0", "monaco-languageclient": "~7.0.1", @@ -4982,9 +4982,9 @@ "optional": true }, "node_modules/lucide-svelte": { - "version": "0.277.0", - "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.277.0.tgz", - "integrity": "sha512-HrlMk0mTOd3MVSWA2ZHR79QrCycLRV4pey4HWGnGGoVFsh0/P7zGyDCJxy/MKe2t4SJtcStLdKIybMHc5pjFSQ==", + "version": "0.293.0", + "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.293.0.tgz", + "integrity": "sha512-nQ6QO6anjjiVoipw3dNptzI6NWuiArlhysVtRpoAWICSQVMR3ybwVf6piUUqbjGe+RfGufLzCl93C/yhsS+p4w==", "peerDependencies": { "svelte": ">=3 <5" } diff --git a/frontend/package.json b/frontend/package.json index 0864a6feae..dcaadb4597 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -113,7 +113,7 @@ "graphql": "^16.7.1", "highlight.js": "^11.8.0", "lodash": "^4.17.21", - "lucide-svelte": "^0.277.0", + "lucide-svelte": "^0.293.0", "monaco-editor": "npm:@codingame/monaco-editor-treemended@>=1.83.5 <1.84.0", "monaco-graphql": "^1.3.0", "monaco-languageclient": "~7.0.1", @@ -402,4 +402,4 @@ "optionalDependencies": { "fsevents": "^2.3.3" } -} +} \ No newline at end of file diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 8459d7a959..9dd76e2177 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -39,7 +39,8 @@ MoreVertical, RefreshCw, Save, - Smartphone + Smartphone, + FileClock } from 'lucide-svelte' import { getContext } from 'svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' @@ -81,6 +82,7 @@ import AppTimeline from './AppTimeline.svelte' import type DiffDrawer from '$lib/components/DiffDrawer.svelte' import { cloneDeep } from 'lodash' + import AppReportsDrawer from './AppReportsDrawer.svelte' async function hash(message) { try { @@ -511,7 +513,13 @@ inputsDrawerOpen = true } }, - + { + displayName: 'Schedule Reports', + icon: FileClock, + action: () => { + appReportingDrawerOpen = true + } + }, ...(savedApp ? [ { @@ -546,6 +554,8 @@ } let rightColumnSelect: 'timeline' | 'detail' = 'timeline' + + let appReportingDrawerOpen = false @@ -1012,6 +1022,8 @@ + +
diff --git a/frontend/src/lib/components/apps/editor/AppPreview.svelte b/frontend/src/lib/components/apps/editor/AppPreview.svelte index 168db72274..6657f9ebae 100644 --- a/frontend/src/lib/components/apps/editor/AppPreview.svelte +++ b/frontend/src/lib/components/apps/editor/AppPreview.svelte @@ -170,6 +170,7 @@ class="{$$props.class} {lockedClasses} {width} h-full bg-surface {app.fullscreen ? '' : 'max-w-7xl'} mx-auto" + id="app-content" > {#if $appStore.grid}
+ import { enterpriseLicense } from '$lib/stores' + import CronInput from '$lib/components/CronInput.svelte' + import ScriptPicker from '$lib/components/ScriptPicker.svelte' + import Section from '$lib/components/Section.svelte' + import Alert from '$lib/components/common/alert/Alert.svelte' + import Drawer from '$lib/components/common/drawer/Drawer.svelte' + import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' + import Tab from '$lib/components/common/tabs/Tab.svelte' + import Tabs from '$lib/components/common/tabs/Tabs.svelte' + import { + FlowService, + RawScript, + ScheduleService, + SettingService, + WorkspaceService + } from '$lib/gen' + import { workspaceStore } from '$lib/stores' + import { emptyString, formatCron, sendUserToast } from '$lib/utils' + import SchemaForm from '$lib/components/SchemaForm.svelte' + import Button from '$lib/components/common/button/Button.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { RotateCw, Save } from 'lucide-svelte' + import { CUSTOM_TAGS_SETTING, WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts' + import { loadSchemaFromPath } from '$lib/infer' + export let appPath: string + export let open = false + + let appReportingEnabled = false + let appReportingStartupDuration = 5 + let appReportingSchedule: { + cron: string + timezone: string + } = { + cron: '0 0 12 * *', + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone + } + let selectedTab: 'email' | 'slack' | 'discord' | 'custom' = 'custom' + let customPath: string | undefined = undefined + let customPathSchema: Record = {} + let args: Record = {} + let areArgsValid = true + + $: customPath + ? loadSchemaFromPath(customPath).then((schema) => { + customPathSchema = schema + ? { + ...schema, + properties: Object.fromEntries( + Object.entries(schema.properties ?? {}).filter( + ([key, _]) => key !== 'pdf' && key !== 'app_path' + ) + ) + } + : {} + }) + : (customPathSchema = {}) + + let isSlackConnectedWorkspace = false + async function getWorspaceSlackSetting() { + const settings = await WorkspaceService.getSettings({ + workspace: $workspaceStore! + }) + if (settings.slack_name) { + isSlackConnectedWorkspace = true + } else { + isSlackConnectedWorkspace = false + } + } + getWorspaceSlackSetting() + + async function getAppReportingInfo() { + const flowPath = appPath + '_reports' + try { + const flow = await FlowService.getFlowByPath({ + workspace: $workspaceStore!, + path: flowPath + }) + const schedule = await ScheduleService.getSchedule({ + workspace: $workspaceStore!, + path: flowPath + }) + appReportingSchedule = { + cron: schedule.schedule, + timezone: schedule.timezone + } + appReportingStartupDuration ??= schedule.args?.startup_duration + args = schedule.args + ? Object.fromEntries( + Object.entries(schedule.args).filter( + ([key, _]) => key !== 'app_path' && key !== 'startup_duration' + ) + ) + : {} + + selectedTab = + flow.value.modules[1]?.value.type === 'script' + ? flow.value.modules[1].value.path === notificationScripts.email.path + ? 'email' + : flow.value.modules[1].value.path === notificationScripts.slack.path + ? 'slack' + : flow.value.modules[1].value.path === notificationScripts.discord.path + ? 'discord' + : 'custom' + : 'custom' + + customPath = + selectedTab === 'custom' && + flow.value.modules[1]?.value.type === 'script' && + !flow.value.modules[1].value.path.startsWith('hub/') + ? flow.value.modules[1].value.path + : undefined + + appReportingEnabled = true + } catch (err) {} + } + + $: appPath && getAppReportingInfo() + + async function disableAppReporting(skipToast = false) { + const flowPath = appPath + '_reports' + await ScheduleService.deleteSchedule({ + workspace: $workspaceStore!, + path: flowPath + }) + await FlowService.deleteFlowByPath({ + workspace: $workspaceStore!, + path: flowPath + }) + if (!skipToast) { + appReportingEnabled = false + sendUserToast('App reporting disabled') + } + } + + const pdfPreviewScript = `import puppeteer from \'puppeteer-core\'; +export async function main(app_path: string, startup_duration = 5) { + const browser = await puppeteer.launch({ headless: \'new\', executablePath: \'/usr/bin/chromium\', args: [\'--no-sandbox\'] }); + const page = await browser.newPage(); + await page.setCookie({ + "name": "token", + "value": Bun.env["WM_TOKEN"], + "domain": Bun.env["WM_BASE_URL"]?.replace(/https?:\\/\\//, \'\') + }) + await page.goto(Bun.env["WM_BASE_URL"] + \'/apps/get/\' + app_path + \'?workspace=\' + Bun.env["WM_WORKSPACE"]); + await new Promise((resolve, _) => { + setTimeout(resolve, startup_duration * 1000) + }) + await page.$eval("#sidebar", el => el.remove()) + await page.$eval("#content", el => el.classList.remove("md:pl-12")) + await page.$eval("#app-edit-btn", el => el.remove()) + const elem = await page.$(\'#app-content\'); + const { height } = await elem.boundingBox(); + await page.setViewport({ width: 1200, height }); + await new Promise((resolve, _) => { + setTimeout(resolve, 200) + }) + const pdf = await page.pdf({ + printBackground: true, + width: 1200, + height + }); + await browser.close(); + return Buffer.from(pdf).toString(\'base64\'); +}` + + const notificationScripts = { + discord: { + path: 'hub/7772/discord', + schema: { + type: 'object', + properties: { + discord_webhook: { + type: 'object', + format: 'resource-discord_webhook', + properties: {}, + required: [], + description: '' + } + }, + required: ['discord_webhook'] + } + }, + slack: { + path: 'hub/7771/slack', // if to be updated, also update it in in backend/windmill-queue/src/jobs.rs + schema: { + type: 'object', + properties: { + channel: { + type: 'string', + default: '' + } + }, + required: ['channel'] + } + }, + email: { + path: 'hub/7774/smtp', + schema: { + type: 'object', + properties: { + smtp: { + type: 'object', + format: 'resource-smtp', + properties: {}, + required: [], + description: '' + }, + from_email: { + type: 'string', + default: '' + }, + to_email: { + type: 'string', + default: '' + } + }, + required: ['smtp', 'from_email', 'to_email'] + } + } + } + + async function updateAppReporting() { + await disableAppReporting(true) + await enableAppReporting(true) + sendUserToast('App reporting updated') + } + + async function enableAppReporting(skipToast = false) { + const flowPath = appPath + '_reports' + + try { + // will only work if the user is super admin + const customTags = ((await SettingService.getGlobal({ + key: CUSTOM_TAGS_SETTING + })) ?? []) as string[] + + if (!customTags.includes('chromium')) { + await SettingService.setGlobal({ + key: CUSTOM_TAGS_SETTING, + requestBody: { + value: [...customTags, 'chromium'] + } + }) + } + } catch (err) {} + + const inputTransforms: { + [key: string]: { + expr: string + type: 'javascript' + } + } = { + app_path: { + type: 'javascript', + expr: 'flow_input.app_path' + }, + pdf: { + type: 'javascript', + expr: 'results.a' + }, + ...Object.fromEntries( + Object.keys(args).map((key) => [ + key, + { + type: 'javascript', + expr: `flow_input.${key}` + } + ]) + ), + ...(selectedTab === 'slack' + ? { + slack: { + type: 'javascript', + expr: 'flow_input.slack' + } + } + : {}) + } + await FlowService.createFlow({ + workspace: $workspaceStore!, + requestBody: { + summary: appPath + ' - Reports flow', + value: { + modules: [ + { + id: 'a', + value: { + type: 'rawscript', + tag: 'chromium', + content: pdfPreviewScript, + language: RawScript.language.BUN, + input_transforms: { + app_path: { + expr: 'flow_input.app_path', + type: 'javascript' + }, + startup_duration: { + expr: 'flow_input.startup_duration', + type: 'javascript' + } + } + } + }, + { + id: 'b', + value: { + type: 'script', + path: + selectedTab === 'custom' + ? customPath || '' + : notificationScripts[selectedTab].path, + input_transforms: inputTransforms + } + } + ] + }, + schema: { + $schema: 'https://json-schema.org/draft/2020-12/schema', + properties: { + app_path: { + description: '', + type: 'string', + default: null, + format: '' + }, + startup_duration: { + description: '', + type: 'integer', + default: 5, + format: '' + }, + ...(selectedTab === 'custom' + ? customPathSchema.properties + : notificationScripts[selectedTab].schema.properties), + ...(selectedTab === 'slack' + ? { + slack: { + description: '', + type: 'object', + format: 'resource-slack', + properties: {}, + required: [] + } + } + : {}) + }, + required: [ + 'app_path', + 'startup_duration', + ...(selectedTab === 'custom' + ? customPathSchema.required + : notificationScripts[selectedTab].schema.required), + ...(selectedTab === 'slack' ? ['slack'] : []) + ], + type: 'object' + }, + path: flowPath + } + }) + + await ScheduleService.createSchedule({ + workspace: $workspaceStore!, + requestBody: { + path: flowPath, + schedule: formatCron(appReportingSchedule.cron), + timezone: appReportingSchedule.timezone, + script_path: flowPath, + is_flow: true, + args: { + app_path: appPath, + startup_duration: appReportingStartupDuration, + ...args, + ...(selectedTab === 'slack' + ? { + slack: '$res:' + WORKSPACE_SLACK_BOT_TOKEN_PATH + } + : {}) + }, + enabled: true + } + }) + appReportingEnabled = true + if (!skipToast) { + sendUserToast('App reporting enabled') + } + } + + let disabled = true + $: disabled = + emptyString(appReportingSchedule.cron) || + (selectedTab === 'custom' && emptyString(customPath)) || + (selectedTab === 'slack' && !isSlackConnectedWorkspace) || + !areArgsValid + + + + (open = false)} + > +
+ { + if (appReportingEnabled) { + disableAppReporting() + } else { + enableAppReporting() + } + }} + {disabled} + /> +
+ + +
+ Send a PDF preview of the app at a given schedule. Enabling this feature will create a flow + and a schedule in your workspace. +

+ For the flow to be executed, you need to set the WORKER_GROUP environment variable of one of + your workers to "reports" or add the tag "chromium" to one of your worker groups. + + +
+ +
+ +
+
+ +
+
+ +
+ + Custom + +
Email{!$enterpriseLicense ? ' (EE only)' : ''}
+ + Slack{!$enterpriseLicense ? ' (EE only)' : ''} + Discord{!$enterpriseLicense ? ' (EE only)' : ''} + + {#if selectedTab === 'custom'} +
+ { + customPath = ev.detail.path + }} + initialPath={customPath} + allowRefresh + /> +
+
+ Pick a script that does whatever with the PDF report. + +
+ + The script chosen is passed the parameters `pdf: string` and `app_path: string` where + `pdf` is the base64 encoded PDF report and `app_path` is the path of the app being + reported. +
+ {/if} + {#if selectedTab === 'slack'} +
+ {#if isSlackConnectedWorkspace} + + {:else} + +
+

+ The workspace needs to be connected to Slack to use this feature. You can configure it here. +

+
+ + {/if} +
+ {/if} +
+ {#if selectedTab !== 'custom' || customPath !== undefined} + {#key selectedTab + JSON.stringify(customPathSchema ?? {})} + + {/key} + {/if} +
+
+
+ + diff --git a/frontend/src/lib/components/common/alert/Alert.svelte b/frontend/src/lib/components/common/alert/Alert.svelte index f8481c9bf4..cb5d4ab039 100644 --- a/frontend/src/lib/components/common/alert/Alert.svelte +++ b/frontend/src/lib/components/common/alert/Alert.svelte @@ -60,7 +60,7 @@
-
+
{tooltip} {/if} -
- -
+ {#if $$slots.default} +
+ +
+ {/if}
diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index a648b36742..f76678637e 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -38,3 +38,8 @@ export const SCRIPT_VIEW_WEBHOOK_INFO_LINK = 'https://www.windmill.dev/docs/core export const SCRIPT_EDITOR_SHOW_EXPLORE_OTHER_SCRIPTS = true export const SCRIPT_CUSTOMISE_SHOW_KIND = true + +export const WORKER_S3_BUCKET_SYNC_SETTING = 'worker_s3_bucket_sync' +export const CUSTOM_TAGS_SETTING = 'custom_tags' + +export const WORKSPACE_SLACK_BOT_TOKEN_PATH = 'f/slack_bot/bot_token' diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 70edfbaa3b..29fda07ffd 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -210,12 +210,12 @@ function argSigToJsonSchemaType( export async function loadSchemaFromPath(path: string, hash?: string): Promise { if (path.startsWith('hub/')) { const { content, language, schema } = await ScriptService.getHubScriptByPath({ path }) - if (language == 'deno' || language == 'nativets') { - const newSchema = emptySchema() - await inferArgs('deno' as SupportedLanguage, content ?? '', newSchema) - return newSchema + if (schema && 'properties' in schema) { + return schema } else { - return schema ?? emptySchema() + const newSchema = emptySchema() + await inferArgs(language as SupportedLanguage, content ?? '', newSchema) + return newSchema } } else if (hash) { const script = await ScriptService.getScriptByHash({ diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 911d3a7d66..177fcbd87d 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -256,6 +256,7 @@
{#if can_write} -
+
diff --git a/frontend/src/routes/(root)/(logged)/workers/+page.svelte b/frontend/src/routes/(root)/(logged)/workers/+page.svelte index f4df65e76e..ceaabd6109 100644 --- a/frontend/src/routes/(root)/(logged)/workers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workers/+page.svelte @@ -9,6 +9,7 @@ import Toggle from '$lib/components/Toggle.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceGroup from '$lib/components/WorkspaceGroup.svelte' + import { CUSTOM_TAGS_SETTING, WORKER_S3_BUCKET_SYNC_SETTING } from '$lib/consts' import { WorkerService, type WorkerPing, SettingService, ConfigService } from '$lib/gen' import { enterpriseLicense, superadmin } from '$lib/stores' import { sendUserToast } from '$lib/toast' @@ -36,9 +37,6 @@ (x) => x[0]?.split(splitter)?.[0] ) - const WORKER_S3_BUCKET_SYNC_SETTING = 'worker_s3_bucket_sync' - const CUSTOM_TAGS_SETTING = 'custom_tags' - let timeSinceLastPing = 0 async function loadWorkers(): Promise {