feat: deprecate .wmillignore in favor of wmill.yaml/includes

cli/conf.ts

@@ -0,0 +1,16 @@
+import { yamlParse } from "./deps.ts";
+
+export interface WmillConfiguration {
+  includes?: string[];
+}
+export async function readConfigFile(): Promise<WmillConfiguration> {
+  try {
+    const conf = yamlParse(
+      await Deno.readTextFile("wmill.yaml")
+    ) as WmillConfiguration;
+
+    return typeof conf == "object" ? conf : ({} as WmillConfiguration);
+  } catch (e) {
+    return {};
+  }
+}

cli/deps.ts

@@ -1,5 +1,3 @@
-import Gitignore from "npm:gitignore-fs";
-
 // windmill
 export { setClient } from "https://deno.land/x/windmill@v1.226.1/mod.ts";
 export * from "https://deno.land/x/windmill@v1.226.1/windmill-api/index.ts";
@@ -42,7 +40,7 @@ export * as cbor from "https://deno.land/x/cbor@v1.4.1/index.js";
 export { default as Murmurhash3 } from "https://deno.land/x/murmurhash@v1.0.0/mod.ts";
 export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index.ts";
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
-export * as ignore from "npm:ignore";
+export { minimatch } from "npm:minimatch";
 export { default as JSZip } from "npm:jszip@3.7.1";
 export * as log from "https://deno.land/std@0.186.0/log/mod.ts";
 export {
@@ -50,3 +48,5 @@ export {
   parse as yamlParse,
 } from "https://deno.land/std@0.184.0/yaml/mod.ts";
 export { open } from "https://deno.land/x/open@v0.0.5/index.ts";
+export { encodeHex } from "https://deno.land/std@0.207.0/encoding/hex.ts";
+export { default as gitignore_parser } from "npm:gitignore-parser";

cli/metadata.ts

@@ -0,0 +1,484 @@
+// deno-lint-ignore-file no-explicit-any
+import { GlobalOptions } from "./types.ts";
+import { colors, encodeHex, log, yamlParse, yamlStringify } from "./deps.ts";
+import {
+  ScriptMetadata,
+  defaultScriptMetadata,
+} from "./bootstrap/script_bootstrap.ts";
+import {
+  instantiate as instantiateWasm,
+  parse_bash,
+  parse_bigquery,
+  parse_deno,
+  parse_go,
+  parse_graphql,
+  parse_mssql,
+  parse_mysql,
+  parse_powershell,
+  parse_python,
+  parse_snowflake,
+  parse_sql,
+} from "./wasm/windmill_parser_wasm.generated.js";
+import { Workspace } from "./workspace.ts";
+import { SchemaProperty } from "./bootstrap/common.ts";
+import { ScriptLanguage } from "./script_common.ts";
+import { inferContentTypeFromFilePath } from "./script_common.ts";
+
+export async function generateAllMetadata() {}
+
+export async function generateMetadataInternal(
+  scriptPath: string,
+  workspace: Workspace,
+  opts: GlobalOptions & {
+    lockOnly?: boolean | undefined;
+    schemaOnly?: boolean | undefined;
+  }
+) {
+  const remotePath = scriptPath
+    .substring(0, scriptPath.indexOf("."))
+    .replaceAll("\\", "/");
+  const metadataWithType = await parseMetadataFile(remotePath, undefined);
+
+  // read script content
+  const scriptContent = await Deno.readTextFile(scriptPath);
+  const metadataContent = await Deno.readTextFile(metadataWithType.path);
+  if (
+    await checkifMetadataUptodate(remotePath, scriptContent + metadataContent)
+  ) {
+    log.info(
+      colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
+    );
+    return;
+  }
+
+  const metadataParsedContent = metadataWithType?.payload as Record<
+    string,
+    any
+  >;
+
+  const language = inferContentTypeFromFilePath(scriptPath);
+  if (!opts.lockOnly) {
+    await updateScriptSchema(
+      scriptContent,
+      language,
+      metadataParsedContent,
+      scriptPath
+    );
+  }
+
+  if (!opts.schemaOnly) {
+    await updateScriptLock(
+      workspace,
+      scriptContent,
+      language,
+      remotePath,
+      metadataParsedContent
+    );
+  }
+
+  let metaPath = remotePath + ".script.yaml";
+  let newMetadataContent = yamlStringify(metadataParsedContent);
+  if (metadataWithType.isJson) {
+    metaPath = remotePath + ".script.json";
+    newMetadataContent = JSON.stringify(metadataParsedContent);
+  }
+  await updateMetadataLock(remotePath, scriptContent + newMetadataContent);
+  await Deno.writeTextFile(metaPath, newMetadataContent);
+}
+
+async function updateScriptSchema(
+  scriptContent: string,
+  language: ScriptLanguage,
+  metadataContent: Record<string, any>,
+  path: string
+): Promise<void> {
+  // infer schema from script content and update it inplace
+  await instantiateWasm();
+  const newSchema = inferSchema(
+    language,
+    scriptContent,
+    metadataContent.schema,
+    path
+  );
+  metadataContent.schema = newSchema;
+}
+
+async function updateScriptLock(
+  workspace: Workspace,
+  scriptContent: string,
+  language: ScriptLanguage,
+  remotePath: string,
+  metadataContent: Record<string, any>
+): Promise<void> {
+  // generate the script lock running a dependency job in Windmill and update it inplace
+  // TODO: update this once the client is released
+  const rawResponse = await fetch(
+    `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
+    {
+      method: "POST",
+      headers: {
+        Cookie: `token=${workspace.token}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        raw_scripts: [
+          {
+            raw_code: scriptContent,
+            language: language,
+            script_path: remotePath,
+          },
+        ],
+        entrypoint: remotePath,
+      }),
+    }
+  );
+
+  try {
+    const response = await rawResponse.json();
+    const lock = response.lock;
+    if (lock === undefined) {
+      throw new Error(
+        `Failed to generate lockfile. Full response was: ${JSON.stringify(
+          response
+        )}`
+      );
+    }
+    metadataContent.lock = lock;
+  } catch {
+    throw new Error(
+      `Failed to generate lockfile. Status was: ${rawResponse.statusText}`
+    );
+  }
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////
+// below functions copied from Windmill's FE inferArgs function. TODO: refactor           //
+////////////////////////////////////////////////////////////////////////////////////////////
+export function inferSchema(
+  language: ScriptLanguage,
+  content: string,
+  currentSchema: any,
+  path: string
+) {
+  let inferedSchema: any;
+  if (language === "python3") {
+    inferedSchema = JSON.parse(parse_python(content));
+  } else if (language === "nativets") {
+    inferedSchema = JSON.parse(parse_deno(content));
+  } else if (language === "bun") {
+    inferedSchema = JSON.parse(parse_deno(content));
+  } else if (language === "deno") {
+    inferedSchema = JSON.parse(parse_deno(content));
+  } else if (language === "go") {
+    inferedSchema = JSON.parse(parse_go(content));
+  } else if (language === "mysql") {
+    inferedSchema = JSON.parse(parse_mysql(content));
+    inferedSchema.args = [
+      { name: "database", typ: { resource: "mysql" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "bigquery") {
+    inferedSchema = JSON.parse(parse_bigquery(content));
+    inferedSchema.args = [
+      { name: "database", typ: { resource: "bigquery" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "snowflake") {
+    inferedSchema = JSON.parse(parse_snowflake(content));
+    inferedSchema.args = [
+      { name: "database", typ: { resource: "snowflake" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "mssql") {
+    inferedSchema = JSON.parse(parse_mssql(content));
+    inferedSchema.args = [
+      { name: "database", typ: { resource: "ms_sql_server" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "postgresql") {
+    inferedSchema = JSON.parse(parse_sql(content));
+    inferedSchema.args = [
+      { name: "database", typ: { resource: "postgresql" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "graphql") {
+    inferedSchema = JSON.parse(parse_graphql(content));
+    inferedSchema.args = [
+      { name: "api", typ: { resource: "graphql" } },
+      ...inferedSchema.args,
+    ];
+  } else if (language === "bash") {
+    inferedSchema = JSON.parse(parse_bash(content));
+  } else if (language === "powershell") {
+    inferedSchema = JSON.parse(parse_powershell(content));
+  } else {
+    throw new Error("Invalid language: " + language);
+  }
+  if (inferedSchema.type == "Invalid") {
+    log.info(
+      colors.yellow(
+        `Script ${path} invalid, it cannot be parsed to infer schema.`
+      )
+    );
+    return defaultScriptMetadata().schema;
+  }
+
+  currentSchema.required = [];
+  const oldProperties = JSON.parse(JSON.stringify(currentSchema.properties));
+  currentSchema.properties = {};
+
+  for (const arg of inferedSchema.args) {
+    if (!(arg.name in oldProperties)) {
+      currentSchema.properties[arg.name] = { description: "", type: "" };
+    } else {
+      currentSchema.properties[arg.name] = oldProperties[arg.name];
+    }
+    currentSchema.properties[arg.name] = sortObject(
+      currentSchema.properties[arg.name]
+    );
+
+    argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
+
+    currentSchema.properties[arg.name].default = arg.default;
+
+    if (!arg.has_default && !currentSchema.required.includes(arg.name)) {
+      currentSchema.required.push(arg.name);
+    }
+  }
+
+  return currentSchema;
+}
+
+function sortObject(obj: any): any {
+  return Object.keys(obj)
+    .sort()
+    .reduce(
+      (acc, key) => ({
+        ...acc,
+        [key]: obj[key],
+      }),
+      {}
+    );
+}
+
+function argSigToJsonSchemaType(
+  typ:
+    | string
+    | { resource: string | null }
+    | {
+        list:
+          | string
+          | { str: any }
+          | { object: { key: string; typ: any }[] }
+          | null;
+      }
+    | { str: string[] | null }
+    | { object: { key: string; typ: any }[] },
+  oldS: SchemaProperty
+): void {
+  const newS: SchemaProperty = { type: "" };
+  if (typ === "int") {
+    newS.type = "integer";
+  } else if (typ === "float") {
+    newS.type = "number";
+  } else if (typ === "bool") {
+    newS.type = "boolean";
+  } else if (typ === "email") {
+    newS.type = "string";
+    newS.format = "email";
+  } else if (typ === "sql") {
+    newS.type = "string";
+    newS.format = "sql";
+  } else if (typ === "yaml") {
+    newS.type = "string";
+    newS.format = "yaml";
+  } else if (typ === "bytes") {
+    newS.type = "string";
+    newS.contentEncoding = "base64";
+  } else if (typ === "datetime") {
+    newS.type = "string";
+    newS.format = "date-time";
+  } else if (typeof typ !== "string" && `object` in typ) {
+    newS.type = "object";
+    if (typ.object) {
+      const properties: Record<string, SchemaProperty> = {};
+      for (const prop of typ.object) {
+        properties[prop.key] = { type: undefined };
+        argSigToJsonSchemaType(prop.typ, properties[prop.key]);
+      }
+      newS.properties = properties;
+    }
+  } else if (typeof typ !== "string" && `str` in typ) {
+    newS.type = "string";
+    if (typ.str) {
+      newS.enum = typ.str;
+    }
+  } else if (typeof typ !== "string" && `resource` in typ) {
+    newS.type = "object";
+    newS.format = `resource-${typ.resource}`;
+  } else if (typeof typ !== "string" && `list` in typ) {
+    newS.type = "array";
+    if (typ.list === "int" || typ.list === "float") {
+      newS.items = { type: "number" };
+    } else if (typ.list === "bytes") {
+      newS.items = { type: "string", contentEncoding: "base64" };
+    } else if (typ.list == "string") {
+      newS.items = { type: "string" };
+    } else if (typ.list && typeof typ.list == "object" && "str" in typ.list) {
+      newS.items = { type: "string", enum: typ.list.str };
+    } else {
+      newS.items = { type: "object" };
+    }
+  } else {
+    newS.type = "object";
+  }
+
+  if (oldS.type != newS.type) {
+    for (const prop of Object.getOwnPropertyNames(newS)) {
+      if (prop != "description") {
+        // @ts-ignore: fix
+        delete oldS[prop];
+      }
+    }
+  } else if (oldS.format == "date-time" && newS.format != "date-time") {
+    delete oldS.format;
+  } else if (oldS.items?.type != newS.items?.type) {
+    delete oldS.items;
+  }
+
+  Object.assign(oldS, newS);
+  if (oldS.format?.startsWith("resource-") && newS.type != "object") {
+    oldS.format = undefined;
+  }
+}
+////////////////////////////////////////////////////////////////////////////////////////////
+// end of refactoring TODO                                                                //
+////////////////////////////////////////////////////////////////////////////////////////////
+
+export async function parseMetadataFile(
+  scriptPath: string,
+  generateMetadataIfMissing:
+    | (GlobalOptions & { path: string; workspaceRemote: Workspace })
+    | undefined
+): Promise<{ isJson: boolean; payload: any; path: string }> {
+  let metadataFilePath = scriptPath + ".script.json";
+  try {
+    await Deno.stat(metadataFilePath);
+    return {
+      path: metadataFilePath,
+      payload: JSON.parse(await Deno.readTextFile(metadataFilePath)),
+      isJson: true,
+    };
+  } catch {
+    try {
+      metadataFilePath = scriptPath + ".script.yaml";
+      await Deno.stat(metadataFilePath);
+      const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath));
+      if (Array.isArray(payload?.["lock"])) {
+        payload["lock"] = payload["lock"].join("\n");
+      }
+      return {
+        path: metadataFilePath,
+        payload,
+        isJson: false,
+      };
+    } catch {
+      // no metadata file at all. Create it
+      log.info(
+        colors.blue(`Creating script metadata file for ${metadataFilePath}`)
+      );
+      metadataFilePath = scriptPath + ".script.yaml";
+      let scriptInitialMetadata = defaultScriptMetadata();
+      const scriptInitialMetadataYaml = yamlStringify(
+        scriptInitialMetadata as Record<string, any>
+      );
+      Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, {
+        createNew: true,
+      });
+
+      if (generateMetadataIfMissing) {
+        log.info(
+          colors.blue(`Generating lockfile and schema for ${metadataFilePath}`)
+        );
+        try {
+          await generateMetadataInternal(
+            generateMetadataIfMissing.path,
+            generateMetadataIfMissing.workspaceRemote,
+            generateMetadataIfMissing
+          );
+
+          scriptInitialMetadata = yamlParse(
+            await Deno.readTextFile(metadataFilePath)
+          ) as ScriptMetadata;
+        } catch (e) {
+          log.info(
+            colors.yellow(
+              `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
+            )
+          );
+        }
+      }
+      return {
+        path: metadataFilePath,
+        payload: scriptInitialMetadata,
+        isJson: false,
+      };
+    }
+  }
+}
+
+interface Lock {
+  locks?: { [path: string]: string };
+}
+
+const WMILL_LOCKFILE = "wmill-lock.yaml";
+export async function readLockfile(): Promise<Lock> {
+  try {
+    const lockfile = await Deno.readTextFile(WMILL_LOCKFILE);
+    const read = yamlParse(lockfile);
+    if (typeof read == "object") {
+      return read as Lock;
+    } else {
+      throw new Error("Invalid lockfile");
+    }
+  } catch {
+    const lock = { locks: {} };
+    await Deno.writeTextFile(WMILL_LOCKFILE, yamlStringify(lock));
+    return lock;
+  }
+}
+
+async function generateHash(content: string): Promise<string> {
+  const messageBuffer = new TextEncoder().encode(content);
+  const hashBuffer = await crypto.subtle.digest("SHA-256", messageBuffer);
+  return encodeHex(hashBuffer);
+}
+
+export async function checkifMetadataUptodate(
+  path: string,
+  requirement: string
+) {
+  const conf = await readLockfile();
+  if (!conf.locks) {
+    return false;
+  }
+  const hash = await generateHash(requirement);
+  return conf?.locks?.[path] == hash;
+}
+
+export async function updateMetadataLock(
+  path: string,
+  requirement: string
+): Promise<void> {
+  const conf = await readLockfile();
+  const hash = await generateHash(requirement);
+  if (!conf?.locks) {
+    conf.locks = {};
+  }
+
+  conf.locks[path] = hash;
+  await Deno.writeTextFile(
+    WMILL_LOCKFILE,
+    yamlStringify(conf as Record<string, any>)
+  );
+}

cli/script.ts

@@ -12,31 +12,20 @@ import {
   ScriptService,
   Table,
   writeAllSync,
-  yamlParse,
   yamlStringify,
 } from "./deps.ts";
 import { deepEqual } from "./utils.ts";
 import {
-  ScriptMetadata,
   defaultScriptMetadata,
   scriptBootstrapCode,
 } from "./bootstrap/script_bootstrap.ts";
-import {
-  instantiate as instantiateWasm,
-  parse_bash,
-  parse_bigquery,
-  parse_deno,
-  parse_go,
-  parse_graphql,
-  parse_mssql,
-  parse_mysql,
-  parse_powershell,
-  parse_python,
-  parse_snowflake,
-  parse_sql,
-} from "./wasm/windmill_parser_wasm.generated.js";
+
 import { Workspace } from "./workspace.ts";
-import { SchemaProperty } from "./bootstrap/common.ts";
+import { generateMetadataInternal, parseMetadataFile } from "./metadata.ts";
+import {
+  ScriptLanguage,
+  inferContentTypeFromFilePath,
+} from "./script_common.ts";
 
 export interface ScriptFile {
   parent_hash?: string;
@@ -68,13 +57,13 @@ async function push(opts: PushOptions, filePath: string) {
   }
 
   await requireLogin(opts);
-  await handleFile(filePath, workspace.workspaceId, [], undefined, false, opts);
+  await handleFile(filePath, workspace, [], undefined, false, opts);
   log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
 }
 
 export async function handleScriptMetadata(
   path: string,
-  workspace: string,
+  workspace: Workspace,
   alreadySynced: string[],
   message: string | undefined,
   lockfileUseArray: boolean
@@ -94,75 +83,9 @@ export async function handleScriptMetadata(
   }
 }
 
-async function parseMetadataFile(
-  scriptPath: string,
-  generateMetadataIfMissing: (GlobalOptions & { path: string }) | undefined
-): Promise<{ isJson: boolean; payload: any } | undefined> {
-  let metadataFilePath = scriptPath + ".script.json";
-  try {
-    await Deno.stat(metadataFilePath);
-    return {
-      payload: JSON.parse(await Deno.readTextFile(metadataFilePath)),
-      isJson: true,
-    };
-  } catch {
-    try {
-      metadataFilePath = scriptPath + ".script.yaml";
-      await Deno.stat(metadataFilePath);
-      const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath));
-      if (Array.isArray(payload?.["lock"])) {
-        payload["lock"] = payload["lock"].join("\n");
-      }
-      return {
-        payload,
-        isJson: false,
-      };
-    } catch {
-      // no metadata file at all. Create it
-      log.info(
-        colors.blue(`Creating script metadata file for ${metadataFilePath}`)
-      );
-      metadataFilePath = scriptPath + ".script.yaml";
-      let scriptInitialMetadata = defaultScriptMetadata();
-      const scriptInitialMetadataYaml = yamlStringify(
-        scriptInitialMetadata as Record<string, any>
-      );
-      Deno.writeTextFile(metadataFilePath, scriptInitialMetadataYaml, {
-        createNew: true,
-      });
-
-      if (generateMetadataIfMissing) {
-        log.info(
-          colors.blue(`Generating lockfile and schema for ${metadataFilePath}`)
-        );
-        try {
-          await generateMetadata(
-            generateMetadataIfMissing,
-            generateMetadataIfMissing.path
-          );
-
-          scriptInitialMetadata = yamlParse(
-            await Deno.readTextFile(metadataFilePath)
-          ) as ScriptMetadata;
-        } catch (e) {
-          log.info(
-            colors.yellow(
-              `Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
-            )
-          );
-        }
-      }
-      return {
-        payload: scriptInitialMetadata,
-        isJson: false,
-      };
-    }
-  }
-}
-
 export async function handleFile(
   path: string,
-  workspace: string,
+  workspace: Workspace,
   alreadySynced: string[],
   message: string | undefined,
   lockfileUseArray: boolean,
@@ -188,14 +111,19 @@ export async function handleFile(
       .substring(0, path.indexOf("."))
       .replaceAll("\\", "/");
     const typed = (
-      await parseMetadataFile(remotePath, opts ? { ...opts, path } : undefined)
+      await parseMetadataFile(
+        remotePath,
+        opts ? { ...opts, path, workspaceRemote: workspace } : undefined
+      )
     )?.payload;
     const language = inferContentTypeFromFilePath(path);
 
+    const workspaceId = workspace.workspaceId;
+
     let remote = undefined;
     try {
       remote = await ScriptService.getScriptByPath({
-        workspace,
+        workspace: workspaceId,
         path: remotePath.replaceAll("\\", "/"),
       });
       log.debug(`Script ${remotePath} exists on remote`);
@@ -236,7 +164,7 @@ export async function handleFile(
         colors.yellow.bold(`Creating script with a parent ${remotePath}`)
       );
       await ScriptService.createScript({
-        workspace,
+        workspace: workspaceId,
         requestBody: {
           content,
           description: typed?.description ?? "",
@@ -263,7 +191,7 @@ export async function handleFile(
       );
       // no parent hash
       await ScriptService.createScript({
-        workspace: workspace,
+        workspace: workspaceId,
         requestBody: {
           content,
           description: typed?.description ?? "",
@@ -347,21 +275,6 @@ export async function findContentFile(filePath: string) {
   return validCandidates[0];
 }
 
-type ScriptLanguage =
-  | "python3"
-  | "deno"
-  | "bun"
-  | "nativets"
-  | "go"
-  | "bash"
-  | "powershell"
-  | "postgresql"
-  | "mysql"
-  | "bigquery"
-  | "snowflake"
-  | "mssql"
-  | "graphql";
-
 export function filePathExtensionFromContentType(
   language: ScriptLanguage
 ): string {
@@ -421,246 +334,6 @@ export function removeExtensionToPath(path: string): string {
   throw new Error("Invalid extension: " + path);
 }
 
-export function inferContentTypeFromFilePath(
-  contentPath: string
-): ScriptLanguage {
-  if (contentPath.endsWith(".py")) {
-    return "python3";
-  } else if (contentPath.endsWith("fetch.ts")) {
-    return "nativets";
-  } else if (contentPath.endsWith("bun.ts")) {
-    return "bun";
-  } else if (contentPath.endsWith(".ts")) {
-    return "deno";
-  } else if (contentPath.endsWith(".go")) {
-    return "go";
-  } else if (contentPath.endsWith(".my.sql")) {
-    return "mysql";
-  } else if (contentPath.endsWith(".bq.sql")) {
-    return "bigquery";
-  } else if (contentPath.endsWith(".sf.sql")) {
-    return "snowflake";
-  } else if (contentPath.endsWith(".ms.sql")) {
-    return "mssql";
-  } else if (contentPath.endsWith(".pg.sql")) {
-    return "postgresql";
-  } else if (contentPath.endsWith(".gql")) {
-    return "graphql";
-  } else if (contentPath.endsWith(".sh")) {
-    return "bash";
-  } else if (contentPath.endsWith(".ps1")) {
-    return "powershell";
-  } else {
-    throw new Error(
-      "Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))
-    );
-  }
-}
-
-////////////////////////////////////////////////////////////////////////////////////////////
-// below functions copied from Windmill's FE inferArgs function. TODO: refactor           //
-////////////////////////////////////////////////////////////////////////////////////////////
-export function inferSchema(
-  language: ScriptLanguage,
-  content: string,
-  currentSchema: any,
-  path: string
-) {
-  let inferedSchema: any;
-  if (language === "python3") {
-    inferedSchema = JSON.parse(parse_python(content));
-  } else if (language === "nativets") {
-    inferedSchema = JSON.parse(parse_deno(content));
-  } else if (language === "bun") {
-    inferedSchema = JSON.parse(parse_deno(content));
-  } else if (language === "deno") {
-    inferedSchema = JSON.parse(parse_deno(content));
-  } else if (language === "go") {
-    inferedSchema = JSON.parse(parse_go(content));
-  } else if (language === "mysql") {
-    inferedSchema = JSON.parse(parse_mysql(content));
-    inferedSchema.args = [
-      { name: "database", typ: { resource: "mysql" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "bigquery") {
-    inferedSchema = JSON.parse(parse_bigquery(content));
-    inferedSchema.args = [
-      { name: "database", typ: { resource: "bigquery" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "snowflake") {
-    inferedSchema = JSON.parse(parse_snowflake(content));
-    inferedSchema.args = [
-      { name: "database", typ: { resource: "snowflake" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "mssql") {
-    inferedSchema = JSON.parse(parse_mssql(content));
-    inferedSchema.args = [
-      { name: "database", typ: { resource: "ms_sql_server" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "postgresql") {
-    inferedSchema = JSON.parse(parse_sql(content));
-    inferedSchema.args = [
-      { name: "database", typ: { resource: "postgresql" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "graphql") {
-    inferedSchema = JSON.parse(parse_graphql(content));
-    inferedSchema.args = [
-      { name: "api", typ: { resource: "graphql" } },
-      ...inferedSchema.args,
-    ];
-  } else if (language === "bash") {
-    inferedSchema = JSON.parse(parse_bash(content));
-  } else if (language === "powershell") {
-    inferedSchema = JSON.parse(parse_powershell(content));
-  } else {
-    throw new Error("Invalid language: " + language);
-  }
-  if (inferedSchema.type == "Invalid") {
-    log.info(
-      colors.yellow(
-        `Script ${path} invalid, it cannot be parsed to infer schema.`
-      )
-    );
-    return defaultScriptMetadata().schema;
-  }
-
-  currentSchema.required = [];
-  const oldProperties = JSON.parse(JSON.stringify(currentSchema.properties));
-  currentSchema.properties = {};
-
-  for (const arg of inferedSchema.args) {
-    if (!(arg.name in oldProperties)) {
-      currentSchema.properties[arg.name] = { description: "", type: "" };
-    } else {
-      currentSchema.properties[arg.name] = oldProperties[arg.name];
-    }
-    currentSchema.properties[arg.name] = sortObject(
-      currentSchema.properties[arg.name]
-    );
-
-    argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
-
-    currentSchema.properties[arg.name].default = arg.default;
-
-    if (!arg.has_default && !currentSchema.required.includes(arg.name)) {
-      currentSchema.required.push(arg.name);
-    }
-  }
-
-  return currentSchema;
-}
-
-function sortObject(obj: any): any {
-  return Object.keys(obj)
-    .sort()
-    .reduce(
-      (acc, key) => ({
-        ...acc,
-        [key]: obj[key],
-      }),
-      {}
-    );
-}
-
-function argSigToJsonSchemaType(
-  typ:
-    | string
-    | { resource: string | null }
-    | {
-        list:
-          | string
-          | { str: any }
-          | { object: { key: string; typ: any }[] }
-          | null;
-      }
-    | { str: string[] | null }
-    | { object: { key: string; typ: any }[] },
-  oldS: SchemaProperty
-): void {
-  const newS: SchemaProperty = { type: "" };
-  if (typ === "int") {
-    newS.type = "integer";
-  } else if (typ === "float") {
-    newS.type = "number";
-  } else if (typ === "bool") {
-    newS.type = "boolean";
-  } else if (typ === "email") {
-    newS.type = "string";
-    newS.format = "email";
-  } else if (typ === "sql") {
-    newS.type = "string";
-    newS.format = "sql";
-  } else if (typ === "yaml") {
-    newS.type = "string";
-    newS.format = "yaml";
-  } else if (typ === "bytes") {
-    newS.type = "string";
-    newS.contentEncoding = "base64";
-  } else if (typ === "datetime") {
-    newS.type = "string";
-    newS.format = "date-time";
-  } else if (typeof typ !== "string" && `object` in typ) {
-    newS.type = "object";
-    if (typ.object) {
-      const properties: Record<string, SchemaProperty> = {};
-      for (const prop of typ.object) {
-        properties[prop.key] = { type: undefined };
-        argSigToJsonSchemaType(prop.typ, properties[prop.key]);
-      }
-      newS.properties = properties;
-    }
-  } else if (typeof typ !== "string" && `str` in typ) {
-    newS.type = "string";
-    if (typ.str) {
-      newS.enum = typ.str;
-    }
-  } else if (typeof typ !== "string" && `resource` in typ) {
-    newS.type = "object";
-    newS.format = `resource-${typ.resource}`;
-  } else if (typeof typ !== "string" && `list` in typ) {
-    newS.type = "array";
-    if (typ.list === "int" || typ.list === "float") {
-      newS.items = { type: "number" };
-    } else if (typ.list === "bytes") {
-      newS.items = { type: "string", contentEncoding: "base64" };
-    } else if (typ.list == "string") {
-      newS.items = { type: "string" };
-    } else if (typ.list && typeof typ.list == "object" && "str" in typ.list) {
-      newS.items = { type: "string", enum: typ.list.str };
-    } else {
-      newS.items = { type: "object" };
-    }
-  } else {
-    newS.type = "object";
-  }
-
-  if (oldS.type != newS.type) {
-    for (const prop of Object.getOwnPropertyNames(newS)) {
-      if (prop != "description") {
-        // @ts-ignore: fix
-        delete oldS[prop];
-      }
-    }
-  } else if (oldS.format == "date-time" && newS.format != "date-time") {
-    delete oldS.format;
-  } else if (oldS.items?.type != newS.items?.type) {
-    delete oldS.items;
-  }
-
-  Object.assign(oldS, newS);
-  if (oldS.format?.startsWith("resource-") && newS.type != "object") {
-    oldS.format = undefined;
-  }
-}
-////////////////////////////////////////////////////////////////////////////////////////////
-// end of refactoring TODO                                                                //
-////////////////////////////////////////////////////////////////////////////////////////////
-
 async function list(opts: GlobalOptions & { showArchived?: boolean }) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
@@ -898,114 +571,7 @@ async function generateMetadata(
   await requireLogin(opts);
 
   // read script metadata file
-  const remotePath = scriptPath
-    .substring(0, scriptPath.indexOf("."))
-    .replaceAll("\\", "/");
-  const metadataWithType = await parseMetadataFile(remotePath, undefined);
-  if (metadataWithType === undefined) {
-    throw new Error("Script metadata file does not exist at this path");
-  }
-
-  // read script content
-  const scriptContent = await Deno.readTextFile(scriptPath);
-
-  const metadataParsedContent = metadataWithType?.payload as Record<
-    string,
-    any
-  >;
-  
-  const language = inferContentTypeFromFilePath(scriptPath);
-  if (!opts.lockOnly) {
-    await updateScriptSchema(
-      scriptContent,
-      language,
-      metadataParsedContent,
-      scriptPath
-    );
-  }
-
-  if (!opts.schemaOnly) {
-    await updateScriptLock(
-      workspace,
-      scriptContent,
-      language,
-      remotePath,
-      metadataParsedContent
-    );
-  }
-
-  let metaPath = remotePath + ".script.yaml";
-  let newMetadataContent = yamlStringify(metadataParsedContent);
-  if (metadataWithType.isJson) {
-    metaPath = remotePath + ".script.json";
-    newMetadataContent = JSON.stringify(metadataParsedContent);
-  }
-  await Deno.writeTextFile(metaPath, newMetadataContent);
-}
-
-async function updateScriptSchema(
-  scriptContent: string,
-  language: ScriptLanguage,
-  metadataContent: Record<string, any>,
-  path: string
-): Promise<void> {
-  // infer schema from script content and update it inplace
-  await instantiateWasm();
-  const newSchema = inferSchema(
-    language,
-    scriptContent,
-    metadataContent.schema,
-    path
-  );
-  metadataContent.schema = newSchema;
-}
-
-async function updateScriptLock(
-  workspace: Workspace,
-  scriptContent: string,
-  language: ScriptLanguage,
-  remotePath: string,
-  metadataContent: Record<string, any>
-): Promise<void> {
-  // generate the script lock running a dependency job in Windmill and update it inplace
-  // TODO: update this once the client is released
-  const rawResponse = await fetch(
-    `${workspace.remote}api/w/${workspace.workspaceId}/jobs/run/dependencies`,
-    {
-      method: "POST",
-      headers: {
-        Cookie: `token=${workspace.token}`,
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        raw_scripts: [
-          {
-            raw_code: scriptContent,
-            language: language,
-            script_path: remotePath,
-          },
-        ],
-        entrypoint: remotePath,
-      }),
-    }
-  );
-
-  try {
-    const response = await rawResponse.json();
-    const lock = response.lock;
-    if (lock === undefined) {
-      throw new Error(
-        `Failed to generate lockfile. Full response was: ${JSON.stringify(
-          response
-        )}`
-      );
-    }
-    metadataContent.lock = lock;
-  } catch {
-    throw new Error(
-      `Failed to generate lockfile. Status was: ${rawResponse.statusText}`
-    );
-  }
+  await generateMetadataInternal(scriptPath, workspace, opts);
 }
 
 const command = new Command()
@@ -1044,6 +610,14 @@ const command = new Command()
   .arguments("<path_to_script_file:string>")
   .option("--lock-only", "re-generate only the lock")
   .option("--schema-only", "re-generate only script schema")
+  .action(generateMetadata as any)
+  .command(
+    "update-all-locks",
+    "re-generate the metadata file updating the lock and the script schema"
+  )
+  .arguments("<path_to_script_file:string>")
+  .option("--lock-only", "re-generate only the lock")
+  .option("--schema-only", "re-generate only script schema")
   .action(generateMetadata as any);
 
 export default command;

cli/script_common.ts

@@ -0,0 +1,50 @@
+export type ScriptLanguage =
+  | "python3"
+  | "deno"
+  | "bun"
+  | "nativets"
+  | "go"
+  | "bash"
+  | "powershell"
+  | "postgresql"
+  | "mysql"
+  | "bigquery"
+  | "snowflake"
+  | "mssql"
+  | "graphql";
+
+export function inferContentTypeFromFilePath(
+  contentPath: string
+): ScriptLanguage {
+  if (contentPath.endsWith(".py")) {
+    return "python3";
+  } else if (contentPath.endsWith("fetch.ts")) {
+    return "nativets";
+  } else if (contentPath.endsWith("bun.ts")) {
+    return "bun";
+  } else if (contentPath.endsWith(".ts")) {
+    return "deno";
+  } else if (contentPath.endsWith(".go")) {
+    return "go";
+  } else if (contentPath.endsWith(".my.sql")) {
+    return "mysql";
+  } else if (contentPath.endsWith(".bq.sql")) {
+    return "bigquery";
+  } else if (contentPath.endsWith(".sf.sql")) {
+    return "snowflake";
+  } else if (contentPath.endsWith(".ms.sql")) {
+    return "mssql";
+  } else if (contentPath.endsWith(".pg.sql")) {
+    return "postgresql";
+  } else if (contentPath.endsWith(".gql")) {
+    return "graphql";
+  } else if (contentPath.endsWith(".sh")) {
+    return "bash";
+  } else if (contentPath.endsWith(".ps1")) {
+    return "powershell";
+  } else {
+    throw new Error(
+      "Invalid language: " + contentPath.substring(contentPath.lastIndexOf("."))
+    );
+  }
+}

cli/sync.ts

@@ -4,7 +4,7 @@ import {
   Command,
   Confirm,
   ensureDir,
-  ignore,
+  minimatch,
   JSZip,
   path,
   ScriptService,
@@ -21,6 +21,7 @@ import {
   yamlParse,
   ScheduleService,
   SEP,
+  gitignore_parser,
 } from "./deps.ts";
 import {
   getTypeStrFromPath,
@@ -36,6 +37,8 @@ import { handleScriptMetadata, removeExtensionToPath } from "./script.ts";
 
 import { handleFile } from "./script.ts";
 import { deepEqual } from "./utils.ts";
+import { read } from "https://deno.land/x/cbor@v1.4.1/decode.js";
+import { readConfigFile } from "./conf.ts";
 
 type DynFSElement = {
   isDirectory: boolean;
@@ -437,28 +440,59 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
 export const isWhitelisted = (p: string) => {
   return p == "." + SEP || p == "" || p == "u" || p == "f" || p == "g";
 };
-export async function ignoreF() {
+
+export async function ignoreF(): Promise<
+  (p: string, isDirectory: boolean) => boolean
+> {
+  const wmillconf = await readConfigFile();
+
+  let whitelist: { approve(file: string): boolean } | undefined = undefined;
+
+  if (wmillconf?.includes) {
+    if (!Array.isArray(wmillconf.includes)) {
+      throw new Error("wmill.yaml/includes must be an array");
+    }
+    if (wmillconf.includes.length > 0) {
+      whitelist = {
+        approve(file: string): boolean {
+          return wmillconf.includes!.some((i) => minimatch(file, i));
+        },
+      };
+    }
+  }
+  let ign:
+    | {
+        denies(file: string): boolean;
+      }
+    | undefined = undefined;
   try {
     const ignoreContent = await Deno.readTextFile(".wmillignore");
     const condensed = ignoreContent
       .split("\n")
       .filter((l) => l != "" && !l.startsWith("#"))
       .join(", ");
-    log.info(colors.gray(`Using .wmillignore file (${condensed})`));
-    const ign = ignore.default().add(ignoreContent);
-    // new Gitignore.default({ initialRules: ignoreContent.split("\n")}).ignoreContent).compile();
-    return (p: string, isDirectory: boolean) => {
-      p = path.relative(Deno.cwd(), p);
+    log.info(
+      colors.gray(
+        `(Deprecated, use wmill.conf/includes instead) Using .wmillignore file (${condensed})`
+      )
+    );
+    ign = gitignore_parser.compile(ignoreContent);
+  } catch {}
 
-      return (
-        !isWhitelisted(p) &&
-        (isNotWmillFile(p, isDirectory) || (!isDirectory && ign.ignores(p)))
-      );
-    };
-  } catch {
-    return (p: string, isDirectory: boolean) =>
-      !isWhitelisted(p) && isNotWmillFile(p, isDirectory);
+  if (ign && whitelist) {
+    throw new Error("Cannot have both .wmillignore and wmill.yaml/includes");
   }
+
+  // new Gitignore.default({ initialRules: ignoreContent.split("\n")}).ignoreContent).compile();
+  return (p: string, isDirectory: boolean) => {
+    return (
+      !isWhitelisted(p) &&
+      (isNotWmillFile(p, isDirectory) ||
+        (!isDirectory &&
+          ((whitelist != undefined && !whitelist.approve(p)) ||
+            (ign != undefined && ign.denies(p)))))
+    );
+  };
 }
 
 async function pull(
@@ -770,7 +804,7 @@ async function push(
         if (
           await handleScriptMetadata(
             change.path,
-            workspace.workspaceId,
+            workspace,
             alreadySynced,
             opts.message,
             lockfileUseArray
@@ -783,7 +817,7 @@ async function push(
         } else if (
           await handleFile(
             change.path,
-            workspace.workspaceId,
+            workspace,
             alreadySynced,
             opts.message,
             lockfileUseArray,
@@ -823,7 +857,7 @@ async function push(
         } else if (
           await handleFile(
             change.path,
-            workspace.workspaceId,
+            workspace,
             alreadySynced,
             opts.message,
             lockfileUseArray,