feat: auto-strip UTF-8 BOM when reading local files in CLI (#8911)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

cli/src/commands/app/app_metadata.ts

@@ -1,5 +1,5 @@
 import path from "node:path";
-import { readFile, mkdir, readdir } from "node:fs/promises";
+import { mkdir, readdir } from "node:fs/promises";
 import { colors } from "@cliffy/ansi/colors";
 import * as log from "../../core/log.ts";
 import { sep as SEP } from "node:path";
@@ -23,7 +23,7 @@ import {
   ScriptLanguage,
   workspaceDependenciesLanguages,
 } from "../../utils/script_common.ts";
-import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
+import { generateHash, getHeaders, readTextFile, writeIfChanged } from "../../utils/utils.ts";
 import { exts } from "../script/script.ts";
 import { FSFSElement, yamlOptions } from "../sync/sync.ts";
 import { Workspace } from "../workspace/workspace.ts";
@@ -178,7 +178,7 @@ export async function generateAppLocksInternal(
         if (typeof content === "string" && content.startsWith("!inline ")) {
           const filePath = appFolder + SEP + content.replace("!inline ", "");
           try {
-            content = await readFile(filePath, "utf-8");
+            content = await readTextFile(filePath);
           } catch {
             return inlineScript;
           }
@@ -893,7 +893,7 @@ export async function inferRunnableSchemaFromFile(
   );
   let content: string;
   try {
-    content = await readFile(fullFilePath, "utf-8");
+    content = await readTextFile(fullFilePath);
   } catch {
     log.warn(colors.yellow(`Could not read file: ${fullFilePath}`));
     return undefined;

cli/src/commands/app/bundle.ts

@@ -5,6 +5,7 @@ import { spawn } from "node:child_process";
 import * as log from "../../core/log.ts";
 import { colors } from "@cliffy/ansi/colors";
 import * as windmillUtils from "@windmill-labs/shared-utils";
+import { readTextFile, readTextFileSync } from "../../utils/utils.ts";
 export interface BundleOptions {
   entryPoint?: string;
   outDir?: string;
@@ -41,7 +42,7 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea
   }
 
   try {
-    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8"));
+    const packageJson = JSON.parse(readTextFileSync(packageJsonPath));
     const allDeps = {
       ...packageJson.dependencies,
       ...packageJson.devDependencies,
@@ -69,7 +70,7 @@ function createSveltePlugin(appDir: string): any {
         const svelte = await import("svelte/compiler");
 
         // Load the file from the file system
-        const source = await fs.promises.readFile(args.path, "utf8");
+        const source = await readTextFile(args.path);
         const filename = path.relative(process.cwd(), args.path);
 
         // This converts a message in Svelte's format to esbuild's format
@@ -269,9 +270,9 @@ export async function createBundle(
       throw new Error(`Expected JS bundle at ${jsPath} but file not found`);
     }
 
-    const jsContent = fs.readFileSync(jsPath, "utf-8");
+    const jsContent = readTextFileSync(jsPath);
     const cssContent = fs.existsSync(cssPath)
-      ? fs.readFileSync(cssPath, "utf-8")
+      ? readTextFileSync(cssPath)
       : "";
 
     try {

cli/src/commands/app/dev.ts

@@ -13,7 +13,7 @@ import * as path from "node:path";
 import process from "node:process";
 import { Buffer } from "node:buffer";
 import { writeFileSync } from "node:fs";
-import { readFile } from "node:fs/promises";
+import { readTextFile } from "../../utils/utils.ts";
 import { WebSocket, WebSocketServer } from "ws";
 import {
   createFrameworkPlugins,
@@ -800,7 +800,7 @@ async function dev(opts: DevOptions, appFolder?: string) {
     const fileName = path.basename(filePath);
 
     try {
-      const sqlContent = await readFile(filePath, "utf-8");
+      const sqlContent = await readTextFile(filePath);
 
       if (!sqlContent.trim()) {
         log.info(colors.gray(`Skipping empty file: ${fileName}`));
@@ -856,7 +856,7 @@ async function dev(opts: DevOptions, appFolder?: string) {
     // If there's a current SQL file being shown, send it to the new client
     if (currentSqlFile && fs.existsSync(currentSqlFile)) {
       try {
-        const sqlContent = await readFile(currentSqlFile, "utf-8");
+        const sqlContent = await readTextFile(currentSqlFile);
         const datatable = await getDatatableConfig();
         const fileName = path.basename(currentSqlFile);
 

cli/src/commands/app/raw_apps.ts

@@ -9,10 +9,10 @@ import { stringify as yamlStringify } from "yaml";
 import * as wmill from "../../../gen/services.gen.ts";
 import { Policy } from "../../../gen/types.gen.ts";
 import path from "node:path";
-import { readFile, readdir } from "node:fs/promises";
+import { readdir } from "node:fs/promises";
 
 import { GlobalOptions, isSuperset } from "../../types.ts";
-import { deepEqual } from "../../utils/utils.ts";
+import { deepEqual, readTextFile } from "../../utils/utils.ts";
 
 import { replaceInlineScripts, repopulateFields } from "./app.ts";
 import { createBundle, detectFrameworks } from "./bundle.ts";
@@ -65,8 +65,8 @@ async function findRunnableContentFile(
     // Check if this is a recognized extension
     if (EXTENSION_TO_LANGUAGE[ext]) {
       try {
-        const content = await readFile(
-          path.join(backendPath, fileName), "utf-8",
+        const content = await readTextFile(
+          path.join(backendPath, fileName),
         );
         return { ext, content };
       } catch {
@@ -164,9 +164,8 @@ export async function loadRunnablesFromBackend(
           // Try to load lock file
           let lock: string | undefined;
           try {
-            lock = await readFile(
+            lock = await readTextFile(
               path.join(backendPath, `${runnableId}.lock`),
-              "utf-8",
             );
           } catch {
             // No lock file, that's fine
@@ -226,8 +225,8 @@ export async function loadRunnablesFromBackend(
         // Try to load lock file
         let lock: string | undefined;
         try {
-          lock = await readFile(
-            path.join(backendPath, `${runnableId}.lock`), "utf-8",
+          lock = await readTextFile(
+            path.join(backendPath, `${runnableId}.lock`),
           );
         } catch {
           // No lock file, that's fine
@@ -319,7 +318,7 @@ async function collectAppFiles(
         ) {
           continue;
         }
-        const content = await readFile(fullPath, "utf-8");
+        const content = await readTextFile(fullPath);
         files[relativePath] = content;
       }
     }

cli/src/commands/dependencies/dependencies.ts

@@ -7,6 +7,7 @@ import * as log from "../../core/log.ts";
 import * as wmill from "../../../gen/services.gen.ts";
 import fs from "node:fs";
 import { workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.ts";
+import { readTextFileSync } from "../../utils/utils.ts";
 
 async function push(
   opts: GlobalOptions,
@@ -19,7 +20,7 @@ async function push(
     throw new Error(`File not found: ${filePath}`);
   }
 
-  const content = fs.readFileSync(filePath, "utf8");
+  const content = readTextFileSync(filePath);
 
   // Use the existing pushWorkspaceDependencies function
   await pushWorkspaceDependencies(

cli/src/commands/dev/dev.ts

@@ -7,7 +7,8 @@ import { WebSocket, WebSocketServer } from "ws";
 import * as getPort from "get-port";
 import * as http from "node:http";
 import * as open from "open";
-import { readFile, realpath } from "node:fs/promises";
+import { realpath } from "node:fs/promises";
+import { readTextFile } from "../../utils/utils.ts";
 import { watch } from "node:fs";
 import { getTypeStrFromPath, GlobalOptions } from "../../types.ts";
 import { ignoreF } from "../sync/sync.ts";
@@ -93,7 +94,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
         )) as FlowFile;
         await replaceInlineScripts(
           localFlow.value.modules,
-          async (path: string) => await readFile(localPath + path, "utf-8"),
+          async (path: string) => await readTextFile(localPath + path),
           log,
           localPath,
           SEP,
@@ -114,7 +115,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
         log.info("Updated " + localPath);
         broadcastChanges(currentLastEdit);
       } else if (typ == "script") {
-        const content = await readFile(cpath, "utf-8");
+        const content = await readTextFile(cpath);
         const splitted = cpath.split(".");
         const wmPath = splitted[0];
         const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs);

cli/src/commands/flow/flow.ts

@@ -7,9 +7,8 @@ import * as log from "../../core/log.ts";
 import { sep as SEP } from "node:path";
 import { stringify as yamlStringify } from "yaml";
 import { yamlParseFile } from "../../utils/yaml.ts";
-import { validateRequiredArgs } from "../../utils/utils.ts";
+import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts";
 import * as wmill from "../../../gen/services.gen.ts";
-import { readFile } from "node:fs/promises";
 import { mkdirSync, writeFileSync } from "node:fs";
 import { buildFolderPath, getMetadataFileName, loadNonDottedPathsSetting } from "../../utils/resource_folders.ts";
 
@@ -157,7 +156,7 @@ export async function pushFlow(
   }
   const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;
 
-  const fileReader = async (path: string) => await readFile(localPath + path, "utf-8");
+  const fileReader = async (path: string) => await readTextFile(localPath + path);
   const missingFiles: string[] = [];
   await replaceInlineScripts(
     localFlow.value.modules,
@@ -545,7 +544,7 @@ async function preview(
   const localFlow = (await yamlParseFile(flowPath + "flow.yaml")) as FlowFile;
 
   // Replace inline scripts with their actual content
-  const fileReader = async (path: string) => await readFile(flowPath + path, "utf-8");
+  const fileReader = async (path: string) => await readTextFile(flowPath + path);
   await replaceInlineScripts(
     localFlow.value.modules,
     fileReader,

cli/src/commands/flow/flow_metadata.ts

@@ -4,7 +4,6 @@ import * as path from "node:path";
 import { sep as SEP } from "node:path";
 import { stringify as yamlStringify } from "yaml";
 import { yamlParseFile } from "../../utils/yaml.ts";
-import { readFile } from "node:fs/promises";
 import { GlobalOptions } from "../../types.ts";
 import {
   readLockfile,
@@ -21,7 +20,7 @@ import { ScriptLanguage } from "../../utils/script_common.ts";
 import { extractInlineScripts as extractInlineScriptsForFlows, extractCurrentMapping } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
 import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
 
-import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
+import { generateHash, getHeaders, readTextFile, writeIfChanged } from "../../utils/utils.ts";
 import { exts } from "../script/script.ts";
 import { FSFSElement, yamlOptions } from "../sync/sync.ts";
 import { Workspace } from "../workspace/workspace.ts";
@@ -109,7 +108,7 @@ export async function generateFlowLockInternal(
         if (content.startsWith("!inline ")) {
           const filePath = folder + SEP + content.replace("!inline ", "");
           try {
-            content = await readFile(filePath, "utf-8");
+            content = await readTextFile(filePath);
           } catch {
             continue;
           }
@@ -192,7 +191,7 @@ export async function generateFlowLockInternal(
     if (!noStaleMessage) {
       log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
     }
-    const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8");
+    const fileReader = async (path: string) => await readTextFile(folder + SEP + path);
 
     // Capture existing module-ID-to-file-path mapping before replaceInlineScripts
     // overwrites the !inline references with actual file content. This preserves

cli/src/commands/instance/instance.ts

@@ -1,4 +1,4 @@
-import { readFile, writeFile, readdir, mkdir, rm, stat } from "node:fs/promises";
+import { writeFile, readdir, mkdir, rm, stat } from "node:fs/promises";
 import { appendFile } from "node:fs/promises";
 import { colors } from "@cliffy/ansi/colors";
 import { Command } from "@cliffy/command";
@@ -39,7 +39,7 @@ import {
   pushInstanceSettings,
   type SimplifiedSettings,
 } from "../../core/settings.ts";
-import { deepEqual } from "../../utils/utils.ts";
+import { deepEqual, readTextFile } from "../../utils/utils.ts";
 import { getActiveWorkspace } from "../workspace/workspace.ts";
 
 export interface Instance {
@@ -52,7 +52,7 @@ export interface Instance {
 export async function allInstances(): Promise<Instance[]> {
   try {
     const file = await getInstancesConfigFilePath();
-    const txt = await readFile(file, "utf-8");
+    const txt = await readTextFile(file);
     return txt
       .split("\n")
       .map((line) => {
@@ -658,7 +658,7 @@ export async function getActiveInstance(opts: {
     return opts.instance;
   }
   try {
-    return await readFile(await getActiveInstanceFilePath(), "utf-8");
+    return await readTextFile(await getActiveInstanceFilePath());
   } catch {
     return undefined;
   }

cli/src/commands/jobs/jobs.ts

@@ -7,6 +7,7 @@ import { Confirm } from "@cliffy/prompt/confirm";
 import * as log from "../../core/log.ts";
 import { mergeConfigWithConfigFile } from "../../core/conf.ts";
 import * as fs from "node:fs/promises";
+import { readTextFile } from "../../utils/utils.ts";
 import * as wmill from "../../../gen/services.gen.ts";
 
 async function pullJobs(
@@ -190,7 +191,7 @@ async function pushJobs(
   // Push completed jobs
   const completedPath = opts.completedFile || "completed_jobs.json";
   try {
-    const completedContent = await fs.readFile(completedPath, "utf-8");
+    const completedContent = await readTextFile(completedPath);
     const completedJobs = JSON.parse(completedContent);
 
     if (!Array.isArray(completedJobs)) {
@@ -218,7 +219,7 @@ async function pushJobs(
   // Push queued jobs
   const queuedPath = opts.queuedFile || "queued_jobs.json";
   try {
-    const queuedContent = await fs.readFile(queuedPath, "utf-8");
+    const queuedContent = await readTextFile(queuedPath);
     const queuedJobs = JSON.parse(queuedContent);
 
     if (!Array.isArray(queuedJobs)) {

cli/src/commands/resource/resource.ts

@@ -1,4 +1,4 @@
-import { mkdir, stat, writeFile, readdir, readFile } from "node:fs/promises";
+import { mkdir, stat, writeFile, readdir } from "node:fs/promises";
 import { stringify as yamlStringify } from "yaml";
 import nodePath from "node:path";
 
@@ -17,7 +17,7 @@ import * as log from "../../core/log.ts";
 import { sep as SEP } from "node:path";
 import * as wmill from "../../../gen/services.gen.ts";
 import { Resource } from "../../../gen/types.gen.ts";
-import { readInlinePathSync } from "../../utils/utils.ts";
+import { readInlinePathSync, readTextFile } from "../../utils/utils.ts";
 import { isWorkspaceSpecificFile } from "../../core/specific_items.ts";
 import { getCurrentGitBranch } from "../../utils/git.ts";
 
@@ -38,7 +38,7 @@ async function readFilesetDirectory(dirPath: string): Promise<Record<string, str
       if (entry.isDirectory()) {
         await walk(entryPath, relPath);
       } else if (entry.isFile()) {
-        result[relPath] = await readFile(entryPath, "utf-8");
+        result[relPath] = await readTextFile(entryPath);
       }
     }
   }

cli/src/commands/script/script.ts

@@ -2,7 +2,7 @@ import { GlobalOptions } from "../../types.ts";
 import { requireLogin } from "../../core/auth.ts";
 import { resolveWorkspace, validatePath } from "../../core/context.ts";
 import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
-import { readFile, writeFile, stat, mkdir } from "node:fs/promises";
+import { writeFile, stat, mkdir } from "node:fs/promises";
 import { Buffer } from "node:buffer";
 import { colors } from "@cliffy/ansi/colors";
 import { Command } from "@cliffy/command";
@@ -12,7 +12,7 @@ import * as log from "../../core/log.ts";
 import { sep as SEP } from "node:path";
 import * as path from "node:path";
 import { stringify as yamlStringify } from "yaml";
-import { deepEqual } from "../../utils/utils.ts";
+import { deepEqual, readTextFile, readTextFileSync } from "../../utils/utils.ts";
 import * as wmill from "../../../gen/services.gen.ts";
 import * as specificItems from "../../core/specific_items.ts";
 import { getCurrentGitBranch } from "../../utils/git.ts";
@@ -114,7 +114,7 @@ export async function computePushMetadataHash(
 ): Promise<string> {
   const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/");
   const metadataWithType = await parseMetadataFile(remotePath, undefined);
-  const metadataContent = await readFile(metadataWithType.path, "utf-8");
+  const metadataContent = await readTextFile(metadataWithType.path);
   return await generateScriptHash({}, content, metadataContent);
 }
 
@@ -141,7 +141,7 @@ async function push(opts: PushOptions, filePath: string) {
 
   // Warn about metadata state before pushing
   try {
-    const content = await readFile(filePath, "utf-8");
+    const content = await readTextFile(filePath);
     const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/");
     const contentHash = await computePushMetadataHash(filePath, content);
     const conf = await readLockfile();
@@ -432,7 +432,7 @@ export async function handleFile(
     } catch {
       log.debug(`Script ${remotePath} does not exist on remote`);
     }
-    const content = await readFile(path, "utf-8");
+    const content = await readTextFile(path);
 
     if (opts?.skipScriptsMetadata) {
       // if (codebase) {
@@ -619,7 +619,7 @@ export async function readModulesFromDisk(
       } else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) {
         // Skip lock files — they're handled as the `lock` field on ScriptModule
         if (exts.some((ext) => entry.name.endsWith(ext))) {
-          const content = fs.readFileSync(fullPath, "utf-8");
+          const content = readTextFileSync(fullPath);
           const language = inferContentTypeFromFilePath(entry.name, defaultTs);
 
           // Check for an accompanying lock file (helper.lock)
@@ -627,7 +627,7 @@ export async function readModulesFromDisk(
           const lockPath = path.join(dirPath, baseName + ".lock");
           let lock: string | undefined;
           if (fs.existsSync(lockPath)) {
-            lock = fs.readFileSync(lockPath, "utf-8");
+            lock = readTextFileSync(lockPath);
           }
 
           modules[relPath] = {
@@ -958,7 +958,7 @@ export async function resolve(input: string): Promise<Record<string, any>> {
     input = new TextDecoder().decode(Buffer.concat(chunks));
   }
   if (input[0] == "@") {
-    input = await readFile(input.substring(1), "utf-8");
+    input = await readTextFile(input.substring(1));
   }
   try {
     return JSON.parse(input);
@@ -1404,7 +1404,7 @@ async function preview(
 
   const codebases = await listSyncCodebases(opts);
   const language = inferContentTypeFromFilePath(filePath, opts?.defaultTs);
-  const content = await readFile(filePath, "utf-8");
+  const content = await readTextFile(filePath);
   const input = opts.data ? await resolve(opts.data) : {};
 
   // Read modules from __mod/ folder if present

cli/src/commands/sync/sync.ts

@@ -1,6 +1,6 @@
 import { requireLogin } from "../../core/auth.ts";
 import { fetchVersion, resolveWorkspace } from "../../core/context.ts";
-import { readFile, writeFile, readdir, stat, rm, copyFile, mkdir } from "node:fs/promises";
+import { writeFile, readdir, stat, rm, copyFile, mkdir } from "node:fs/promises";
 import { colors } from "@cliffy/ansi/colors";
 import { Command } from "@cliffy/command";
 import { Confirm } from "@cliffy/prompt/confirm";
@@ -42,6 +42,7 @@ import {
   isFilesetResource,
   isRawAppFile,
   isWorkspaceDependencies,
+  readTextFile,
 } from "../../utils/utils.ts";
 import {
   getEffectiveSettings,
@@ -325,7 +326,7 @@ export async function FSFSElement(
         }
       },
       async getContentText(): Promise<string> {
-        const content = await readFile(localP, "utf-8");
+        const content = await readTextFile(localP);
         const itemPath = localP.substring(p.length + 1);
         const r = await addCodebaseDigestIfRelevant(
           itemPath,
@@ -2331,7 +2332,7 @@ export async function pull(
       if (change.name === "edited") {
         if (opts.stateful) {
           try {
-            const currentLocal = await readFile(target, "utf-8");
+            const currentLocal = await readTextFile(target);
             if (
               currentLocal !== change.before &&
               currentLocal !== change.after
@@ -3307,7 +3308,7 @@ export async function push(
 
                   const newObj = parseFromPath(
                     resourceFilePath,
-                    await readFile(resourceFilePath, "utf-8"),
+                    await readTextFile(resourceFilePath),
                   );
 
                   // For branch-specific resources, push to the base path on the workspace server
@@ -3342,7 +3343,7 @@ export async function push(
 
                   const newObj = parseFromPath(
                     resourceFilePath,
-                    await readFile(resourceFilePath, "utf-8"),
+                    await readTextFile(resourceFilePath),
                   );
 
                   let serverPath = resourceFilePath;

cli/src/commands/workspace/workspace.ts

@@ -1,4 +1,5 @@
-import { readFile, writeFile, open as fsOpen } from "node:fs/promises";
+import { writeFile, open as fsOpen } from "node:fs/promises";
+import { readTextFile } from "../../utils/utils.ts";
 import process from "node:process";
 import { GlobalOptions } from "../../types.ts";
 import {
@@ -31,7 +32,7 @@ export async function allWorkspaces(
 ): Promise<Workspace[]> {
   try {
     const file = await getWorkspaceConfigFilePath(configDirOverride);
-    const txt = await readFile(file, "utf-8");
+    const txt = await readTextFile(file);
     return txt
       .split("\n")
       .map((line) => {
@@ -55,7 +56,7 @@ async function getActiveWorkspaceName(
   }
   try {
     const file = await getActiveWorkspaceConfigFilePath(opts?.configDir);
-    return await readFile(file, "utf-8");
+    return await readTextFile(file);
   } catch {
     return undefined;
   }

cli/src/core/branch-profiles.ts

@@ -1,5 +1,6 @@
 import * as log from "./log.ts";
-import { readFile, writeFile } from "node:fs/promises";
+import { writeFile } from "node:fs/promises";
+import { readTextFile } from "../utils/utils.ts";
 import { getStore } from "./store.ts";
 
 export interface BranchProfileMapping {
@@ -17,7 +18,7 @@ export async function getBranchProfilesPath(configDirOverride?: string): Promise
 export async function loadBranchProfiles(configDirOverride?: string): Promise<BranchProfileMapping> {
   try {
     const path = await getBranchProfilesPath(configDirOverride);
-    const content = await readFile(path, "utf-8");
+    const content = await readTextFile(path);
     return JSON.parse(content);
   } catch {
     // File doesn't exist or invalid JSON - return empty mapping

cli/src/guidance/writer.ts

@@ -1,4 +1,5 @@
-import { cp, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { cp, mkdir, readdir, stat, writeFile } from "node:fs/promises";
+import { readTextFile } from "../utils/utils.ts";
 import { join } from "node:path";
 import { generateAgentsMdContent } from "./core.ts";
 import {
@@ -47,7 +48,7 @@ export async function writeAiGuidanceFiles(
     overwrite: options.overwriteProjectGuidance ?? false,
     content:
       options.agentsSourcePath != null
-        ? await readFile(options.agentsSourcePath, "utf8")
+        ? await readTextFile(options.agentsSourcePath)
         : generateAgentsMdContent(buildSkillsReference(skillMetadata)),
   });
 
@@ -56,7 +57,7 @@ export async function writeAiGuidanceFiles(
     overwrite: options.overwriteProjectGuidance ?? false,
     content:
       options.claudeSourcePath != null
-        ? await readFile(options.claudeSourcePath, "utf8")
+        ? await readTextFile(options.claudeSourcePath)
         : CLAUDE_MD_DEFAULT,
   });
 
@@ -202,7 +203,7 @@ async function readSkillMetadataFromDirectory(skillsDir: string): Promise<Resolv
       continue;
     }
 
-    const content = await readFile(skillPath, "utf8");
+    const content = await readTextFile(skillPath);
     skills.push(parseSkillMetadata(content, entry.name));
   }
 

cli/src/types.ts

@@ -5,7 +5,6 @@ import * as path from "node:path";
 import { sep as SEP } from "node:path";
 import { stringify as yamlStringify } from "yaml";
 import { yamlParseContent } from "./utils/yaml.ts";
-import { readFileSync } from "node:fs";
 import { pushApp } from "./commands/app/app.ts";
 import { pushFolder } from "./commands/folder/folder.ts";
 import { pushFlow } from "./commands/flow/flow.ts";
@@ -14,7 +13,7 @@ import { pushResourceType } from "./commands/resource-type/resource-type.ts";
 import { pushVariable } from "./commands/variable/variable.ts";
 import { yamlOptions } from "./commands/sync/sync.ts";
 import { showDiffs } from "./core/conf.ts";
-import { deepEqual, isFileResource, isFilesetResource, isWorkspaceDependencies } from "./utils/utils.ts";
+import { deepEqual, isFileResource, isFilesetResource, isWorkspaceDependencies, readTextFileSync } from "./utils/utils.ts";
 import { pushSchedule } from "./commands/schedule/schedule.ts";
 import { pushWorkspaceUser } from "./commands/user/user.ts";
 import { pushGroup } from "./commands/user/user.ts";
@@ -237,9 +236,9 @@ export function parseFromPath(p: string, content: string): any {
 }
 export function parseFromFile(p: string): any {
   if (p.endsWith(".json")) {
-    return JSON.parse(readFileSync(p, "utf-8"));
+    return JSON.parse(readTextFileSync(p));
   } else if (p.endsWith(".yaml") || p.endsWith(".yml")) {
-    return yamlParseContent(p, readFileSync(p, "utf-8"));
+    return yamlParseContent(p, readTextFileSync(p));
   } else {
     throw new Error("Could not read file " + p);
   }

cli/src/utils/local_path_scripts.ts

@@ -1,5 +1,6 @@
 import { execFileSync } from "node:child_process";
-import { readFile, stat } from "node:fs/promises";
+import { stat } from "node:fs/promises";
+import { readTextFile } from "./utils.ts";
 import type { SyncCodebase } from "./codebase.ts";
 import { parseMetadataFileIfExists } from "./metadata.ts";
 import { inferContentTypeFromFilePath } from "./script_common.ts";
@@ -16,7 +17,7 @@ export class UnsupportedLocalPathScriptPreviewError extends Error {
 
 async function readOptionalLock(scriptPath: string): Promise<string | undefined> {
   try {
-    return await readFile(scriptPath + ".script.lock", "utf-8");
+    return await readTextFile(scriptPath + ".script.lock");
   } catch {
     return undefined;
   }
@@ -138,7 +139,7 @@ export async function resolvePreviewLocalScriptState(
 
     return {
       filePath,
-      content: await readFile(filePath, "utf-8"),
+      content: await readTextFile(filePath),
       language,
       lock: normalizeOptionalLock(rawLock),
       tag: metadata?.payload?.tag,

cli/src/utils/metadata.ts

@@ -4,7 +4,7 @@ import { colors } from "@cliffy/ansi/colors";
 import * as log from "../core/log.ts";
 import { stringify as yamlStringify } from "yaml";
 import { yamlParseFile } from "./yaml.ts";
-import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises";
+import { writeFile, stat, rm, readdir } from "node:fs/promises";
 import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
 import * as path from "node:path";
 import { createRequire } from "node:module";
@@ -21,7 +21,7 @@ import {
 import { inferContentTypeFromFilePath } from "./script_common.ts";
 import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts";
 import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
-import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts";
+import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts";
 
 import { SyncCodebase } from "./codebase.ts";
 import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts";
@@ -66,7 +66,7 @@ export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Pro
       if (entry.isDirectory()) continue;
 
       const filePath = `dependencies/${entry.name}`;
-      const content = await readFile(filePath, "utf-8");
+      const content = await readTextFile(filePath);
 
       // Find matching language
       for (const lang of workspaceDependenciesLanguages) {
@@ -153,7 +153,7 @@ export async function filterWorkspaceDependenciesForScripts(
     if (content.startsWith("!inline ")) {
       const filePath = folder + sep + content.replace("!inline ", "");
       try {
-        content = await readFile(filePath, "utf-8");
+        content = await readTextFile(filePath);
       } catch {
         continue;
       }
@@ -212,8 +212,8 @@ export async function generateScriptMetadataInternal(
   );
 
   // read script content
-  const scriptContent = await readFile(scriptPath, "utf-8");
-  const metadataContent = await readFile(metadataWithType.path, "utf-8");
+  const scriptContent = await readTextFile(scriptPath);
+  const metadataContent = await readTextFile(metadataWithType.path);
 
   const filteredRawWorkspaceDependencies = filterWorkspaceDependencies(
     rawWorkspaceDependencies,
@@ -744,7 +744,7 @@ async function updateModuleLocks(
         if (!changedModules.includes(normalizedRelPath)) continue;
       }
 
-      const moduleContent = readFileSync(fullPath, "utf-8");
+      const moduleContent = readTextFileSync(fullPath);
       const moduleRemotePath = scriptRemotePath + "/" + relPath;
 
       log.debug(`Generating lock for module ${relPath}`);
@@ -986,7 +986,7 @@ export async function parseMetadataFileIfExists(
   let metadataFilePath = scriptPath + ".script.json";
   try {
     await stat(metadataFilePath);
-    const payload = JSON.parse(await readFile(metadataFilePath, "utf-8"));
+    const payload = JSON.parse(await readTextFile(metadataFilePath));
     replaceLock(payload);
     return {
       path: metadataFilePath,
@@ -1028,7 +1028,7 @@ export async function parseMetadataFile(
     await stat(metadataFilePath);
     return {
       path: metadataFilePath,
-      payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
+      payload: JSON.parse(await readTextFile(metadataFilePath)),
       isJson: true,
     };
   } catch {
@@ -1051,7 +1051,7 @@ export async function parseMetadataFile(
         await stat(metadataFilePath);
         return {
           path: metadataFilePath,
-          payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
+          payload: JSON.parse(await readTextFile(metadataFilePath)),
           isJson: true,
         };
       } catch {
@@ -1229,7 +1229,7 @@ async function computeModuleHashes(
         } catch {
           continue;
         }
-        const content = readFileSync(fullPath, "utf-8");
+        const content = readTextFileSync(fullPath);
         const normalizedPath = normalizeLockPath(relPath);
         hashes[normalizedPath] = await generateHash(
           content + JSON.stringify(rawWorkspaceDependencies)

cli/src/utils/utils.ts

@@ -131,9 +131,53 @@ export async function generateHashFromBuffer(
   return Buffer.from(hashBuffer).toString("hex");
 }
 
+function decodeBufferAsUtf8(buf: Buffer, path: string | URL): string {
+  if (buf.length >= 2) {
+    if (buf[0] === 0xff && buf[1] === 0xfe) {
+      if (buf.length >= 4 && buf[2] === 0x00 && buf[3] === 0x00) {
+        throw new Error(
+          `File ${path} is encoded as UTF-32 LE, which is not supported. Please convert it to UTF-8.`
+        );
+      }
+      throw new Error(
+        `File ${path} is encoded as UTF-16 LE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+    if (buf[0] === 0xfe && buf[1] === 0xff) {
+      throw new Error(
+        `File ${path} is encoded as UTF-16 BE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+    if (buf.length >= 4 && buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0xfe && buf[3] === 0xff) {
+      throw new Error(
+        `File ${path} is encoded as UTF-32 BE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+  }
+  if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
+    return buf.subarray(3).toString("utf-8");
+  }
+  return buf.toString("utf-8");
+}
+
+export function stripBom(content: string): string {
+  if (content.charCodeAt(0) === 0xfeff) {
+    return content.slice(1);
+  }
+  return content;
+}
+
+export async function readTextFile(path: string | URL): Promise<string> {
+  return decodeBufferAsUtf8(await readFile(path), path);
+}
+
+export function readTextFileSync(path: string | URL): string {
+  return decodeBufferAsUtf8(readFileSync(path), path);
+}
+
 export function readInlinePathSync(path: string): string {
   try {
-    return readFileSync(path.replaceAll("/", SEP), "utf-8");
+    return readTextFileSync(path.replaceAll("/", SEP));
   } catch (error) {
     log.warn(`Error reading inline path: ${path}, ${error}`);
     return "";
@@ -253,7 +297,7 @@ export async function getIsWin(): Promise<boolean> {
  */
 export function writeIfChanged(path: string, content: string): boolean {
   try {
-    const existing = readFileSync(path, "utf-8");
+    const existing = readTextFileSync(path);
     if (existing === content) {
       return false; // Content unchanged, skip write
     }

cli/src/utils/yaml.ts

@@ -1,6 +1,6 @@
 import { parse as yamlParse } from "yaml";
 import type { ParseOptions, DocumentOptions, SchemaOptions, ToJSOptions, ScalarTag } from "yaml";
-import { readFile } from "node:fs/promises";
+import { readTextFile } from "./utils.ts";
 
 // Custom YAML tags that resolve `!inline value` and `!inline_fileset value`
 // back to their string-prefix form ("!inline value").
@@ -26,7 +26,7 @@ type YamlParseOptions = ParseOptions & DocumentOptions & SchemaOptions & ToJSOpt
 
 export async function yamlParseFile(path: string, options: YamlParseOptions = {}) {
   try {
-    return yamlParse(await readFile(path, "utf-8"), {
+    return yamlParse(await readTextFile(path), {
       ...options,
       customTags: [...WINDMILL_CUSTOM_TAGS, ...((options.customTags as ScalarTag[] | undefined) ?? [])],
     });

cli/test/utils_unit.test.ts

@@ -4,7 +4,10 @@
  */
 
 import { expect, test, describe } from "bun:test";
-import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize, validateRequiredArgs } from "../src/utils/utils.ts";
+import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize, validateRequiredArgs, stripBom, readTextFile, readTextFileSync } from "../src/utils/utils.ts";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import {
   getTypeStrFromPath,
   removeType,
@@ -634,6 +637,71 @@ describe("validateRequiredArgs", () => {
   });
 });
 
+// =============================================================================
+// BOM handling
+// =============================================================================
+
+describe("stripBom", () => {
+  test("strips UTF-8 BOM", () => {
+    expect(stripBom("hello")).toBe("hello");
+  });
+
+  test("returns input unchanged when no BOM", () => {
+    expect(stripBom("hello")).toBe("hello");
+    expect(stripBom("")).toBe("");
+  });
+});
+
+describe("readTextFile / readTextFileSync", () => {
+  const tmp = mkdtempSync(join(tmpdir(), "wmill-bom-"));
+
+  test("reads plain UTF-8 file", async () => {
+    const f = join(tmp, "plain.txt");
+    writeFileSync(f, Buffer.from("hello world", "utf-8"));
+    expect(await readTextFile(f)).toBe("hello world");
+    expect(readTextFileSync(f)).toBe("hello world");
+  });
+
+  test("strips UTF-8 BOM", async () => {
+    const f = join(tmp, "bom.txt");
+    writeFileSync(f, Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")]));
+    expect(await readTextFile(f)).toBe("hello");
+    expect(readTextFileSync(f)).toBe("hello");
+  });
+
+  test("throws on UTF-16 LE BOM", async () => {
+    const f = join(tmp, "utf16le.txt");
+    writeFileSync(f, Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")]));
+    await expect(readTextFile(f)).rejects.toThrow(/UTF-16 LE/);
+    expect(() => readTextFileSync(f)).toThrow(/UTF-16 LE/);
+  });
+
+  test("throws on UTF-16 BE BOM", async () => {
+    const f = join(tmp, "utf16be.txt");
+    writeFileSync(f, Buffer.from([0xfe, 0xff, 0x00, 0x68]));
+    await expect(readTextFile(f)).rejects.toThrow(/UTF-16 BE/);
+    expect(() => readTextFileSync(f)).toThrow(/UTF-16 BE/);
+  });
+
+  test("throws on UTF-32 LE BOM", async () => {
+    const f = join(tmp, "utf32le.txt");
+    writeFileSync(f, Buffer.from([0xff, 0xfe, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00]));
+    await expect(readTextFile(f)).rejects.toThrow(/UTF-32 LE/);
+  });
+
+  test("empty file reads as empty string", async () => {
+    const f = join(tmp, "empty.txt");
+    writeFileSync(f, Buffer.alloc(0));
+    expect(await readTextFile(f)).toBe("");
+    expect(readTextFileSync(f)).toBe("");
+  });
+
+  // cleanup
+  test("cleanup", () => {
+    rmSync(tmp, { recursive: true, force: true });
+  });
+});
+
 // =============================================================================
 // TarAsZip adapter
 // =============================================================================