feat: auto-strip UTF-8 BOM when reading local files in CLI (#8911)

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

cli/src/utils/local_path_scripts.ts

@@ -1,5 +1,6 @@
 import { execFileSync } from "node:child_process";
-import { readFile, stat } from "node:fs/promises";
+import { stat } from "node:fs/promises";
+import { readTextFile } from "./utils.ts";
 import type { SyncCodebase } from "./codebase.ts";
 import { parseMetadataFileIfExists } from "./metadata.ts";
 import { inferContentTypeFromFilePath } from "./script_common.ts";
@@ -16,7 +17,7 @@ export class UnsupportedLocalPathScriptPreviewError extends Error {
 
 async function readOptionalLock(scriptPath: string): Promise<string | undefined> {
   try {
-    return await readFile(scriptPath + ".script.lock", "utf-8");
+    return await readTextFile(scriptPath + ".script.lock");
   } catch {
     return undefined;
   }
@@ -138,7 +139,7 @@ export async function resolvePreviewLocalScriptState(
 
     return {
       filePath,
-      content: await readFile(filePath, "utf-8"),
+      content: await readTextFile(filePath),
       language,
       lock: normalizeOptionalLock(rawLock),
       tag: metadata?.payload?.tag,

cli/src/utils/metadata.ts

@@ -4,7 +4,7 @@ import { colors } from "@cliffy/ansi/colors";
 import * as log from "../core/log.ts";
 import { stringify as yamlStringify } from "yaml";
 import { yamlParseFile } from "./yaml.ts";
-import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises";
+import { writeFile, stat, rm, readdir } from "node:fs/promises";
 import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
 import * as path from "node:path";
 import { createRequire } from "node:module";
@@ -21,7 +21,7 @@ import {
 import { inferContentTypeFromFilePath } from "./script_common.ts";
 import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts";
 import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
-import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts";
+import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts";
 
 import { SyncCodebase } from "./codebase.ts";
 import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts";
@@ -66,7 +66,7 @@ export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Pro
       if (entry.isDirectory()) continue;
 
       const filePath = `dependencies/${entry.name}`;
-      const content = await readFile(filePath, "utf-8");
+      const content = await readTextFile(filePath);
 
       // Find matching language
       for (const lang of workspaceDependenciesLanguages) {
@@ -153,7 +153,7 @@ export async function filterWorkspaceDependenciesForScripts(
     if (content.startsWith("!inline ")) {
       const filePath = folder + sep + content.replace("!inline ", "");
       try {
-        content = await readFile(filePath, "utf-8");
+        content = await readTextFile(filePath);
       } catch {
         continue;
       }
@@ -212,8 +212,8 @@ export async function generateScriptMetadataInternal(
   );
 
   // read script content
-  const scriptContent = await readFile(scriptPath, "utf-8");
-  const metadataContent = await readFile(metadataWithType.path, "utf-8");
+  const scriptContent = await readTextFile(scriptPath);
+  const metadataContent = await readTextFile(metadataWithType.path);
 
   const filteredRawWorkspaceDependencies = filterWorkspaceDependencies(
     rawWorkspaceDependencies,
@@ -744,7 +744,7 @@ async function updateModuleLocks(
         if (!changedModules.includes(normalizedRelPath)) continue;
       }
 
-      const moduleContent = readFileSync(fullPath, "utf-8");
+      const moduleContent = readTextFileSync(fullPath);
       const moduleRemotePath = scriptRemotePath + "/" + relPath;
 
       log.debug(`Generating lock for module ${relPath}`);
@@ -986,7 +986,7 @@ export async function parseMetadataFileIfExists(
   let metadataFilePath = scriptPath + ".script.json";
   try {
     await stat(metadataFilePath);
-    const payload = JSON.parse(await readFile(metadataFilePath, "utf-8"));
+    const payload = JSON.parse(await readTextFile(metadataFilePath));
     replaceLock(payload);
     return {
       path: metadataFilePath,
@@ -1028,7 +1028,7 @@ export async function parseMetadataFile(
     await stat(metadataFilePath);
     return {
       path: metadataFilePath,
-      payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
+      payload: JSON.parse(await readTextFile(metadataFilePath)),
       isJson: true,
     };
   } catch {
@@ -1051,7 +1051,7 @@ export async function parseMetadataFile(
         await stat(metadataFilePath);
         return {
           path: metadataFilePath,
-          payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
+          payload: JSON.parse(await readTextFile(metadataFilePath)),
           isJson: true,
         };
       } catch {
@@ -1229,7 +1229,7 @@ async function computeModuleHashes(
         } catch {
           continue;
         }
-        const content = readFileSync(fullPath, "utf-8");
+        const content = readTextFileSync(fullPath);
         const normalizedPath = normalizeLockPath(relPath);
         hashes[normalizedPath] = await generateHash(
           content + JSON.stringify(rawWorkspaceDependencies)

cli/src/utils/utils.ts

@@ -131,9 +131,53 @@ export async function generateHashFromBuffer(
   return Buffer.from(hashBuffer).toString("hex");
 }
 
+function decodeBufferAsUtf8(buf: Buffer, path: string | URL): string {
+  if (buf.length >= 2) {
+    if (buf[0] === 0xff && buf[1] === 0xfe) {
+      if (buf.length >= 4 && buf[2] === 0x00 && buf[3] === 0x00) {
+        throw new Error(
+          `File ${path} is encoded as UTF-32 LE, which is not supported. Please convert it to UTF-8.`
+        );
+      }
+      throw new Error(
+        `File ${path} is encoded as UTF-16 LE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+    if (buf[0] === 0xfe && buf[1] === 0xff) {
+      throw new Error(
+        `File ${path} is encoded as UTF-16 BE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+    if (buf.length >= 4 && buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0xfe && buf[3] === 0xff) {
+      throw new Error(
+        `File ${path} is encoded as UTF-32 BE, which is not supported. Please convert it to UTF-8.`
+      );
+    }
+  }
+  if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) {
+    return buf.subarray(3).toString("utf-8");
+  }
+  return buf.toString("utf-8");
+}
+
+export function stripBom(content: string): string {
+  if (content.charCodeAt(0) === 0xfeff) {
+    return content.slice(1);
+  }
+  return content;
+}
+
+export async function readTextFile(path: string | URL): Promise<string> {
+  return decodeBufferAsUtf8(await readFile(path), path);
+}
+
+export function readTextFileSync(path: string | URL): string {
+  return decodeBufferAsUtf8(readFileSync(path), path);
+}
+
 export function readInlinePathSync(path: string): string {
   try {
-    return readFileSync(path.replaceAll("/", SEP), "utf-8");
+    return readTextFileSync(path.replaceAll("/", SEP));
   } catch (error) {
     log.warn(`Error reading inline path: ${path}, ${error}`);
     return "";
@@ -253,7 +297,7 @@ export async function getIsWin(): Promise<boolean> {
  */
 export function writeIfChanged(path: string, content: string): boolean {
   try {
-    const existing = readFileSync(path, "utf-8");
+    const existing = readTextFileSync(path);
     if (existing === content) {
       return false; // Content unchanged, skip write
     }

cli/src/utils/yaml.ts

@@ -1,6 +1,6 @@
 import { parse as yamlParse } from "yaml";
 import type { ParseOptions, DocumentOptions, SchemaOptions, ToJSOptions, ScalarTag } from "yaml";
-import { readFile } from "node:fs/promises";
+import { readTextFile } from "./utils.ts";
 
 // Custom YAML tags that resolve `!inline value` and `!inline_fileset value`
 // back to their string-prefix form ("!inline value").
@@ -26,7 +26,7 @@ type YamlParseOptions = ParseOptions & DocumentOptions & SchemaOptions & ToJSOpt
 
 export async function yamlParseFile(path: string, options: YamlParseOptions = {}) {
   try {
-    return yamlParse(await readFile(path, "utf-8"), {
+    return yamlParse(await readTextFile(path), {
       ...options,
       customTags: [...WINDMILL_CUSTOM_TAGS, ...((options.customTags as ScalarTag[] | undefined) ?? [])],
     });