diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 37965a6c60..2589bf2fca 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { readFile, mkdir, readdir } from "node:fs/promises"; +import { mkdir, readdir } from "node:fs/promises"; import { colors } from "@cliffy/ansi/colors"; import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; @@ -23,7 +23,7 @@ import { ScriptLanguage, workspaceDependenciesLanguages, } from "../../utils/script_common.ts"; -import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; +import { generateHash, getHeaders, readTextFile, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; @@ -178,7 +178,7 @@ export async function generateAppLocksInternal( if (typeof content === "string" && content.startsWith("!inline ")) { const filePath = appFolder + SEP + content.replace("!inline ", ""); try { - content = await readFile(filePath, "utf-8"); + content = await readTextFile(filePath); } catch { return inlineScript; } @@ -893,7 +893,7 @@ export async function inferRunnableSchemaFromFile( ); let content: string; try { - content = await readFile(fullFilePath, "utf-8"); + content = await readTextFile(fullFilePath); } catch { log.warn(colors.yellow(`Could not read file: ${fullFilePath}`)); return undefined; diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 8d15488491..0f82f7b8c4 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -5,6 +5,7 @@ import { spawn } from "node:child_process"; import * as log from "../../core/log.ts"; import { colors } from "@cliffy/ansi/colors"; import * as windmillUtils from "@windmill-labs/shared-utils"; +import { readTextFile, readTextFileSync } from "../../utils/utils.ts"; export interface BundleOptions { entryPoint?: string; outDir?: string; @@ -41,7 +42,7 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea } try { - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf-8")); + const packageJson = JSON.parse(readTextFileSync(packageJsonPath)); const allDeps = { ...packageJson.dependencies, ...packageJson.devDependencies, @@ -69,7 +70,7 @@ function createSveltePlugin(appDir: string): any { const svelte = await import("svelte/compiler"); // Load the file from the file system - const source = await fs.promises.readFile(args.path, "utf8"); + const source = await readTextFile(args.path); const filename = path.relative(process.cwd(), args.path); // This converts a message in Svelte's format to esbuild's format @@ -269,9 +270,9 @@ export async function createBundle( throw new Error(`Expected JS bundle at ${jsPath} but file not found`); } - const jsContent = fs.readFileSync(jsPath, "utf-8"); + const jsContent = readTextFileSync(jsPath); const cssContent = fs.existsSync(cssPath) - ? fs.readFileSync(cssPath, "utf-8") + ? readTextFileSync(cssPath) : ""; try { diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index c9fe8f67ec..628849c375 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -13,7 +13,7 @@ import * as path from "node:path"; import process from "node:process"; import { Buffer } from "node:buffer"; import { writeFileSync } from "node:fs"; -import { readFile } from "node:fs/promises"; +import { readTextFile } from "../../utils/utils.ts"; import { WebSocket, WebSocketServer } from "ws"; import { createFrameworkPlugins, @@ -800,7 +800,7 @@ async function dev(opts: DevOptions, appFolder?: string) { const fileName = path.basename(filePath); try { - const sqlContent = await readFile(filePath, "utf-8"); + const sqlContent = await readTextFile(filePath); if (!sqlContent.trim()) { log.info(colors.gray(`Skipping empty file: ${fileName}`)); @@ -856,7 +856,7 @@ async function dev(opts: DevOptions, appFolder?: string) { // If there's a current SQL file being shown, send it to the new client if (currentSqlFile && fs.existsSync(currentSqlFile)) { try { - const sqlContent = await readFile(currentSqlFile, "utf-8"); + const sqlContent = await readTextFile(currentSqlFile); const datatable = await getDatatableConfig(); const fileName = path.basename(currentSqlFile); diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 6e71a3bf30..3cd5837e4d 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -9,10 +9,10 @@ import { stringify as yamlStringify } from "yaml"; import * as wmill from "../../../gen/services.gen.ts"; import { Policy } from "../../../gen/types.gen.ts"; import path from "node:path"; -import { readFile, readdir } from "node:fs/promises"; +import { readdir } from "node:fs/promises"; import { GlobalOptions, isSuperset } from "../../types.ts"; -import { deepEqual } from "../../utils/utils.ts"; +import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { replaceInlineScripts, repopulateFields } from "./app.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; @@ -65,8 +65,8 @@ async function findRunnableContentFile( // Check if this is a recognized extension if (EXTENSION_TO_LANGUAGE[ext]) { try { - const content = await readFile( - path.join(backendPath, fileName), "utf-8", + const content = await readTextFile( + path.join(backendPath, fileName), ); return { ext, content }; } catch { @@ -164,9 +164,8 @@ export async function loadRunnablesFromBackend( // Try to load lock file let lock: string | undefined; try { - lock = await readFile( + lock = await readTextFile( path.join(backendPath, `${runnableId}.lock`), - "utf-8", ); } catch { // No lock file, that's fine @@ -226,8 +225,8 @@ export async function loadRunnablesFromBackend( // Try to load lock file let lock: string | undefined; try { - lock = await readFile( - path.join(backendPath, `${runnableId}.lock`), "utf-8", + lock = await readTextFile( + path.join(backendPath, `${runnableId}.lock`), ); } catch { // No lock file, that's fine @@ -319,7 +318,7 @@ async function collectAppFiles( ) { continue; } - const content = await readFile(fullPath, "utf-8"); + const content = await readTextFile(fullPath); files[relativePath] = content; } } diff --git a/cli/src/commands/dependencies/dependencies.ts b/cli/src/commands/dependencies/dependencies.ts index cde4fe256c..2daf2d180a 100644 --- a/cli/src/commands/dependencies/dependencies.ts +++ b/cli/src/commands/dependencies/dependencies.ts @@ -7,6 +7,7 @@ import * as log from "../../core/log.ts"; import * as wmill from "../../../gen/services.gen.ts"; import fs from "node:fs"; import { workspaceDependenciesPathToLanguageAndFilename } from "../../utils/metadata.ts"; +import { readTextFileSync } from "../../utils/utils.ts"; async function push( opts: GlobalOptions, @@ -19,7 +20,7 @@ async function push( throw new Error(`File not found: ${filePath}`); } - const content = fs.readFileSync(filePath, "utf8"); + const content = readTextFileSync(filePath); // Use the existing pushWorkspaceDependencies function await pushWorkspaceDependencies( diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index 5352270f15..b49aad2073 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -7,7 +7,8 @@ import { WebSocket, WebSocketServer } from "ws"; import * as getPort from "get-port"; import * as http from "node:http"; import * as open from "open"; -import { readFile, realpath } from "node:fs/promises"; +import { realpath } from "node:fs/promises"; +import { readTextFile } from "../../utils/utils.ts"; import { watch } from "node:fs"; import { getTypeStrFromPath, GlobalOptions } from "../../types.ts"; import { ignoreF } from "../sync/sync.ts"; @@ -93,7 +94,7 @@ async function dev(opts: GlobalOptions & SyncOptions) { )) as FlowFile; await replaceInlineScripts( localFlow.value.modules, - async (path: string) => await readFile(localPath + path, "utf-8"), + async (path: string) => await readTextFile(localPath + path), log, localPath, SEP, @@ -114,7 +115,7 @@ async function dev(opts: GlobalOptions & SyncOptions) { log.info("Updated " + localPath); broadcastChanges(currentLastEdit); } else if (typ == "script") { - const content = await readFile(cpath, "utf-8"); + const content = await readTextFile(cpath); const splitted = cpath.split("."); const wmPath = splitted[0]; const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs); diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index a613aa2b0f..0fa59c84ea 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -7,9 +7,8 @@ import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; -import { validateRequiredArgs } from "../../utils/utils.ts"; +import { readTextFile, validateRequiredArgs } from "../../utils/utils.ts"; import * as wmill from "../../../gen/services.gen.ts"; -import { readFile } from "node:fs/promises"; import { mkdirSync, writeFileSync } from "node:fs"; import { buildFolderPath, getMetadataFileName, loadNonDottedPathsSetting } from "../../utils/resource_folders.ts"; @@ -157,7 +156,7 @@ export async function pushFlow( } const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile; - const fileReader = async (path: string) => await readFile(localPath + path, "utf-8"); + const fileReader = async (path: string) => await readTextFile(localPath + path); const missingFiles: string[] = []; await replaceInlineScripts( localFlow.value.modules, @@ -545,7 +544,7 @@ async function preview( const localFlow = (await yamlParseFile(flowPath + "flow.yaml")) as FlowFile; // Replace inline scripts with their actual content - const fileReader = async (path: string) => await readFile(flowPath + path, "utf-8"); + const fileReader = async (path: string) => await readTextFile(flowPath + path); await replaceInlineScripts( localFlow.value.modules, fileReader, diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 3c1c8cbd15..4110004959 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -4,7 +4,6 @@ import * as path from "node:path"; import { sep as SEP } from "node:path"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "../../utils/yaml.ts"; -import { readFile } from "node:fs/promises"; import { GlobalOptions } from "../../types.ts"; import { readLockfile, @@ -21,7 +20,7 @@ import { ScriptLanguage } from "../../utils/script_common.ts"; import { extractInlineScripts as extractInlineScriptsForFlows, extractCurrentMapping } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; -import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; +import { generateHash, getHeaders, readTextFile, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; @@ -109,7 +108,7 @@ export async function generateFlowLockInternal( if (content.startsWith("!inline ")) { const filePath = folder + SEP + content.replace("!inline ", ""); try { - content = await readFile(filePath, "utf-8"); + content = await readTextFile(filePath); } catch { continue; } @@ -192,7 +191,7 @@ export async function generateFlowLockInternal( if (!noStaleMessage) { log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`); } - const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8"); + const fileReader = async (path: string) => await readTextFile(folder + SEP + path); // Capture existing module-ID-to-file-path mapping before replaceInlineScripts // overwrites the !inline references with actual file content. This preserves diff --git a/cli/src/commands/instance/instance.ts b/cli/src/commands/instance/instance.ts index 3f21189074..6289dd2b01 100644 --- a/cli/src/commands/instance/instance.ts +++ b/cli/src/commands/instance/instance.ts @@ -1,4 +1,4 @@ -import { readFile, writeFile, readdir, mkdir, rm, stat } from "node:fs/promises"; +import { writeFile, readdir, mkdir, rm, stat } from "node:fs/promises"; import { appendFile } from "node:fs/promises"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; @@ -39,7 +39,7 @@ import { pushInstanceSettings, type SimplifiedSettings, } from "../../core/settings.ts"; -import { deepEqual } from "../../utils/utils.ts"; +import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { getActiveWorkspace } from "../workspace/workspace.ts"; export interface Instance { @@ -52,7 +52,7 @@ export interface Instance { export async function allInstances(): Promise { try { const file = await getInstancesConfigFilePath(); - const txt = await readFile(file, "utf-8"); + const txt = await readTextFile(file); return txt .split("\n") .map((line) => { @@ -658,7 +658,7 @@ export async function getActiveInstance(opts: { return opts.instance; } try { - return await readFile(await getActiveInstanceFilePath(), "utf-8"); + return await readTextFile(await getActiveInstanceFilePath()); } catch { return undefined; } diff --git a/cli/src/commands/jobs/jobs.ts b/cli/src/commands/jobs/jobs.ts index 17a58d11f2..e7b421ab9d 100644 --- a/cli/src/commands/jobs/jobs.ts +++ b/cli/src/commands/jobs/jobs.ts @@ -7,6 +7,7 @@ import { Confirm } from "@cliffy/prompt/confirm"; import * as log from "../../core/log.ts"; import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import * as fs from "node:fs/promises"; +import { readTextFile } from "../../utils/utils.ts"; import * as wmill from "../../../gen/services.gen.ts"; async function pullJobs( @@ -190,7 +191,7 @@ async function pushJobs( // Push completed jobs const completedPath = opts.completedFile || "completed_jobs.json"; try { - const completedContent = await fs.readFile(completedPath, "utf-8"); + const completedContent = await readTextFile(completedPath); const completedJobs = JSON.parse(completedContent); if (!Array.isArray(completedJobs)) { @@ -218,7 +219,7 @@ async function pushJobs( // Push queued jobs const queuedPath = opts.queuedFile || "queued_jobs.json"; try { - const queuedContent = await fs.readFile(queuedPath, "utf-8"); + const queuedContent = await readTextFile(queuedPath); const queuedJobs = JSON.parse(queuedContent); if (!Array.isArray(queuedJobs)) { diff --git a/cli/src/commands/resource/resource.ts b/cli/src/commands/resource/resource.ts index 054ed91d06..f8a8713928 100644 --- a/cli/src/commands/resource/resource.ts +++ b/cli/src/commands/resource/resource.ts @@ -1,4 +1,4 @@ -import { mkdir, stat, writeFile, readdir, readFile } from "node:fs/promises"; +import { mkdir, stat, writeFile, readdir } from "node:fs/promises"; import { stringify as yamlStringify } from "yaml"; import nodePath from "node:path"; @@ -17,7 +17,7 @@ import * as log from "../../core/log.ts"; import { sep as SEP } from "node:path"; import * as wmill from "../../../gen/services.gen.ts"; import { Resource } from "../../../gen/types.gen.ts"; -import { readInlinePathSync } from "../../utils/utils.ts"; +import { readInlinePathSync, readTextFile } from "../../utils/utils.ts"; import { isWorkspaceSpecificFile } from "../../core/specific_items.ts"; import { getCurrentGitBranch } from "../../utils/git.ts"; @@ -38,7 +38,7 @@ async function readFilesetDirectory(dirPath: string): Promise { const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/"); const metadataWithType = await parseMetadataFile(remotePath, undefined); - const metadataContent = await readFile(metadataWithType.path, "utf-8"); + const metadataContent = await readTextFile(metadataWithType.path); return await generateScriptHash({}, content, metadataContent); } @@ -141,7 +141,7 @@ async function push(opts: PushOptions, filePath: string) { // Warn about metadata state before pushing try { - const content = await readFile(filePath, "utf-8"); + const content = await readTextFile(filePath); const remotePath = removeExtensionToPath(filePath).replaceAll(SEP, "/"); const contentHash = await computePushMetadataHash(filePath, content); const conf = await readLockfile(); @@ -432,7 +432,7 @@ export async function handleFile( } catch { log.debug(`Script ${remotePath} does not exist on remote`); } - const content = await readFile(path, "utf-8"); + const content = await readTextFile(path); if (opts?.skipScriptsMetadata) { // if (codebase) { @@ -619,7 +619,7 @@ export async function readModulesFromDisk( } else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) { // Skip lock files — they're handled as the `lock` field on ScriptModule if (exts.some((ext) => entry.name.endsWith(ext))) { - const content = fs.readFileSync(fullPath, "utf-8"); + const content = readTextFileSync(fullPath); const language = inferContentTypeFromFilePath(entry.name, defaultTs); // Check for an accompanying lock file (helper.lock) @@ -627,7 +627,7 @@ export async function readModulesFromDisk( const lockPath = path.join(dirPath, baseName + ".lock"); let lock: string | undefined; if (fs.existsSync(lockPath)) { - lock = fs.readFileSync(lockPath, "utf-8"); + lock = readTextFileSync(lockPath); } modules[relPath] = { @@ -958,7 +958,7 @@ export async function resolve(input: string): Promise> { input = new TextDecoder().decode(Buffer.concat(chunks)); } if (input[0] == "@") { - input = await readFile(input.substring(1), "utf-8"); + input = await readTextFile(input.substring(1)); } try { return JSON.parse(input); @@ -1404,7 +1404,7 @@ async function preview( const codebases = await listSyncCodebases(opts); const language = inferContentTypeFromFilePath(filePath, opts?.defaultTs); - const content = await readFile(filePath, "utf-8"); + const content = await readTextFile(filePath); const input = opts.data ? await resolve(opts.data) : {}; // Read modules from __mod/ folder if present diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index ad7a895fe8..b08afb86e2 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1,6 +1,6 @@ import { requireLogin } from "../../core/auth.ts"; import { fetchVersion, resolveWorkspace } from "../../core/context.ts"; -import { readFile, writeFile, readdir, stat, rm, copyFile, mkdir } from "node:fs/promises"; +import { writeFile, readdir, stat, rm, copyFile, mkdir } from "node:fs/promises"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; import { Confirm } from "@cliffy/prompt/confirm"; @@ -42,6 +42,7 @@ import { isFilesetResource, isRawAppFile, isWorkspaceDependencies, + readTextFile, } from "../../utils/utils.ts"; import { getEffectiveSettings, @@ -325,7 +326,7 @@ export async function FSFSElement( } }, async getContentText(): Promise { - const content = await readFile(localP, "utf-8"); + const content = await readTextFile(localP); const itemPath = localP.substring(p.length + 1); const r = await addCodebaseDigestIfRelevant( itemPath, @@ -2331,7 +2332,7 @@ export async function pull( if (change.name === "edited") { if (opts.stateful) { try { - const currentLocal = await readFile(target, "utf-8"); + const currentLocal = await readTextFile(target); if ( currentLocal !== change.before && currentLocal !== change.after @@ -3307,7 +3308,7 @@ export async function push( const newObj = parseFromPath( resourceFilePath, - await readFile(resourceFilePath, "utf-8"), + await readTextFile(resourceFilePath), ); // For branch-specific resources, push to the base path on the workspace server @@ -3342,7 +3343,7 @@ export async function push( const newObj = parseFromPath( resourceFilePath, - await readFile(resourceFilePath, "utf-8"), + await readTextFile(resourceFilePath), ); let serverPath = resourceFilePath; diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index d2bc2a5bbc..d8cc16b01b 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -1,4 +1,5 @@ -import { readFile, writeFile, open as fsOpen } from "node:fs/promises"; +import { writeFile, open as fsOpen } from "node:fs/promises"; +import { readTextFile } from "../../utils/utils.ts"; import process from "node:process"; import { GlobalOptions } from "../../types.ts"; import { @@ -31,7 +32,7 @@ export async function allWorkspaces( ): Promise { try { const file = await getWorkspaceConfigFilePath(configDirOverride); - const txt = await readFile(file, "utf-8"); + const txt = await readTextFile(file); return txt .split("\n") .map((line) => { @@ -55,7 +56,7 @@ async function getActiveWorkspaceName( } try { const file = await getActiveWorkspaceConfigFilePath(opts?.configDir); - return await readFile(file, "utf-8"); + return await readTextFile(file); } catch { return undefined; } diff --git a/cli/src/core/branch-profiles.ts b/cli/src/core/branch-profiles.ts index 80c72b0705..bacbcc51aa 100644 --- a/cli/src/core/branch-profiles.ts +++ b/cli/src/core/branch-profiles.ts @@ -1,5 +1,6 @@ import * as log from "./log.ts"; -import { readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; +import { readTextFile } from "../utils/utils.ts"; import { getStore } from "./store.ts"; export interface BranchProfileMapping { @@ -17,7 +18,7 @@ export async function getBranchProfilesPath(configDirOverride?: string): Promise export async function loadBranchProfiles(configDirOverride?: string): Promise { try { const path = await getBranchProfilesPath(configDirOverride); - const content = await readFile(path, "utf-8"); + const content = await readTextFile(path); return JSON.parse(content); } catch { // File doesn't exist or invalid JSON - return empty mapping diff --git a/cli/src/guidance/writer.ts b/cli/src/guidance/writer.ts index 0acab52784..b301272882 100644 --- a/cli/src/guidance/writer.ts +++ b/cli/src/guidance/writer.ts @@ -1,4 +1,5 @@ -import { cp, mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises"; +import { cp, mkdir, readdir, stat, writeFile } from "node:fs/promises"; +import { readTextFile } from "../utils/utils.ts"; import { join } from "node:path"; import { generateAgentsMdContent } from "./core.ts"; import { @@ -47,7 +48,7 @@ export async function writeAiGuidanceFiles( overwrite: options.overwriteProjectGuidance ?? false, content: options.agentsSourcePath != null - ? await readFile(options.agentsSourcePath, "utf8") + ? await readTextFile(options.agentsSourcePath) : generateAgentsMdContent(buildSkillsReference(skillMetadata)), }); @@ -56,7 +57,7 @@ export async function writeAiGuidanceFiles( overwrite: options.overwriteProjectGuidance ?? false, content: options.claudeSourcePath != null - ? await readFile(options.claudeSourcePath, "utf8") + ? await readTextFile(options.claudeSourcePath) : CLAUDE_MD_DEFAULT, }); @@ -202,7 +203,7 @@ async function readSkillMetadataFromDirectory(skillsDir: string): Promise { try { - return await readFile(scriptPath + ".script.lock", "utf-8"); + return await readTextFile(scriptPath + ".script.lock"); } catch { return undefined; } @@ -138,7 +139,7 @@ export async function resolvePreviewLocalScriptState( return { filePath, - content: await readFile(filePath, "utf-8"), + content: await readTextFile(filePath), language, lock: normalizeOptionalLock(rawLock), tag: metadata?.payload?.tag, diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 26e050d5b2..e4032d4857 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -4,7 +4,7 @@ import { colors } from "@cliffy/ansi/colors"; import * as log from "../core/log.ts"; import { stringify as yamlStringify } from "yaml"; import { yamlParseFile } from "./yaml.ts"; -import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises"; +import { writeFile, stat, rm, readdir } from "node:fs/promises"; import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs"; import * as path from "node:path"; import { createRequire } from "node:module"; @@ -21,7 +21,7 @@ import { import { inferContentTypeFromFilePath } from "./script_common.ts"; import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts"; import { findCodebase, yamlOptions } from "../commands/sync/sync.ts"; -import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; +import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts"; @@ -66,7 +66,7 @@ export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Pro if (entry.isDirectory()) continue; const filePath = `dependencies/${entry.name}`; - const content = await readFile(filePath, "utf-8"); + const content = await readTextFile(filePath); // Find matching language for (const lang of workspaceDependenciesLanguages) { @@ -153,7 +153,7 @@ export async function filterWorkspaceDependenciesForScripts( if (content.startsWith("!inline ")) { const filePath = folder + sep + content.replace("!inline ", ""); try { - content = await readFile(filePath, "utf-8"); + content = await readTextFile(filePath); } catch { continue; } @@ -212,8 +212,8 @@ export async function generateScriptMetadataInternal( ); // read script content - const scriptContent = await readFile(scriptPath, "utf-8"); - const metadataContent = await readFile(metadataWithType.path, "utf-8"); + const scriptContent = await readTextFile(scriptPath); + const metadataContent = await readTextFile(metadataWithType.path); const filteredRawWorkspaceDependencies = filterWorkspaceDependencies( rawWorkspaceDependencies, @@ -744,7 +744,7 @@ async function updateModuleLocks( if (!changedModules.includes(normalizedRelPath)) continue; } - const moduleContent = readFileSync(fullPath, "utf-8"); + const moduleContent = readTextFileSync(fullPath); const moduleRemotePath = scriptRemotePath + "/" + relPath; log.debug(`Generating lock for module ${relPath}`); @@ -986,7 +986,7 @@ export async function parseMetadataFileIfExists( let metadataFilePath = scriptPath + ".script.json"; try { await stat(metadataFilePath); - const payload = JSON.parse(await readFile(metadataFilePath, "utf-8")); + const payload = JSON.parse(await readTextFile(metadataFilePath)); replaceLock(payload); return { path: metadataFilePath, @@ -1028,7 +1028,7 @@ export async function parseMetadataFile( await stat(metadataFilePath); return { path: metadataFilePath, - payload: JSON.parse(await readFile(metadataFilePath, "utf-8")), + payload: JSON.parse(await readTextFile(metadataFilePath)), isJson: true, }; } catch { @@ -1051,7 +1051,7 @@ export async function parseMetadataFile( await stat(metadataFilePath); return { path: metadataFilePath, - payload: JSON.parse(await readFile(metadataFilePath, "utf-8")), + payload: JSON.parse(await readTextFile(metadataFilePath)), isJson: true, }; } catch { @@ -1229,7 +1229,7 @@ async function computeModuleHashes( } catch { continue; } - const content = readFileSync(fullPath, "utf-8"); + const content = readTextFileSync(fullPath); const normalizedPath = normalizeLockPath(relPath); hashes[normalizedPath] = await generateHash( content + JSON.stringify(rawWorkspaceDependencies) diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index eb8babfff3..ff9fe9b78c 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -131,9 +131,53 @@ export async function generateHashFromBuffer( return Buffer.from(hashBuffer).toString("hex"); } +function decodeBufferAsUtf8(buf: Buffer, path: string | URL): string { + if (buf.length >= 2) { + if (buf[0] === 0xff && buf[1] === 0xfe) { + if (buf.length >= 4 && buf[2] === 0x00 && buf[3] === 0x00) { + throw new Error( + `File ${path} is encoded as UTF-32 LE, which is not supported. Please convert it to UTF-8.` + ); + } + throw new Error( + `File ${path} is encoded as UTF-16 LE, which is not supported. Please convert it to UTF-8.` + ); + } + if (buf[0] === 0xfe && buf[1] === 0xff) { + throw new Error( + `File ${path} is encoded as UTF-16 BE, which is not supported. Please convert it to UTF-8.` + ); + } + if (buf.length >= 4 && buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0xfe && buf[3] === 0xff) { + throw new Error( + `File ${path} is encoded as UTF-32 BE, which is not supported. Please convert it to UTF-8.` + ); + } + } + if (buf.length >= 3 && buf[0] === 0xef && buf[1] === 0xbb && buf[2] === 0xbf) { + return buf.subarray(3).toString("utf-8"); + } + return buf.toString("utf-8"); +} + +export function stripBom(content: string): string { + if (content.charCodeAt(0) === 0xfeff) { + return content.slice(1); + } + return content; +} + +export async function readTextFile(path: string | URL): Promise { + return decodeBufferAsUtf8(await readFile(path), path); +} + +export function readTextFileSync(path: string | URL): string { + return decodeBufferAsUtf8(readFileSync(path), path); +} + export function readInlinePathSync(path: string): string { try { - return readFileSync(path.replaceAll("/", SEP), "utf-8"); + return readTextFileSync(path.replaceAll("/", SEP)); } catch (error) { log.warn(`Error reading inline path: ${path}, ${error}`); return ""; @@ -253,7 +297,7 @@ export async function getIsWin(): Promise { */ export function writeIfChanged(path: string, content: string): boolean { try { - const existing = readFileSync(path, "utf-8"); + const existing = readTextFileSync(path); if (existing === content) { return false; // Content unchanged, skip write } diff --git a/cli/src/utils/yaml.ts b/cli/src/utils/yaml.ts index 52ec682067..566537d58e 100644 --- a/cli/src/utils/yaml.ts +++ b/cli/src/utils/yaml.ts @@ -1,6 +1,6 @@ import { parse as yamlParse } from "yaml"; import type { ParseOptions, DocumentOptions, SchemaOptions, ToJSOptions, ScalarTag } from "yaml"; -import { readFile } from "node:fs/promises"; +import { readTextFile } from "./utils.ts"; // Custom YAML tags that resolve `!inline value` and `!inline_fileset value` // back to their string-prefix form ("!inline value"). @@ -26,7 +26,7 @@ type YamlParseOptions = ParseOptions & DocumentOptions & SchemaOptions & ToJSOpt export async function yamlParseFile(path: string, options: YamlParseOptions = {}) { try { - return yamlParse(await readFile(path, "utf-8"), { + return yamlParse(await readTextFile(path), { ...options, customTags: [...WINDMILL_CUSTOM_TAGS, ...((options.customTags as ScalarTag[] | undefined) ?? [])], }); diff --git a/cli/test/utils_unit.test.ts b/cli/test/utils_unit.test.ts index e5bcea4b92..de95848524 100644 --- a/cli/test/utils_unit.test.ts +++ b/cli/test/utils_unit.test.ts @@ -4,7 +4,10 @@ */ import { expect, test, describe } from "bun:test"; -import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize, validateRequiredArgs } from "../src/utils/utils.ts"; +import { deepEqual, isFileResource, isFilesetResource, toCamel, capitalize, validateRequiredArgs, stripBom, readTextFile, readTextFileSync } from "../src/utils/utils.ts"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { getTypeStrFromPath, removeType, @@ -634,6 +637,71 @@ describe("validateRequiredArgs", () => { }); }); +// ============================================================================= +// BOM handling +// ============================================================================= + +describe("stripBom", () => { + test("strips UTF-8 BOM", () => { + expect(stripBom("hello")).toBe("hello"); + }); + + test("returns input unchanged when no BOM", () => { + expect(stripBom("hello")).toBe("hello"); + expect(stripBom("")).toBe(""); + }); +}); + +describe("readTextFile / readTextFileSync", () => { + const tmp = mkdtempSync(join(tmpdir(), "wmill-bom-")); + + test("reads plain UTF-8 file", async () => { + const f = join(tmp, "plain.txt"); + writeFileSync(f, Buffer.from("hello world", "utf-8")); + expect(await readTextFile(f)).toBe("hello world"); + expect(readTextFileSync(f)).toBe("hello world"); + }); + + test("strips UTF-8 BOM", async () => { + const f = join(tmp, "bom.txt"); + writeFileSync(f, Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from("hello", "utf-8")])); + expect(await readTextFile(f)).toBe("hello"); + expect(readTextFileSync(f)).toBe("hello"); + }); + + test("throws on UTF-16 LE BOM", async () => { + const f = join(tmp, "utf16le.txt"); + writeFileSync(f, Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from("hello", "utf16le")])); + await expect(readTextFile(f)).rejects.toThrow(/UTF-16 LE/); + expect(() => readTextFileSync(f)).toThrow(/UTF-16 LE/); + }); + + test("throws on UTF-16 BE BOM", async () => { + const f = join(tmp, "utf16be.txt"); + writeFileSync(f, Buffer.from([0xfe, 0xff, 0x00, 0x68])); + await expect(readTextFile(f)).rejects.toThrow(/UTF-16 BE/); + expect(() => readTextFileSync(f)).toThrow(/UTF-16 BE/); + }); + + test("throws on UTF-32 LE BOM", async () => { + const f = join(tmp, "utf32le.txt"); + writeFileSync(f, Buffer.from([0xff, 0xfe, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00])); + await expect(readTextFile(f)).rejects.toThrow(/UTF-32 LE/); + }); + + test("empty file reads as empty string", async () => { + const f = join(tmp, "empty.txt"); + writeFileSync(f, Buffer.alloc(0)); + expect(await readTextFile(f)).toBe(""); + expect(readTextFileSync(f)).toBe(""); + }); + + // cleanup + test("cleanup", () => { + rmSync(tmp, { recursive: true, force: true }); + }); +}); + // ============================================================================= // TarAsZip adapter // =============================================================================