feat(editor): add and remove UTF-8 BOM (#2717)

The encoding status menu of a UTF-8 file gets Add UTF-8 BOM / Remove UTF-8 BOM, and the status bar shows "UTF-8 BOM" for a file that has one. The change marks the document modified and is written on save (at once with auto-save).

Co-authored-by: YannABC <9603157+YannABC@users.noreply.github.com>
This commit is contained in:
Yann
2026-11-11 21:00:48 +08:00
committed by GitHub
parent 584baa3f71
commit 833ae4c3c4
13 changed files with 170 additions and 25 deletions
+10 -3
View File
@@ -5,7 +5,7 @@ use crate::rpc::Dispatcher;
use ide_core::{DocId, IdeError, IdeResult, OutlineItem, Range};
use ide_language::{FoldRange, SyntaxHighlights};
use ide_text::LineEnding;
use ide_workspace::{DocumentInfo, HistoryEntry};
use ide_workspace::{DocumentEncoding, DocumentInfo, HistoryEntry};
use std::path::PathBuf;
/// Runs blocking work (disk I/O, parsing, large copies) on the blocking pool,
@@ -29,18 +29,24 @@ pub async fn doc_ui_mode(host: &Host, window: &str, path: PathBuf) -> IdeResult<
blocking(move || Ok(project.documents.ui_mode_of(&path))).await
}
/// "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding label.
/// "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding and BOM state.
pub async fn doc_set_encoding(
host: &Host,
window: &str,
doc: DocId,
encoding: String,
reload: bool,
) -> IdeResult<String> {
) -> IdeResult<DocumentEncoding> {
let project = host.project(window)?;
host.blocking(move || project.documents.set_encoding(doc, &encoding, reload)).await
}
/// Adds or removes the BOM on a UTF-8 document; the change is written on save.
pub async fn doc_set_bom(host: &Host, window: &str, doc: DocId, has_bom: bool) -> IdeResult<DocumentEncoding> {
let project = host.project(window)?;
host.blocking(move || project.documents.set_bom(doc, has_bom)).await
}
pub fn doc_set_line_ending(host: &Host, window: &str, doc: DocId, line_ending: LineEnding) -> IdeResult<()> {
host.project(window)?.documents.set_line_ending(doc, line_ending)
}
@@ -175,6 +181,7 @@ pub(crate) fn register(d: &mut Dispatcher) {
d.add("doc_set_encoding", |c| async move {
doc_set_encoding(&c.host, &c.window, c.arg("doc")?, c.arg("encoding")?, c.arg("reload")?).await
});
d.add("doc_set_bom", |c| async move { doc_set_bom(&c.host, &c.window, c.arg("doc")?, c.arg("hasBom")?).await });
d.add("doc_set_line_ending", |c| async move {
blocking(move || doc_set_line_ending(&c.host, &c.window, c.arg("doc")?, c.arg("lineEnding")?)).await
});
+30 -3
View File
@@ -21,6 +21,7 @@ pub struct DocumentInfo {
pub version: u32,
pub language: LanguageId,
pub encoding: String,
pub has_bom: bool,
pub line_ending: LineEnding,
pub read_only: bool,
pub dirty: bool,
@@ -41,6 +42,14 @@ pub struct DocumentInfo {
pub ui_mode: Option<String>,
}
/// Encoding and BOM currently applied to an open document.
#[derive(Clone, Debug, Serialize, Deserialize, Type)]
#[serde(rename_all = "camelCase")]
pub struct DocumentEncoding {
pub encoding: String,
pub has_bom: bool,
}
/// Current text and identity of a document, for analyses (outline, folding).
#[derive(Clone, Debug)]
pub struct TextSnapshot {
@@ -326,6 +335,7 @@ impl Documents {
version: st.doc.version(),
language: st.language.clone(),
encoding: st.encoding.clone(),
has_bom: st.has_bom,
line_ending: st.doc.line_ending(),
read_only: st.read_only,
dirty: st.dirty,
@@ -822,7 +832,7 @@ impl Documents {
/// Changes the document's encoding. `reload`: re-decode the file on disk with
/// it (refused while there are unsaved changes); otherwise convert: the text
/// is kept and the next save writes it in the new encoding.
pub fn set_encoding(&self, doc: DocId, encoding: &str, reload: bool) -> IdeResult<String> {
pub fn set_encoding(&self, doc: DocId, encoding: &str, reload: bool) -> IdeResult<DocumentEncoding> {
self.with_doc(doc, |st| {
if st.uri.is_some() {
return Err(read_only_error(&st.path));
@@ -842,7 +852,7 @@ impl Documents {
self.replace_locked(doc, st, &loaded.text)?;
// Reloading is not an edit: the buffer matches the file again.
self.set_dirty(doc, st, false);
return Ok(loaded.encoding);
return Ok(DocumentEncoding { encoding: st.encoding.clone(), has_bom: st.has_bom });
}
if st.read_only {
return Err(read_only_error(&st.path));
@@ -856,7 +866,24 @@ impl Documents {
st.encoding = encoding.to_owned();
st.has_bom = has_bom;
self.set_dirty(doc, st, true);
Ok(st.encoding.clone())
Ok(DocumentEncoding { encoding: st.encoding.clone(), has_bom: st.has_bom })
})
}
/// Adds or removes the BOM on a UTF-8 document; the change is written on save.
pub fn set_bom(&self, doc: DocId, has_bom: bool) -> IdeResult<DocumentEncoding> {
self.with_doc(doc, |st| {
if st.uri.is_some() || st.read_only {
return Err(read_only_error(&st.path));
}
if st.encoding != "utf-8" || st.native_to_ascii {
return Err(IdeError::invalid("the BOM can only be changed for UTF-8 documents"));
}
if st.has_bom != has_bom {
st.has_bom = has_bom;
self.set_dirty(doc, st, true);
}
Ok(DocumentEncoding { encoding: st.encoding.clone(), has_bom: st.has_bom })
})
}
+1 -1
View File
@@ -21,7 +21,7 @@ pub mod unity;
pub mod window_groups;
pub mod workspace;
pub use documents::{DetectLanguage, DocumentInfo, Documents, TextSnapshot};
pub use documents::{DetectLanguage, DocumentEncoding, DocumentInfo, Documents, TextSnapshot};
pub use editorconfig::{EditorConfig, EditorConfigProps};
pub use file_defaults::FileDefaults;
pub use file_moves::{FileMove, FileMoves, UndoOutcome};
+29 -2
View File
@@ -486,11 +486,15 @@ fn reload_and_convert_encoding() {
let (bytes, _, _) = encoding_rs::GBK.encode("中文内容");
fs::write(&path, &bytes).unwrap();
let info = f.docs.open(&path).unwrap();
assert_eq!(f.docs.set_encoding(info.id, "gbk", true).unwrap(), "gbk");
let encoding = f.docs.set_encoding(info.id, "gbk", true).unwrap();
assert_eq!(encoding.encoding, "gbk");
assert!(!encoding.has_bom);
assert_eq!(f.docs.snapshot(info.id).unwrap().text, "中文内容");
// Convert: same text, written in the new encoding on save.
assert_eq!(f.docs.set_encoding(info.id, "utf-8", false).unwrap(), "utf-8");
let encoding = f.docs.set_encoding(info.id, "utf-8", false).unwrap();
assert_eq!(encoding.encoding, "utf-8");
assert!(!encoding.has_bom);
f.docs.save(info.id).unwrap();
assert_eq!(fs::read_to_string(&path).unwrap(), "中文内容");
@@ -501,6 +505,29 @@ fn reload_and_convert_encoding() {
assert!(f.docs.set_encoding(info.id, "gbk", true).is_err());
}
#[test]
fn utf8_bom_is_added_and_removed_on_save() {
let f = fixture();
let path = f.root.join("bom.txt");
fs::write(&path, "text").unwrap();
let info = f.docs.open(&path).unwrap();
assert!(!info.has_bom);
assert!(f.docs.set_bom(info.id, true).unwrap().has_bom);
assert_eq!(fs::read(&path).unwrap(), b"text", "written on save only");
f.docs.save(info.id).unwrap();
assert_eq!(fs::read(&path).unwrap(), b"\xEF\xBB\xBFtext");
assert_eq!(f.docs.snapshot(info.id).unwrap().text, "text");
assert!(!f.docs.set_bom(info.id, false).unwrap().has_bom);
f.docs.save(info.id).unwrap();
assert_eq!(fs::read(&path).unwrap(), b"text");
// Other encodings have no BOM to toggle.
f.docs.set_encoding(info.id, "gbk", false).unwrap();
assert!(f.docs.set_bom(info.id, true).is_err());
}
#[cfg(unix)]
#[test]
fn symlinked_path_maps_to_same_document() {
+14 -3
View File
@@ -6,7 +6,7 @@ use ide_core::{ChangeSet, DocId, IdeResult, OutlineItem, Range};
pub use ide_host::ops::doc::*;
use ide_language::{FoldRange, SyntaxHighlights};
use ide_text::LineEnding;
use ide_workspace::{DocumentInfo, HistoryEntry};
use ide_workspace::{DocumentEncoding, DocumentInfo, HistoryEntry};
use std::path::PathBuf;
use tauri::{Manager, State, Window};
@@ -87,7 +87,7 @@ pub async fn doc_ui_mode(window: Window, state: State<'_, AppState>, path: PathB
ide_host::ops::doc::doc_ui_mode(&state.host, window.label(), path).await
}
/// "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding label.
/// "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding and BOM state.
#[tauri::command]
#[specta::specta]
pub async fn doc_set_encoding(
@@ -96,10 +96,21 @@ pub async fn doc_set_encoding(
doc: DocId,
encoding: String,
reload: bool,
) -> IdeResult<String> {
) -> IdeResult<DocumentEncoding> {
ide_host::ops::doc::doc_set_encoding(&state.host, window.label(), doc, encoding, reload).await
}
#[tauri::command]
#[specta::specta]
pub async fn doc_set_bom(
window: Window,
state: State<'_, AppState>,
doc: DocId,
has_bom: bool,
) -> IdeResult<DocumentEncoding> {
ide_host::ops::doc::doc_set_bom(&state.host, window.label(), doc, has_bom).await
}
#[tauri::command]
#[specta::specta]
pub async fn doc_set_line_ending(
+1
View File
@@ -194,6 +194,7 @@ pub fn builder() -> tauri_specta::Builder<tauri::Wry> {
doc::doc_snapshot,
doc::doc_ui_mode,
doc::doc_set_encoding,
doc::doc_set_bom,
doc::doc_set_line_ending,
doc::doc_outline,
doc::doc_folding,
+10 -2
View File
@@ -298,8 +298,9 @@ export const commands = {
* without opening it: previews of closed files.
*/
docUiMode: (path: string) => typedError<string | null, IdeError>(__TAURI_INVOKE("doc_ui_mode", { path })),
/** "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding label. */
docSetEncoding: (doc: DocId, encoding: string, reload: boolean) => typedError<string, IdeError>(__TAURI_INVOKE("doc_set_encoding", { doc, encoding, reload })),
/** "Reload with Encoding" (`reload`) or "Convert to Encoding"; returns the new encoding and BOM state. */
docSetEncoding: (doc: DocId, encoding: string, reload: boolean) => typedError<DocumentEncoding, IdeError>(__TAURI_INVOKE("doc_set_encoding", { doc, encoding, reload })),
docSetBom: (doc: DocId, hasBom: boolean) => typedError<DocumentEncoding, IdeError>(__TAURI_INVOKE("doc_set_bom", { doc, hasBom })),
docSetLineEnding: (doc: DocId, lineEnding: LineEnding) => typedError<null, IdeError>(__TAURI_INVOKE("doc_set_line_ending", { doc, lineEnding })),
/**
* Structure view: LSP document symbols when available (with tree-sitter
@@ -4016,6 +4017,12 @@ export type DockerVolume = {
labels?: { [key in string]: string },
};
/** Encoding and BOM currently applied to an open document. */
export type DocumentEncoding = {
encoding: string,
hasBom: boolean,
};
/** What the UI needs to show an opened document. */
export type DocumentInfo = {
id: DocId,
@@ -4024,6 +4031,7 @@ export type DocumentInfo = {
version: number,
language: LanguageId,
encoding: string,
hasBom: boolean,
lineEnding: LineEnding,
readOnly: boolean,
dirty: boolean,
+31 -8
View File
@@ -12,6 +12,7 @@ import {
isTauri,
onIdeEvent,
type DocId,
type DocumentEncoding,
type DocumentInfo,
type FoldRange,
type LanguageInfo_Serialize,
@@ -47,8 +48,10 @@ export interface DocBackend {
syncFromDisk?(): Promise<void>;
snapshot(doc: DocId): Promise<DocumentInfo>;
setLineEnding(doc: DocId, lineEnding: LineEnding): Promise<void>;
/** "Reload with Encoding" (`reload`) or "Convert to Encoding"; resolves to the new encoding. */
setEncoding(doc: DocId, encoding: string, reload: boolean): Promise<string>;
/** "Reload with Encoding" (`reload`) or "Convert to Encoding"; resolves to the new encoding and BOM state. */
setEncoding(doc: DocId, encoding: string, reload: boolean): Promise<DocumentEncoding>;
/** Adds or removes the BOM on a UTF-8 document; the change is written on save. */
setBom(doc: DocId, hasBom: boolean): Promise<DocumentEncoding>;
outline(doc: DocId): Promise<OutlineItem[]>;
folding(doc: DocId): Promise<FoldRange[]>;
/** Tree-sitter highlighting, for languages without a CodeMirror mode. */
@@ -83,6 +86,7 @@ export const ipcBackend: DocBackend = {
snapshot: (doc) => call(commands.docSnapshot(doc)),
setLineEnding: async (doc, le) => void (await call(commands.docSetLineEnding(doc, le))),
setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)),
setBom: (doc, hasBom) => call(commands.docSetBom(doc, hasBom)),
outline: (doc) => call(commands.docOutline(doc)),
folding: (doc) => call(commands.docFolding(doc)),
highlights: (doc) => call(commands.docHighlights(doc)),
@@ -108,9 +112,11 @@ interface MemDoc {
path: string;
text: string;
saved: string;
savedBom: boolean;
version: number;
refs: number;
lineEnding: LineEnding;
hasBom: boolean;
readOnly: boolean;
language: string;
uiMode?: string;
@@ -377,6 +383,7 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
const dirtyHandlers = new Set<(e: { doc: DocId; dirty: boolean }) => void>();
let nextId = 1;
const store = new Map(Object.entries(files));
const savedBoms = new Map<string, boolean>();
const info = (d: MemDoc): DocumentInfo => ({
id: d.id,
@@ -385,9 +392,10 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
version: d.version,
language: d.language,
encoding: "UTF-8",
hasBom: d.hasBom,
lineEnding: d.lineEnding,
readOnly: d.readOnly,
dirty: d.text !== d.saved,
dirty: d.text !== d.saved || d.hasBom !== d.savedBom,
...(d.uri ? { uri: d.uri } : {}),
...(d.uiMode ? { uiMode: d.uiMode } : {}),
});
@@ -411,7 +419,7 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
return d;
};
const setDirty = (d: MemDoc, before: boolean) => {
const now = d.text !== d.saved;
const now = d.text !== d.saved || d.hasBom !== d.savedBom;
if (now !== before) dirtyHandlers.forEach((h) => h({ doc: d.id, dirty: now }));
};
const delay = <T>(v: () => T): Promise<T> =>
@@ -439,15 +447,18 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
return info(d);
}
const text = store.get(path) ?? sampleFor(path);
const hasBom = savedBoms.get(path) ?? false;
const ext = path.slice(path.lastIndexOf(".") + 1).toLowerCase();
const d: MemDoc = {
id: nextId++,
path,
text,
saved: text,
savedBom: hasBom,
version: 0,
refs: 1,
lineEnding: "lf",
hasBom,
readOnly: false,
language: EXT_LANG[ext] ?? "plaintext",
};
@@ -473,9 +484,11 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
path: uri,
text,
saved: text,
savedBom: false,
version: 0,
refs: 1,
lineEnding: "lf",
hasBom: false,
readOnly: true,
language: ext === "class" ? "java" : (EXT_LANG[ext] ?? "plaintext"),
uri,
@@ -496,7 +509,7 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
if (d.readOnly) throw new IpcError({ kind: "permissionDenied", message: "read-only" });
if (baseVersion !== d.version)
throw new IpcError({ kind: "outOfSync", message: `base ${baseVersion}, document at ${d.version}` });
const before = d.text !== d.saved;
const before = d.text !== d.saved || d.hasBom !== d.savedBom;
d.text = applyChanges(d.text, changes);
d.version++;
setDirty(d, before);
@@ -505,9 +518,11 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
},
async save(doc) {
const d = get(doc);
const before = d.text !== d.saved;
const before = d.text !== d.saved || d.hasBom !== d.savedBom;
d.saved = d.text;
d.savedBom = d.hasBom;
store.set(d.path, d.text);
savedBoms.set(d.path, d.hasBom);
setDirty(d, before);
},
async snapshot(doc) {
@@ -516,8 +531,16 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
async setLineEnding(doc, le) {
get(doc).lineEnding = le;
},
async setEncoding(_doc, encoding) {
return encoding;
async setEncoding(doc, encoding) {
return { encoding, hasBom: get(doc).hasBom };
},
async setBom(doc, hasBom) {
const d = get(doc);
if (d.readOnly) throw new IpcError({ kind: "permissionDenied", message: "read-only" });
const before = d.text !== d.saved || d.hasBom !== d.savedBom;
d.hasBom = hasBom;
setDirty(d, before);
return { encoding: "utf-8", hasBom: d.hasBom };
},
async outline(doc) {
// Brace-based outline good enough for the dev shell.
+1 -1
View File
@@ -1463,7 +1463,7 @@ export class EditorController implements SlotHost {
}
try {
const encoding = await this.manager.backend.setEncoding(meta.doc, meta.encoding, true);
model.setMeta({ encoding });
model.setMeta({ encoding: encoding.encoding, hasBom: encoding.hasBom });
} catch (e) {
notifications().notify({
level: "error",
+2
View File
@@ -82,6 +82,7 @@ export interface DocMeta {
language: string;
uiMode: string | null;
encoding: string;
hasBom: boolean;
lineEnding: LineEnding;
readOnly: boolean;
dirty: boolean;
@@ -248,6 +249,7 @@ export class DocModel {
language: info.language,
uiMode: uiMode ?? this.desc?.name ?? null,
encoding: info.encoding,
hasBom: info.hasBom,
lineEnding: info.lineEnding,
// Library sources (External Libraries) open read-only, like IDEA's.
readOnly: info.readOnly || (!this.virtual && isLibraryPath(info.path)),
+35 -2
View File
@@ -352,6 +352,7 @@ const ENCODINGS = ENCODING_CHOICES;
/** File encoding widget (IDEA: click → choose an encoding, then Reload or Convert). */
export function EncodingStatus() {
const meta = useActiveMeta();
const prefs = useEditorPrefs();
const [open, setOpen] = useState(false);
const ref = useRef<HTMLButtonElement>(null);
if (!meta) return null;
@@ -360,7 +361,7 @@ export function EncodingStatus() {
if (!model) return;
void editorController()
.manager.backend.setEncoding(meta.doc, encoding, reload)
.then((enc) => model.setMeta({ encoding: enc, ...(reload ? {} : { dirty: true }) }))
.then((enc) => model.setMeta({ encoding: enc.encoding, hasBom: enc.hasBom, ...(reload ? {} : { dirty: true }) }))
.catch((e: unknown) =>
notifications().notify({
level: "error",
@@ -370,6 +371,27 @@ export function EncodingStatus() {
);
};
const current = meta.encoding.toLowerCase();
const nativeToAsciiProperties = prefs.encodings.nativeToAscii && meta.path.endsWith(".properties");
const canToggleBom = current === "utf-8" && !nativeToAsciiProperties;
const label = ENCODINGS.find(([id]) => id === current)?.[1] ?? meta.encoding;
const applyBom = (hasBom: boolean) => {
const controller = editorController();
const model = controller.manager.get(meta.path);
if (!model) return;
void controller.manager.backend
.setBom(meta.doc, hasBom)
.then(async (enc) => {
model.setMeta({ encoding: enc.encoding, hasBom: enc.hasBom, dirty: true });
if (controller.autoSave()) await model.save("auto");
})
.catch((e: unknown) =>
notifications().notify({
level: "error",
title: tx("Cannot change the UTF-8 BOM"),
body: e instanceof Error ? e.message : String(e),
}),
);
};
const group = (reload: boolean) =>
ENCODINGS.filter(([id]) => id !== current).map(([id, label]) => ({
id: `${reload ? "reload" : "convert"}.${id}`,
@@ -385,7 +407,7 @@ export function EncodingStatus() {
aria-expanded={open}
onClick={() => setOpen((v) => !v)}
>
{ENCODINGS.find(([id]) => id === current)?.[1] ?? meta.encoding}
{meta.hasBom ? `${label} BOM` : label}
</BarButton>
<Menu
open={open}
@@ -396,6 +418,17 @@ export function EncodingStatus() {
items={[
{ id: "reload", label: "Reload with Encoding", disabled: meta.dirty, submenu: group(true) },
{ id: "convert", label: "Convert to Encoding", disabled: meta.readOnly, submenu: group(false) },
...(canToggleBom
? [
{ kind: "separator" as const },
{
id: "toggle-bom",
label: meta.hasBom ? "Remove UTF-8 BOM" : "Add UTF-8 BOM",
disabled: meta.readOnly,
run: () => applyBom(!meta.hasBom),
},
]
: []),
{ kind: "separator" },
{
id: "configure",
+3
View File
@@ -117,6 +117,9 @@
"Move the worktree at {path} to {to}?": "Move the worktree at {path} to {to}?",
"Cannot move a worktree containing the open project.": "Cannot move a worktree containing the open project.",
"Add Environment File": "Add Environment File",
"Add UTF-8 BOM": "Add UTF-8 BOM",
"Remove UTF-8 BOM": "Remove UTF-8 BOM",
"Cannot change the UTF-8 BOM": "Cannot change the UTF-8 BOM",
"Convert to cURL and Copy": "Convert to cURL and Copy",
"Add Environment to Public File": "Add Environment to Public File",
"Add Environment to Private File": "Add Environment to Private File",
+3
View File
@@ -844,6 +844,9 @@
"Continue Revert": "继续还原",
"Contributes": "贡献",
"Convert to Encoding": "转换为编码",
"Add UTF-8 BOM": "添加 UTF-8 BOM",
"Remove UTF-8 BOM": "移除 UTF-8 BOM",
"Cannot change the UTF-8 BOM": "无法更改 UTF-8 BOM",
"Copied code keeps the editor's syntax colors and font when pasted into documents and mail.": "复制的代码粘贴到文档和邮件中时会保留编辑器的语法颜色和字体。",
"Copy": "复制",
"Copy / Paste Special": "复制/选择性粘贴",