Files
qingjian/tools/corpus/levels.py
T
yenharvey 7a2ccc7691 v0.1.0:首个版本
青简(Qingjian):用 Rust 写的拼音输入法,候选词旁多一条正在学的那门语言的译词,一次只学一种。
核心引擎平台无关(crates/),macOS 输入法(apps/macos)已可用,CLI 测试工具(apps/cli),
产品数据生成工具(tools/),设计文档、计划与用户文档在 docs/,发版流程见 docs/notes/release.md。
2026-09-07 19:17:46 +08:00

111 lines
4.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# /// script
# requires-python = ">=3.11"
# dependencies = []
# ///
"""把词汇等级表转成青简的 `levels-<语言>.tsv`(`词\t等级`,`# levels` 行给出等级从易到难的顺序)。
英文:CEFR-J Wordlist 1.5(A1–B2)+ Octanove Vocabulary Profile(C1/C2),`headword,pos,CEFR,…`,
斜杠分隔的拼写变体各算一条,同一个词多个词性取最低等级。
日文:elzup/jlpt-word-list 的 N5–N1(源自 tanos.co.uk),`expression,reading,meaning,tags`,
`; ` 分隔的写法各算一条,去掉 `~` 与括号说明,同一个词出现在多级取最易的一级。
用法:
uv run tools/corpus/levels.py en data/levels/cefrj-vocabulary-profile-1.5.csv \
data/levels/octanove-vocabulary-profile-c1c2-1.0.csv -o assets/levels/levels-en.tsv
uv run tools/corpus/levels.py ja data/levels/jlpt-n5.csv data/levels/jlpt-n4.csv \
data/levels/jlpt-n3.csv data/levels/jlpt-n2.csv data/levels/jlpt-n1.csv -o assets/levels/levels-ja.tsv
"""
from __future__ import annotations
import argparse
import csv
import re
from pathlib import Path
EN_LEVELS = ["A1", "A2", "B1", "B2", "C1", "C2"]
JA_LEVELS = ["N5", "N4", "N3", "N2", "N1"]
EN_HEADER = [
"# 英文词汇等级(CEFR)。来源:The CEFR-J Wordlist Version 1.5, compiled by Yukio Tono, Tokyo University of Foreign Studies,",
"# retrieved from http://www.cefr-j.org/download.html(A1–B2,研究与商业用途免费,须署名);",
"# Octanove Vocabulary Profile C1/C2 1.0(CC BY-SA 4.0,https://github.com/openlanguageprofiles/olp-en-cefrj)。",
"# 由 tools/corpus/levels.py 生成。格式:词\\t等级;`# levels` 行是等级从易到难的顺序。",
]
JA_HEADER = [
"# 日文词汇等级(JLPT)。来源:tanos.co.uk 的 JLPT 词表(Jonathan Waller,CC BY),",
"# 经 elzup/jlpt-word-list(MIT,https://github.com/elzup/jlpt-word-list)整理为 CSV。",
"# 由 tools/corpus/levels.py 生成。格式:词\\t等级;`# levels` 行是等级从易到难的顺序。",
]
def clean_english(headword: str) -> list[str]:
"""`a.m./A.M./am/AM` → 各拼写;统一小写。"""
words = []
for part in headword.split("/"):
word = part.strip().lower()
if word and word not in words:
words.append(word)
return words
def clean_japanese(expression: str) -> list[str]:
"""`足; 脚` → 两条;`~円` → `円`;`こす (みずを~)` → `こす`。"""
words = []
for part in re.split(r"[;;]", expression):
word = re.sub(r"[((].*?[))]", "", part)
word = word.replace("~", "").replace("〜", "").strip()
if word and word not in words:
words.append(word)
return words
def convert(language: str, sources: list[Path]) -> tuple[list[str], dict[str, int], list[str]]:
if language == "en":
levels, header = EN_LEVELS, EN_HEADER
else:
levels, header = JA_LEVELS, JA_HEADER
rank = {level: index for index, level in enumerate(levels)}
table: dict[str, int] = {}
for source in sources:
with source.open(encoding="utf-8", newline="") as handle:
for row in csv.DictReader(handle):
if language == "en":
level = row["CEFR"].strip()
words = clean_english(row["headword"])
else:
tags = row["tags"].split()
# 文件名定级(n5.csv 里的就是 N5),tags 只用来兜底
match = re.search(r"n([1-5])", source.stem)
level = f"N{match.group(1)}" if match else next(
(t.replace("JLPT_N", "N") for t in tags if t.startswith("JLPT_N")), ""
)
words = clean_japanese(row["expression"])
if level not in rank:
continue
for word in words:
table[word] = min(table.get(word, rank[level]), rank[level])
return levels, table, header
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("language", choices=["en", "ja"])
parser.add_argument("sources", nargs="+", type=Path)
parser.add_argument("-o", "--output", type=Path, required=True)
args = parser.parse_args()
levels, table, header = convert(args.language, args.sources)
lines = header + ["# levels\t" + "\t".join(levels)]
for word, index in sorted(table.items(), key=lambda item: (item[1], item[0])):
lines.append(f"{word}\t{levels[index]}")
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text("\n".join(lines) + "\n", encoding="utf-8")
counts = {level: 0 for level in levels}
for index in table.values():
counts[levels[index]] += 1
print(f"{args.output}: {len(table)} 词 {counts}")
if __name__ == "__main__":
main()