mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 08:09:27 +00:00
## Summary
Replaces the AWS Translate i18n pipeline with a DeepSeek (LLM) backend,
standardizes locale files to BCP-47 naming with an `en-US` base, adds
new languages, and backfills **every locale to 100%**.
## What changed
### Translation engine (DeepSeek)
- Swap AWS Translate → DeepSeek via its OpenAI-compatible API
(`scripts/translations/`). Auth via the `DEEPSEEK_API_KEY` secret;
per-request 60s timeout.
- **Batched** translation (50/call) with **per-string validation**:
interpolation placeholders (`{count}`, `%s`, `@:key`), vue-i18n
**pluralization pipes** (`{n} Day \| {n} Days`), and literal escapes
(`{'@'}`, `{'{'}`, `{'\|'}`) are preserved or the string is rejected and
retried.
- Robust recovery: index-aligned batches, split-on-mismatch to isolate a
single bad string, temperature-perturbed retries, and rejection of
empty/whitespace output (never blanks a label). Removed the old
`TRANSLATION_MAX_KEYS` safety cap.
### Locale naming + base language
- All 15 locale files renamed to **BCP-47 hyphen** form (`en-US.json`,
`zh-CN.json`, `zh-TW.json`, `fr-FR.json`, …) — matching
`navigator.language` / `Intl` / HTML `lang`.
- **English base switched `en-gb` → `en-us`** (app code,
`fallbackLocale`, BCP-47 map, selector, defaults). British-Pound
currency and date-formatting `en-GB` usages left untouched. Existing
`en-gb` cookies fall back cleanly to English — no migration needed.
### New languages
- **Russian (`ru`), Polish (`pl`), Vietnamese (`vi`)** — fully generated
and wired into `localeFileMap`, `APP_LOCALE_TO_BCP47`, and the language
selector.
- **Traditional Chinese (`zh-TW`)** now maintained by the translator
alongside Simplified (`zh-CN`).
- RTL languages (`ar`, `fa`) intentionally **deferred** until the app
has `dir="rtl"` support.
### Full backfill (one-time)
- Every pre-existing locale filled from ~5.9k up to the full **8,269
keys** (~26.9k strings). `.translation_state.json` rebuilt complete, so
future runs are incremental.
### Nav i18n bug fix
- The **Dashboards** and **Data** nav group labels were hardcoded
English (bypassing i18n) and stayed English in every locale. Now
resolved via `t()` (`menu.dashboard`, new `menu.data`), localized and
re-resolving on language change.
## Verification
- Full web unit suite: **39,035 passed / 414 skipped / 0 failed** (1,167
files).
- `type-check` clean (0 TS errors).
- Locale registry sync spec green; all 15 files = 8,269 leaves.
- Translation engine verified live against `deepseek-v4-flash`.
## Notes
- Since all locales are now 100%, the post-merge `update-translations`
workflow will find nothing pending (no follow-up translation PR).
- `ar`/`fa` to be added with RTL support in a follow-up.
254 lines
10 KiB
YAML
254 lines
10 KiB
YAML
name: Auto-update Features JSON
|
|
|
|
on:
|
|
pull_request:
|
|
types: [opened, edited]
|
|
branches:
|
|
- main
|
|
|
|
permissions:
|
|
contents: write
|
|
pull-requests: write
|
|
|
|
jobs:
|
|
update-features:
|
|
runs-on: ubuntu-latest
|
|
|
|
steps:
|
|
- name: Checkout PR branch
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.ref }}
|
|
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Parse PR description and update features.json
|
|
uses: actions/github-script@v9
|
|
id: parse-feature
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// Get PR body
|
|
const prBody = context.payload.pull_request.body || '';
|
|
|
|
// Check if PR contains feature update section
|
|
const featureRegex = /<!-- FEATURE_UPDATE_START -->([\s\S]*?)<!-- FEATURE_UPDATE_END -->/;
|
|
const match = prBody.match(featureRegex);
|
|
|
|
if (!match) {
|
|
console.log('No feature update section found in PR description');
|
|
core.setOutput('has_feature', 'false');
|
|
return;
|
|
}
|
|
|
|
console.log('Feature update section found!');
|
|
core.setOutput('has_feature', 'true');
|
|
|
|
// Parse the feature data from the matched section
|
|
const featureSection = match[1].trim();
|
|
|
|
// Try to parse as JSON or YAML-like format
|
|
let featureData;
|
|
try {
|
|
// Try JSON first
|
|
const jsonMatch = featureSection.match(/```json\s*([\s\S]*?)\s*```/);
|
|
if (jsonMatch) {
|
|
featureData = JSON.parse(jsonMatch[1]);
|
|
} else {
|
|
// Parse key-value format
|
|
featureData = {};
|
|
const lines = featureSection.split('\n');
|
|
for (const line of lines) {
|
|
const kvMatch = line.match(/^\s*-?\s*\*?\*?(\w+)\*?\*?\s*:\s*(.+)$/);
|
|
if (kvMatch) {
|
|
const key = kvMatch[1].toLowerCase().replace(/\s+/g, '_');
|
|
let value = kvMatch[2].trim();
|
|
|
|
// Handle boolean values
|
|
if (value === 'true' || value === 'yes') value = true;
|
|
if (value === 'false' || value === 'no') value = false;
|
|
|
|
// Handle nested objects (availability)
|
|
if (key.startsWith('availability_')) {
|
|
if (!featureData.availability) featureData.availability = {};
|
|
const edition = key.replace('availability_', '');
|
|
featureData.availability[edition] = value;
|
|
} else {
|
|
featureData[key] = value;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log('Parsed feature data:', JSON.stringify(featureData, null, 2));
|
|
|
|
// Validate required fields
|
|
if (!featureData.id) {
|
|
throw new Error('Missing required field: id');
|
|
}
|
|
if (!featureData.availability) {
|
|
throw new Error('Missing required field: availability');
|
|
}
|
|
if (!featureData.availability.opensource ||
|
|
!featureData.availability.enterprise ||
|
|
!featureData.availability.cloud) {
|
|
throw new Error('Missing required availability fields (opensource, enterprise, cloud)');
|
|
}
|
|
|
|
// Read existing features.json
|
|
const featuresPath = path.join(process.env.GITHUB_WORKSPACE, 'web/src/constants/features.json');
|
|
const featuresContent = fs.readFileSync(featuresPath, 'utf8');
|
|
const featuresJson = JSON.parse(featuresContent);
|
|
|
|
// Check if feature already exists
|
|
const existingIndex = featuresJson.features.findIndex(f => f.id === featureData.id);
|
|
|
|
if (existingIndex >= 0) {
|
|
console.log(`Updating existing feature: ${featureData.id}`);
|
|
featuresJson.features[existingIndex] = featureData;
|
|
core.setOutput('action', 'updated');
|
|
} else {
|
|
console.log(`Adding new feature: ${featureData.id}`);
|
|
|
|
// Determine position
|
|
let position = featureData.position || 'end';
|
|
delete featureData.position; // Remove position field from feature data
|
|
|
|
if (position === 'start') {
|
|
featuresJson.features.unshift(featureData);
|
|
} else if (typeof position === 'number') {
|
|
featuresJson.features.splice(position, 0, featureData);
|
|
} else {
|
|
featuresJson.features.push(featureData);
|
|
}
|
|
core.setOutput('action', 'added');
|
|
}
|
|
|
|
// Write updated features.json
|
|
fs.writeFileSync(featuresPath, JSON.stringify(featuresJson, null, 2) + '\n', 'utf8');
|
|
|
|
core.setOutput('feature_id', featureData.id);
|
|
core.setOutput('success', 'true');
|
|
|
|
} catch (error) {
|
|
console.error('Error parsing feature data:', error.message);
|
|
core.setOutput('success', 'false');
|
|
core.setOutput('error', error.message);
|
|
throw error;
|
|
}
|
|
|
|
- name: Check for translation keys
|
|
if: steps.parse-feature.outputs.success == 'true'
|
|
id: check-translations
|
|
uses: actions/github-script@v9
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const featureId = '${{ steps.parse-feature.outputs.feature_id }}';
|
|
const expectedKey = `feature_${featureId}`;
|
|
|
|
// Check if translation exists in en-US.json
|
|
const enJsonPath = path.join(process.env.GITHUB_WORKSPACE, 'web/src/locales/languages/en-US.json');
|
|
const enJson = JSON.parse(fs.readFileSync(enJsonPath, 'utf8'));
|
|
|
|
const hasTranslation = enJson.about && enJson.about[expectedKey];
|
|
core.setOutput('has_translation', hasTranslation ? 'true' : 'false');
|
|
core.setOutput('translation_key', `about.${expectedKey}`);
|
|
|
|
- name: Commit changes
|
|
if: steps.parse-feature.outputs.success == 'true'
|
|
run: |
|
|
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
|
git config --local user.name "github-actions[bot]"
|
|
|
|
if git diff --quiet web/src/constants/features.json; then
|
|
echo "No changes to commit"
|
|
exit 0
|
|
fi
|
|
|
|
git add web/src/constants/features.json
|
|
git commit -m "chore: auto-update features.json from PR description
|
|
|
|
Feature ${{ steps.parse-feature.outputs.action }}: ${{ steps.parse-feature.outputs.feature_id }}
|
|
|
|
🤖 Automated update via GitHub Actions"
|
|
|
|
git push
|
|
|
|
- name: Comment on PR - Success
|
|
if: steps.parse-feature.outputs.success == 'true'
|
|
uses: actions/github-script@v9
|
|
with:
|
|
script: |
|
|
const featureId = '${{ steps.parse-feature.outputs.feature_id }}';
|
|
const action = '${{ steps.parse-feature.outputs.action }}';
|
|
const hasTranslation = '${{ steps.check-translations.outputs.has_translation }}' === 'true';
|
|
const translationKey = '${{ steps.check-translations.outputs.translation_key }}';
|
|
|
|
let message = `### ✅ Feature ${action === 'added' ? 'Added' : 'Updated'} Successfully!\n\n`;
|
|
message += `**Feature ID:** \`${featureId}\`\n`;
|
|
message += `**Action:** ${action}\n\n`;
|
|
message += `📝 **File Updated:** \`web/src/constants/features.json\`\n\n`;
|
|
|
|
if (!hasTranslation) {
|
|
message += `### ⚠️ Translation Missing\n\n`;
|
|
message += `Please add the translation key \`${translationKey}\` to:\n`;
|
|
message += `- \`web/src/locales/languages/en-US.json\`\n`;
|
|
message += `- All other language files\n\n`;
|
|
message += `Example for \`en-US.json\`:\n`;
|
|
message += `\`\`\`json\n`;
|
|
message += `{\n`;
|
|
message += ` "about": {\n`;
|
|
message += ` "feature_${featureId}": "Your Feature Name Here"\n`;
|
|
message += ` }\n`;
|
|
message += `}\n`;
|
|
message += `\`\`\`\n`;
|
|
} else {
|
|
message += `✅ Translation key \`${translationKey}\` found!\n`;
|
|
}
|
|
|
|
message += `\n---\n`;
|
|
message += `🤖 This feature was automatically added from your PR description.`;
|
|
|
|
github.rest.issues.createComment({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: message
|
|
});
|
|
|
|
- name: Comment on PR - Error
|
|
if: steps.parse-feature.outputs.success == 'false'
|
|
uses: actions/github-script@v9
|
|
with:
|
|
script: |
|
|
const error = '${{ steps.parse-feature.outputs.error }}';
|
|
|
|
const message = `### ❌ Feature Update Failed\n\n` +
|
|
`**Error:** ${error}\n\n` +
|
|
`Please check your feature definition format in the PR description.\n\n` +
|
|
`See the [Feature Update Guide](../../web/src/constants/FEATURES_JSON_GUIDE.md) for the correct format.`;
|
|
|
|
github.rest.issues.createComment({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: message
|
|
});
|
|
|
|
- name: Add label
|
|
if: steps.parse-feature.outputs.success == 'true'
|
|
uses: actions/github-script@v9
|
|
with:
|
|
script: |
|
|
github.rest.issues.addLabels({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
labels: ['feature-update', 'auto-generated']
|
|
});
|