github-mirrors/openclaw
1
0
Fork 0
mirror of https://fastgit.cc/github.com/openclaw/openclaw synced 2026-05-01 06:26:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(docs): allow partial i18n doc batches

Browse Source
This commit is contained in:
Vincent Koc
2026-04-29 18:19:32 -07:00
parent 5e8c396bb8
commit cab86dc325
2 changed files with 97 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
scripts/docs-i18n/main.go
View File

@@ -27,28 +27,30 @@ type docResult struct {
}
type runConfig struct {
targetLang string
sourceLang string
docsRoot string
tmPath string
mode string
thinking string
overwrite bool
maxFiles int
parallel int
targetLang string
sourceLang string
docsRoot string
tmPath string
mode string
thinking string
overwrite bool
allowPartial bool
maxFiles int
parallel int
}
func main() {
var (
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
sourceLang = flag.String("src", "en", "source language")
docsRoot = flag.String("docs", "docs", "docs root")
tmPath = flag.String("tm", "", "translation memory path")
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)")
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
sourceLang = flag.String("src", "en", "source language")
docsRoot = flag.String("docs", "docs", "docs root")
tmPath = flag.String("tm", "", "translation memory path")
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)")
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
allowPartial = flag.Bool("allow-partial", false, "write successful doc-mode outputs even when another file fails")
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
)
flag.Parse()
files := flag.Args()
@@ -57,15 +59,16 @@ func main() {
}
if err := runDocsI18N(context.Background(), runConfig{
targetLang: *targetLang,
sourceLang: *sourceLang,
docsRoot: *docsRoot,
tmPath: *tmPath,
mode: *mode,
thinking: *thinking,
overwrite: *overwrite,
maxFiles: *maxFiles,
parallel: *parallel,
targetLang: *targetLang,
sourceLang: *sourceLang,
docsRoot: *docsRoot,
tmPath: *tmPath,
mode: *mode,
thinking: *thinking,
overwrite: *overwrite,
allowPartial: *allowPartial,
maxFiles: *maxFiles,
parallel: *parallel,
}, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
return NewCodexTranslator(srcLang, tgtLang, glossary, thinking)
}); err != nil {
@@ -127,7 +130,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
processed := 0
skipped := 0
localizedFiles := []string{}
var runErr error
var translationErr error
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
switch cfg.mode {
@@ -138,7 +141,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
skipped += skip
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
translationErr = err
}
} else {
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
@@ -151,7 +154,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
skipped += skip
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
translationErr = err
}
}
case "segment":
@@ -167,21 +170,25 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
processed += proc
localizedFiles = append(localizedFiles, outputs...)
if err != nil {
runErr = err
translationErr = err
}
default:
return fmt.Errorf("unknown mode: %s", cfg.mode)
}
if err := tm.Save(); err != nil && runErr == nil {
runErr = err
if err := tm.Save(); err != nil {
return err
}
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil && runErr == nil {
runErr = err
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil {
return err
}
elapsed := time.Since(start).Round(time.Millisecond)
log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
return runErr
if translationErr != nil && cfg.allowPartial && cfg.mode == "doc" && processed > 0 {
log.Printf("docs-i18n: allowing partial doc output after translation error: %v", translationErr)
return nil
}
return translationErr
}
func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, []string, error) {

54
scripts/docs-i18n/main_test.go
View File

@@ -2,6 +2,8 @@ package main
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
@@ -49,6 +51,24 @@ func (transcriptFrontmatterTranslator) TranslateRaw(_ context.Context, text, _,
func (transcriptFrontmatterTranslator) Close() {}
type partialFailTranslator struct{}
func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
if strings.Contains(text, "FAIL") {
return "", errors.New("translation failed")
}
return text, nil
}
func (partialFailTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
if strings.Contains(text, "FAIL") {
return "", errors.New("translation failed")
}
return text, nil
}
func (partialFailTranslator) Close() {}
func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
t.Parallel()
@@ -99,6 +119,40 @@ func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
}
}
func TestRunDocsI18NAllowPartialKeepsSuccessfulDocOutputs(t *testing.T) {
t.Parallel()
docsRoot := t.TempDir()
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
okPath := filepath.Join(docsRoot, "aaa-ok.md")
failPath := filepath.Join(docsRoot, "zzz-fail.md")
writeFile(t, okPath, "# Gateway\n")
writeFile(t, failPath, "# FAIL\n")
err := runDocsI18N(context.Background(), runConfig{
targetLang: "zh-CN",
sourceLang: "en",
docsRoot: docsRoot,
mode: "doc",
thinking: "high",
overwrite: true,
allowPartial: true,
parallel: 1,
}, []string{okPath, failPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
return partialFailTranslator{}, nil
})
if err != nil {
t.Fatalf("runDocsI18N failed despite partial output: %v", err)
}
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-ok.md")); !strings.Contains(got, "# Gateway") {
t.Fatalf("expected successful output to be written, got:\n%s", got)
}
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-fail.md")); err == nil {
t.Fatal("did not expect failed output to be written")
}
}
func TestTranslateSnippetDoesNotCacheFallbackToSource(t *testing.T) {
t.Parallel()