From cab86dc32541a664b8048f62bd47a0ada294fc41 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Apr 2026 18:19:32 -0700 Subject: [PATCH] fix(docs): allow partial i18n doc batches --- scripts/docs-i18n/main.go | 79 ++++++++++++++++++---------------- scripts/docs-i18n/main_test.go | 54 +++++++++++++++++++++++ 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/scripts/docs-i18n/main.go b/scripts/docs-i18n/main.go index 2f2323483af..79ce83e7a32 100644 --- a/scripts/docs-i18n/main.go +++ b/scripts/docs-i18n/main.go @@ -27,28 +27,30 @@ type docResult struct { } type runConfig struct { - targetLang string - sourceLang string - docsRoot string - tmPath string - mode string - thinking string - overwrite bool - maxFiles int - parallel int + targetLang string + sourceLang string + docsRoot string + tmPath string + mode string + thinking string + overwrite bool + allowPartial bool + maxFiles int + parallel int } func main() { var ( - targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)") - sourceLang = flag.String("src", "en", "source language") - docsRoot = flag.String("docs", "docs", "docs root") - tmPath = flag.String("tm", "", "translation memory path") - mode = flag.String("mode", "segment", "translation mode (segment|doc)") - thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)") - overwrite = flag.Bool("overwrite", false, "overwrite existing translations") - maxFiles = flag.Int("max", 0, "max files to process (0 = all)") - parallel = flag.Int("parallel", 1, "parallel workers for doc mode") + targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)") + sourceLang = flag.String("src", "en", "source language") + docsRoot = flag.String("docs", "docs", "docs root") + tmPath = flag.String("tm", "", "translation memory path") + mode = flag.String("mode", "segment", "translation mode (segment|doc)") + thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)") + overwrite = flag.Bool("overwrite", false, "overwrite existing translations") + allowPartial = flag.Bool("allow-partial", false, "write successful doc-mode outputs even when another file fails") + maxFiles = flag.Int("max", 0, "max files to process (0 = all)") + parallel = flag.Int("parallel", 1, "parallel workers for doc mode") ) flag.Parse() files := flag.Args() @@ -57,15 +59,16 @@ func main() { } if err := runDocsI18N(context.Background(), runConfig{ - targetLang: *targetLang, - sourceLang: *sourceLang, - docsRoot: *docsRoot, - tmPath: *tmPath, - mode: *mode, - thinking: *thinking, - overwrite: *overwrite, - maxFiles: *maxFiles, - parallel: *parallel, + targetLang: *targetLang, + sourceLang: *sourceLang, + docsRoot: *docsRoot, + tmPath: *tmPath, + mode: *mode, + thinking: *thinking, + overwrite: *overwrite, + allowPartial: *allowPartial, + maxFiles: *maxFiles, + parallel: *parallel, }, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) { return NewCodexTranslator(srcLang, tgtLang, glossary, thinking) }); err != nil { @@ -127,7 +130,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat processed := 0 skipped := 0 localizedFiles := []string{} - var runErr error + var translationErr error log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel) switch cfg.mode { @@ -138,7 +141,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat skipped += skip localizedFiles = append(localizedFiles, outputs...) if err != nil { - runErr = err + translationErr = err } } else { translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking) @@ -151,7 +154,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat skipped += skip localizedFiles = append(localizedFiles, outputs...) if err != nil { - runErr = err + translationErr = err } } case "segment": @@ -167,21 +170,25 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat processed += proc localizedFiles = append(localizedFiles, outputs...) if err != nil { - runErr = err + translationErr = err } default: return fmt.Errorf("unknown mode: %s", cfg.mode) } - if err := tm.Save(); err != nil && runErr == nil { - runErr = err + if err := tm.Save(); err != nil { + return err } - if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil && runErr == nil { - runErr = err + if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil { + return err } elapsed := time.Since(start).Round(time.Millisecond) log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed) - return runErr + if translationErr != nil && cfg.allowPartial && cfg.mode == "doc" && processed > 0 { + log.Printf("docs-i18n: allowing partial doc output after translation error: %v", translationErr) + return nil + } + return translationErr } func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, []string, error) { diff --git a/scripts/docs-i18n/main_test.go b/scripts/docs-i18n/main_test.go index b1f21cc86ea..4427d5c0b3a 100644 --- a/scripts/docs-i18n/main_test.go +++ b/scripts/docs-i18n/main_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "errors" + "os" "path/filepath" "strings" "testing" @@ -49,6 +51,24 @@ func (transcriptFrontmatterTranslator) TranslateRaw(_ context.Context, text, _, func (transcriptFrontmatterTranslator) Close() {} +type partialFailTranslator struct{} + +func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) { + if strings.Contains(text, "FAIL") { + return "", errors.New("translation failed") + } + return text, nil +} + +func (partialFailTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) { + if strings.Contains(text, "FAIL") { + return "", errors.New("translation failed") + } + return text, nil +} + +func (partialFailTranslator) Close() {} + func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) { t.Parallel() @@ -99,6 +119,40 @@ func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) { } } +func TestRunDocsI18NAllowPartialKeepsSuccessfulDocOutputs(t *testing.T) { + t.Parallel() + + docsRoot := t.TempDir() + writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]") + writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`) + okPath := filepath.Join(docsRoot, "aaa-ok.md") + failPath := filepath.Join(docsRoot, "zzz-fail.md") + writeFile(t, okPath, "# Gateway\n") + writeFile(t, failPath, "# FAIL\n") + + err := runDocsI18N(context.Background(), runConfig{ + targetLang: "zh-CN", + sourceLang: "en", + docsRoot: docsRoot, + mode: "doc", + thinking: "high", + overwrite: true, + allowPartial: true, + parallel: 1, + }, []string{okPath, failPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) { + return partialFailTranslator{}, nil + }) + if err != nil { + t.Fatalf("runDocsI18N failed despite partial output: %v", err) + } + if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-ok.md")); !strings.Contains(got, "# Gateway") { + t.Fatalf("expected successful output to be written, got:\n%s", got) + } + if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-fail.md")); err == nil { + t.Fatal("did not expect failed output to be written") + } +} + func TestTranslateSnippetDoesNotCacheFallbackToSource(t *testing.T) { t.Parallel()