ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(docs-i18n): normalize added list containers

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-23 09:15:48 -04:00
parent 856f534880
commit bd1ab1ef67
3 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/docs-i18n/doc_chunked_raw.go
+5 -3
View File
@@ -128,7 +128,7 @@ func validateDocBodyFencedLiterals(source, translated string) error {
return nil
}
func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) {
source := strings.Join(blocks, "")
if strings.TrimSpace(source) == "" {
return source, nil
@@ -147,6 +147,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
translated = sanitizeDocChunkProtocolWrappers(source, translated)
translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
translated = reapplyCommonIndent(translated, commonIndent)
translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders)
translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil {
log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated))
@@ -176,7 +177,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
return "", fmt.Errorf("%s: %w", chunkID, err)
}
func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) {
sourceStructure := summarizeDocChunkStructure(source)
if sourceStructure.fenceCount != 0 {
return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID)
@@ -197,6 +198,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk
translated = sanitizeDocChunkProtocolWrappers(source, translated)
translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
translated = reapplyCommonIndent(translated, commonIndent)
translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders)
translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
return "", validationErr
@@ -737,7 +739,7 @@ func containsProtocolWrapperToken(text string) bool {
return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart))
}
func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) {
func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) {
var out strings.Builder
translatedGroups := make([]string, 0, len(groups))
for index, group := range groups {
scripts/docs-i18n/doc_mode_test.go
+6 -2
View File
@@ -294,7 +294,7 @@ func (t *docSyntaxMaskingTranslator) Translate(_ context.Context, text, _, _ str
func (t *docSyntaxMaskingTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
t.rawInputs = append(t.rawInputs, text)
translated := strings.ReplaceAll(text, "Visible prose", "Видимый текст")
translated = regexp.MustCompile(`(?m)^(__OC_I18N_\d+__)`).ReplaceAllString(translated, " $1")
translated = regexp.MustCompile(`(?m)^(__OC_I18N_\d+__)`).ReplaceAllString(translated, " 1. $1")
return translated, nil
}
@@ -758,6 +758,8 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing.
" __OC_I18N_000001__Top level",
"> __OC_I18N_000002__Nested",
" > __OC_I18N_000003__Quoted",
"1. __OC_I18N_000001__Numbered wrapper",
" - __OC_I18N_000002__Bullet wrapper",
" __OC_I18N_000004__ prose",
"",
}, "\n")
@@ -765,6 +767,8 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing.
"__OC_I18N_000001__Top level",
"__OC_I18N_000002__Nested",
"__OC_I18N_000003__Quoted",
"__OC_I18N_000001__Numbered wrapper",
"__OC_I18N_000002__Bullet wrapper",
" __OC_I18N_000004__ prose",
"",
}, "\n")
@@ -800,7 +804,7 @@ func TestEscapeUnexpectedMarkdownListMarkersPreservesFencedExamples(t *testing.T
"",
}, "\n")
if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]struct{}{"__OC_I18N_000001__": {}}); got != want {
if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]string{"__OC_I18N_000001__": "- "}); got != want {
t.Fatalf("unexpected escaped list markers:\n%s\nwant:\n%s", got, want)
}
}
scripts/docs-i18n/masking.go
+5 -5
View File
@@ -14,7 +14,7 @@ var (
linkLabelRe = regexp.MustCompile(`!?\[([^\]\r\n]+)\]\(([^)\r\n]+)\)`)
placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`)
listMarkerRe = regexp.MustCompile(`^([ \t]*(?:>[ \t]*)*)([-+*]|[0-9]+[.)])([ \t]+)`)
listContainerPrefixRe = regexp.MustCompile(`^[ \t]*(?:>[ \t]*)*$`)
listContainerPrefixRe = regexp.MustCompile(`^[ \t]*(?:(?:>[ \t]*)|(?:(?:[-+*]|[0-9]+[.)])[ \t]+))*$`)
// Hard validation stays limited to low-ambiguity composite literals. Plain numbers remain
// model-visible so target-language plurals and ordinals can change grammar without false failures.
numericValueRe = regexp.MustCompile(`(?:0[xX][0-9A-Za-z_]+|0[bB][0-9A-Za-z_]+|0[oO][0-9A-Za-z_]+|[0-9]+(?:\.[0-9]+)?(?::[0-9]+(?:\.[0-9]+)?)+|[0-9]+(?:\.[0-9]+)?(?:/[0-9]+(?:\.[0-9]+)?)+|(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)[eE][+-]?[0-9]+)`)
@@ -150,18 +150,18 @@ func normalizeMaskedListMarkerPlaceholders(text string, mapping map[string]strin
return strings.Join(lines, "")
}
func maskedListMarkerPlaceholders(mapping map[string]string) map[string]struct{} {
placeholders := make(map[string]struct{})
func maskedListMarkerPlaceholders(mapping map[string]string) map[string]string {
placeholders := make(map[string]string)
for placeholder, original := range mapping {
markerSpan := listMarkerRe.FindStringIndex(original)
if markerSpan != nil && markerSpan[0] == 0 && markerSpan[1] == len(original) {
placeholders[placeholder] = struct{}{}
placeholders[placeholder] = original
}
}
return placeholders
}
func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]struct{}) string {
func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]string) string {
ranges := markdownListMarkerRanges(text)
if len(ranges) == 0 {
return text