diff --git a/scripts/docs-i18n/doc_chunked_raw.go b/scripts/docs-i18n/doc_chunked_raw.go index 5d415d731a39..72131599bf61 100644 --- a/scripts/docs-i18n/doc_chunked_raw.go +++ b/scripts/docs-i18n/doc_chunked_raw.go @@ -128,7 +128,7 @@ func validateDocBodyFencedLiterals(source, translated string) error { return nil } -func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { +func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) { source := strings.Join(blocks, "") if strings.TrimSpace(source) == "" { return source, nil @@ -147,6 +147,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun translated = sanitizeDocChunkProtocolWrappers(source, translated) translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) translated = reapplyCommonIndent(translated, commonIndent) + translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders) translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil { log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated)) @@ -176,7 +177,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun return "", fmt.Errorf("%s: %w", chunkID, err) } -func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { +func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunkID, source string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) { sourceStructure := summarizeDocChunkStructure(source) if sourceStructure.fenceCount != 0 { return "", fmt.Errorf("%s: raw leaf fallback not applicable", chunkID) @@ -197,6 +198,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk translated = sanitizeDocChunkProtocolWrappers(source, translated) translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) translated = reapplyCommonIndent(translated, commonIndent) + translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders) translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil { return "", validationErr @@ -737,7 +739,7 @@ func containsProtocolWrapperToken(text string) bool { return strings.Contains(lower, strings.ToLower(bodyTagStart)) || strings.Contains(lower, strings.ToLower(frontmatterTagStart)) } -func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]struct{}, srcLang, tgtLang string) (string, error) { +func translatePlannedDocChunkGroups(ctx context.Context, translator docsTranslator, chunkID, source string, groups [][]string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) { var out strings.Builder translatedGroups := make([]string, 0, len(groups)) for index, group := range groups { diff --git a/scripts/docs-i18n/doc_mode_test.go b/scripts/docs-i18n/doc_mode_test.go index 077825e10daf..43dbffd45858 100644 --- a/scripts/docs-i18n/doc_mode_test.go +++ b/scripts/docs-i18n/doc_mode_test.go @@ -294,7 +294,7 @@ func (t *docSyntaxMaskingTranslator) Translate(_ context.Context, text, _, _ str func (t *docSyntaxMaskingTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) { t.rawInputs = append(t.rawInputs, text) translated := strings.ReplaceAll(text, "Visible prose", "Видимый текст") - translated = regexp.MustCompile(`(?m)^(__OC_I18N_\d+__)`).ReplaceAllString(translated, " $1") + translated = regexp.MustCompile(`(?m)^(__OC_I18N_\d+__)`).ReplaceAllString(translated, " 1. $1") return translated, nil } @@ -758,6 +758,8 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing. " __OC_I18N_000001__Top level", "> __OC_I18N_000002__Nested", " > __OC_I18N_000003__Quoted", + "1. __OC_I18N_000001__Numbered wrapper", + " - __OC_I18N_000002__Bullet wrapper", " __OC_I18N_000004__ prose", "", }, "\n") @@ -765,6 +767,8 @@ func TestNormalizeMaskedListMarkerPlaceholdersRemovesAddedContainers(t *testing. "__OC_I18N_000001__Top level", "__OC_I18N_000002__Nested", "__OC_I18N_000003__Quoted", + "__OC_I18N_000001__Numbered wrapper", + "__OC_I18N_000002__Bullet wrapper", " __OC_I18N_000004__ prose", "", }, "\n") @@ -800,7 +804,7 @@ func TestEscapeUnexpectedMarkdownListMarkersPreservesFencedExamples(t *testing.T "", }, "\n") - if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]struct{}{"__OC_I18N_000001__": {}}); got != want { + if got := escapeUnexpectedMarkdownListMarkers(translated, map[string]string{"__OC_I18N_000001__": "- "}); got != want { t.Fatalf("unexpected escaped list markers:\n%s\nwant:\n%s", got, want) } } diff --git a/scripts/docs-i18n/masking.go b/scripts/docs-i18n/masking.go index 05d42f03f608..0fc92e633116 100644 --- a/scripts/docs-i18n/masking.go +++ b/scripts/docs-i18n/masking.go @@ -14,7 +14,7 @@ var ( linkLabelRe = regexp.MustCompile(`!?\[([^\]\r\n]+)\]\(([^)\r\n]+)\)`) placeholderRe = regexp.MustCompile(`__OC_I18N_\d+__`) listMarkerRe = regexp.MustCompile(`^([ \t]*(?:>[ \t]*)*)([-+*]|[0-9]+[.)])([ \t]+)`) - listContainerPrefixRe = regexp.MustCompile(`^[ \t]*(?:>[ \t]*)*$`) + listContainerPrefixRe = regexp.MustCompile(`^[ \t]*(?:(?:>[ \t]*)|(?:(?:[-+*]|[0-9]+[.)])[ \t]+))*$`) // Hard validation stays limited to low-ambiguity composite literals. Plain numbers remain // model-visible so target-language plurals and ordinals can change grammar without false failures. numericValueRe = regexp.MustCompile(`(?:0[xX][0-9A-Za-z_]+|0[bB][0-9A-Za-z_]+|0[oO][0-9A-Za-z_]+|[0-9]+(?:\.[0-9]+)?(?::[0-9]+(?:\.[0-9]+)?)+|[0-9]+(?:\.[0-9]+)?(?:/[0-9]+(?:\.[0-9]+)?)+|(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+)[eE][+-]?[0-9]+)`) @@ -150,18 +150,18 @@ func normalizeMaskedListMarkerPlaceholders(text string, mapping map[string]strin return strings.Join(lines, "") } -func maskedListMarkerPlaceholders(mapping map[string]string) map[string]struct{} { - placeholders := make(map[string]struct{}) +func maskedListMarkerPlaceholders(mapping map[string]string) map[string]string { + placeholders := make(map[string]string) for placeholder, original := range mapping { markerSpan := listMarkerRe.FindStringIndex(original) if markerSpan != nil && markerSpan[0] == 0 && markerSpan[1] == len(original) { - placeholders[placeholder] = struct{}{} + placeholders[placeholder] = original } } return placeholders } -func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]struct{}) string { +func escapeUnexpectedMarkdownListMarkers(text string, listPlaceholders map[string]string) string { ranges := markdownListMarkerRanges(text) if len(ranges) == 0 { return text