fix(docs-i18n): recognize isolated indented fences

This commit is contained in:
Peter Steinberger
2026-07-13 12:26:01 -04:00
parent 8867a9da0b
commit a13796bcfd
5 changed files with 78 additions and 24 deletions
+31 -2
View File
@@ -773,6 +773,21 @@ func TestValidateDocChunkTranslationRejectsChangedTripleBacktickCodeSpan(t *test
}
}
func TestValidateDocChunkTranslationRejectsChangedLineStartTripleBacktickCodeSpan(t *testing.T) {
t.Parallel()
source := "```foo``` is the value.\n```\ncode\n```\n"
translated := "```bar``` es el valor.\n```\ncode\n```\n"
err := validateDocChunkTranslation(source, translated)
if err == nil {
t.Fatal("expected changed line-start triple-backtick code span to be rejected")
}
if !strings.Contains(err.Error(), "inline code mismatch") {
t.Fatalf("expected inline code mismatch, got %v", err)
}
}
func TestValidateDocChunkTranslationRejectsChangedMultilineCodeSpan(t *testing.T) {
t.Parallel()
@@ -896,6 +911,20 @@ func TestValidateDocChunkTranslationRejectsCodeAfterComponentFence(t *testing.T)
}
}
func TestValidateDocChunkTranslationAllowsTranslatedProseInIsolatedIndentedFence(t *testing.T) {
t.Parallel()
source := " ```json5\n provider: \"firecrawl\", // optional; omit for auto-detect\n ```\n"
translated := " ```json5\n provider: \"firecrawl\", // необязательно; опустите для автоопределения\n ```\n"
if values := extractMarkdownInlineCodeValues(source); len(values) != 0 {
t.Fatalf("expected custom indented fence to be excluded from inline code, got %q", values)
}
if err := validateDocChunkTranslation(source, translated); err != nil {
t.Fatalf("expected translated fence prose to validate, got %v", err)
}
}
func TestValidateDocChunkTranslationRejectsChangedCodeInSplitComponentBody(t *testing.T) {
t.Parallel()
@@ -2296,8 +2325,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
t.Fatalf("expected translated read_when entry in output:\n%s", text)
}
if !strings.Contains(text, "prompt_version: 23") {
t.Fatalf("expected prompt version 23 in output metadata:\n%s", text)
if !strings.Contains(text, "prompt_version: 24") {
t.Fatalf("expected prompt version 24 in output metadata:\n%s", text)
}
}
+33 -18
View File
@@ -183,6 +183,7 @@ func markdownListParentItemPath(list *ast.List) string {
func extractMarkdownInlineCodeValues(body string) []string {
parseSource := []byte(normalizeDocComponentsForMarkdownParse(body))
fencedRanges := markdownClosedLiteralFenceByteRanges(string(parseSource))
doc := goldmark.New(goldmark.WithExtensions(extension.GFM)).Parser().Parse(text.NewReader(parseSource))
values := []string{}
_ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
@@ -191,6 +192,9 @@ func extractMarkdownInlineCodeValues(body string) []string {
}
span, ok := node.(*ast.CodeSpan)
if ok {
if byteRange, found := markdownCodeSpanContentRange(span); found && rangeOverlapsAny(byteRange, fencedRanges) {
return ast.WalkContinue, nil
}
values = append(values, string(span.Text(parseSource)))
}
return ast.WalkContinue, nil
@@ -319,8 +323,12 @@ func parseMarkdownLiteralFenceOpening(line string) (markdownLiteralFenceState, b
if delimiter == "" {
return markdownLiteralFenceState{}, false
}
infoText := strings.TrimSpace(remaining[len(delimiter):])
if delimiter[0] == '`' && strings.Contains(infoText, "`") {
return markdownLiteralFenceState{}, false
}
info := ""
if fields := strings.Fields(strings.TrimSpace(remaining[len(delimiter):])); len(fields) > 0 {
if fields := strings.Fields(infoText); len(fields) > 0 {
info = strings.ToLower(fields[0])
}
return markdownLiteralFenceState{delimiter: delimiter, quoteDepth: quoteDepth, info: info, containerIndent: containerIndent}, true
@@ -620,7 +628,7 @@ func fencedMarkerName(value string) (string, bool) {
}
func extractFallbackBacktickValues(body string) []string {
fenced := markdownFencedCodeRanges(body)
fenced := append(markdownFencedCodeRanges(body), markdownClosedLiteralFenceByteRanges(body)...)
values := []string{}
for _, span := range markdownBlockBacktickRanges(body) {
if rangeOverlapsAny(span, fenced) {
@@ -638,6 +646,27 @@ func extractFallbackBacktickValues(body string) []string {
return values
}
func markdownCodeSpanContentRange(span *ast.CodeSpan) ([2]int, bool) {
start, end := -1, -1
for child := span.FirstChild(); child != nil; child = child.NextSibling() {
textNode, ok := child.(*ast.Text)
if !ok {
continue
}
segment := textNode.Segment
if start < 0 || segment.Start < start {
start = segment.Start
}
if segment.Stop > end {
end = segment.Stop
}
}
if start < 0 || end < start {
return [2]int{}, false
}
return [2]int{start, end}, true
}
func markdownFencedCodeRanges(body string) [][2]int {
source := []byte(body)
doc := goldmark.New(goldmark.WithExtensions(extension.GFM)).Parser().Parse(text.NewReader(source))
@@ -869,22 +898,8 @@ func markdownCodeSpanRanges(body string) [][2]int {
if !ok {
return ast.WalkContinue, nil
}
start, end := -1, -1
for child := span.FirstChild(); child != nil; child = child.NextSibling() {
textNode, ok := child.(*ast.Text)
if !ok {
continue
}
segment := textNode.Segment
if start < 0 || segment.Start < start {
start = segment.Start
}
if segment.Stop > end {
end = segment.Stop
}
}
if start >= 0 && end >= start {
ranges = append(ranges, [2]int{start, end})
if byteRange, found := markdownCodeSpanContentRange(span); found {
ranges = append(ranges, byteRange)
}
return ast.WalkContinue, nil
})
+12 -2
View File
@@ -114,6 +114,14 @@ func maskMarkdownDocSyntax(text string, nextPlaceholder func() string, placehold
}
func markdownLiteralFenceByteRanges(text string) [][2]int {
return markdownLiteralFenceByteRangesWithMode(text, true)
}
func markdownClosedLiteralFenceByteRanges(text string) [][2]int {
return markdownLiteralFenceByteRangesWithMode(text, false)
}
func markdownLiteralFenceByteRangesWithMode(text string, includeUnclosed bool) [][2]int {
ranges := make([][2]int, 0)
state := markdownLiteralFenceState{}
start := -1
@@ -129,7 +137,9 @@ func markdownLiteralFenceByteRanges(text string) [][2]int {
offset += len(line)
continue
}
ranges = append(ranges, [2]int{start, offset})
if includeUnclosed {
ranges = append(ranges, [2]int{start, offset})
}
state = markdownLiteralFenceState{}
start = -1
}
@@ -139,7 +149,7 @@ func markdownLiteralFenceByteRanges(text string) [][2]int {
}
offset += len(line)
}
if state.delimiter != "" {
if includeUnclosed && state.delimiter != "" {
ranges = append(ranges, [2]int{start, len(text)})
}
return ranges
+1 -1
View File
@@ -12,7 +12,7 @@ import (
const (
workflowVersion = 16
promptVersion = 23
promptVersion = 24
docsI18nEngineName = "codex"
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"
+1 -1
View File
@@ -8,7 +8,7 @@ import (
func TestCacheNamespaceIncludesPromptVersion(t *testing.T) {
t.Parallel()
if want := "prompt=23"; !strings.Contains(cacheNamespace(), want) {
if want := "prompt=24"; !strings.Contains(cacheNamespace(), want) {
t.Fatalf("expected cache namespace to contain %q, got %q", want, cacheNamespace())
}
}