fix(docs-i18n): stabilize split validation

This commit is contained in:
Peter Steinberger
2026-07-15 19:20:35 -04:00
parent 370b80b74d
commit 4d4b1762fc
3 changed files with 68 additions and 12 deletions
+48 -7
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"fmt"
"os"
"path/filepath"
"regexp"
@@ -783,7 +784,7 @@ func TestMaskMarkdownDocSyntaxPreservesCanonicalNestedBackticks(t *testing.T) {
}
}
func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.T) {
func TestMaskMarkdownDocSyntaxProtectsProductLinksInsideRawHTML(t *testing.T) {
t.Parallel()
source := strings.Join([]string{
@@ -800,11 +801,11 @@ func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.
mapping := map[string]string{}
masked := maskMarkdownDocSyntax(source, state.Next, &placeholders, mapping)
if strings.Contains(masked, "[Discord]") {
t.Fatalf("expected protected link label %q to be masked:\n%s", "Discord", masked)
if strings.Contains(masked, "[Discord](/channels/discord)") {
t.Fatalf("expected protected link %q to be masked:\n%s", "Discord", masked)
}
if strings.Contains(masked, "[Render](https://render.com/docs)") {
t.Fatalf("expected contextual product link label %q to be masked:\n%s", "Render", masked)
t.Fatalf("expected contextual product link %q to be masked:\n%s", "Render", masked)
}
if !strings.Contains(masked, "[Groups]") {
t.Fatalf("expected ordinary link label to remain translatable:\n%s", masked)
@@ -813,7 +814,30 @@ func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.
t.Fatalf("expected contextual ordinary-word label to remain translatable:\n%s", masked)
}
if restored := unmaskMarkdown(masked, placeholders, mapping); restored != source {
t.Fatalf("protected link-label round trip changed source:\n%s\nwant:\n%s", restored, source)
t.Fatalf("protected-link round trip changed source:\n%s\nwant:\n%s", restored, source)
}
}
func TestMaskMarkdownDocSyntaxKeepsProtectedLinkAssociationOpaque(t *testing.T) {
t.Parallel()
source := "Read [Slack](/channels/slack) and nearby Slack setup notes.\n"
state := NewPlaceholderState(source)
placeholders := []string{}
mapping := map[string]string{}
masked := maskMarkdownDocSyntax(source, state.Next, &placeholders, mapping)
if strings.Contains(masked, "[Slack]") || strings.Contains(masked, "/channels/slack") {
t.Fatalf("expected protected link label and destination to share one opaque placeholder:\n%s", masked)
}
if !strings.Contains(masked, "nearby Slack setup notes") {
t.Fatalf("expected ordinary surrounding product prose to remain visible:\n%s", masked)
}
if len(placeholders) != 1 || mapping[placeholders[0]] != "[Slack](/channels/slack)" {
t.Fatalf("unexpected protected-link placeholder mapping: placeholders=%v mapping=%v", placeholders, mapping)
}
if restored := unmaskMarkdown(masked, placeholders, mapping); restored != source {
t.Fatalf("protected-link round trip changed source:\n%s\nwant:\n%s", restored, source)
}
}
@@ -2418,8 +2442,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
t.Fatalf("expected translated read_when entry in output:\n%s", text)
}
if !strings.Contains(text, "prompt_version: 30") {
t.Fatalf("expected prompt version 30 in output metadata:\n%s", text)
if !strings.Contains(text, fmt.Sprintf("prompt_version: %d", promptVersion)) {
t.Fatalf("expected prompt version %d in output metadata:\n%s", promptVersion, text)
}
}
@@ -2516,6 +2540,23 @@ func TestExtractNumericValuesKeepsLowAmbiguityComposites(t *testing.T) {
}
}
func TestExtractNumericValuesKeepsClockCoreBeforeMeridiemSuffix(t *testing.T) {
t.Parallel()
if got := strings.Join(extractNumericValues("At 5am, meet again by 6:14am."), ","); got != "6:14" {
t.Fatalf("unexpected clock values: %q", got)
}
if err := validateDocChunkTranslation(
"At 5am, meet again by 6:14am.\n",
"सुबह 5 बजे मिलें और 6:14 बजे तक फिर मिलें।\n",
); err != nil {
t.Fatalf("expected detached translated clock suffix to preserve the numeric core: %v", err)
}
if got := extractNumericValues("Versions v6:14am and 6:14amx stay unprotected."); len(got) != 0 {
t.Fatalf("unexpected embedded clock values: %v", got)
}
}
func TestValidateDocChunkTranslationRejectsDroppedDuplicateLink(t *testing.T) {
t.Parallel()
+19 -4
View File
@@ -84,7 +84,7 @@ func maskMarkdownDocSyntax(text string, nextPlaceholder func() string, placehold
inlineRanges = append(inlineRanges, span)
}
}
inlineRanges = append(inlineRanges, protectedMarkdownLinkLabelRanges(text)...)
inlineRanges = append(inlineRanges, protectedMarkdownLinkRanges(text)...)
masked := maskByteRanges(text, inlineRanges, nextPlaceholder, placeholders, mapping)
listRanges := make([][2]int, 0)
@@ -118,7 +118,7 @@ func maskMarkdownDocSyntax(text string, nextPlaceholder func() string, placehold
return maskByteRanges(masked, listRanges, nextPlaceholder, placeholders, mapping)
}
func protectedMarkdownLinkLabelRanges(text string) [][2]int {
func protectedMarkdownLinkRanges(text string) [][2]int {
ranges := make([][2]int, 0)
for _, match := range linkLabelRe.FindAllStringSubmatchIndex(text, -1) {
if len(match) < 6 {
@@ -127,7 +127,9 @@ func protectedMarkdownLinkLabelRanges(text string) [][2]int {
label := text[match[2]:match[3]]
destination := markdownInlineLinkDestination(text[match[4]:match[5]])
if isProtectedProductLinkLabel(label, destination) {
ranges = append(ranges, [2]int{match[2], match[3]})
// Keep the protected label attached to its original destination even when
// recursive chunk retries isolate or recombine the surrounding prose.
ranges = append(ranges, [2]int{match[0], match[1]})
}
}
return ranges
@@ -154,7 +156,9 @@ func extractNumericValues(text string) []string {
values := make([]string, 0)
for _, span := range numericValueRe.FindAllStringIndex(text, -1) {
candidate := [2]int{span[0], span[1]}
if hasCompositeNumericLeadingContinuation(text, candidate[0]) || hasCompositeNumericContinuation(text, candidate[1]) || rangeOverlapsAny(candidate, protocolRanges) {
if hasCompositeNumericLeadingContinuation(text, candidate[0]) ||
(hasCompositeNumericContinuation(text, candidate[1]) && !hasClockMeridiemSuffix(text, candidate)) ||
rangeOverlapsAny(candidate, protocolRanges) {
continue
}
values = append(values, text[span[0]:span[1]])
@@ -162,6 +166,17 @@ func extractNumericValues(text string) []string {
return values
}
func hasClockMeridiemSuffix(text string, span [2]int) bool {
if !strings.Contains(text[span[0]:span[1]], ":") || span[1]+2 > len(text) {
return false
}
suffix := strings.ToLower(text[span[1] : span[1]+2])
if suffix != "am" && suffix != "pm" {
return false
}
return span[1]+2 == len(text) || !isCompositeNumericWordByte(text[span[1]+2])
}
func hasCompositeNumericLeadingContinuation(text string, position int) bool {
if position == 0 {
return false
+1 -1
View File
@@ -12,7 +12,7 @@ import (
const (
workflowVersion = 16
promptVersion = 30
promptVersion = 31
docsI18nEngineName = "codex"
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"