mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(docs-i18n): stabilize split validation
This commit is contained in:
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@@ -783,7 +784,7 @@ func TestMaskMarkdownDocSyntaxPreservesCanonicalNestedBackticks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.T) {
|
||||
func TestMaskMarkdownDocSyntaxProtectsProductLinksInsideRawHTML(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := strings.Join([]string{
|
||||
@@ -800,11 +801,11 @@ func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.
|
||||
mapping := map[string]string{}
|
||||
masked := maskMarkdownDocSyntax(source, state.Next, &placeholders, mapping)
|
||||
|
||||
if strings.Contains(masked, "[Discord]") {
|
||||
t.Fatalf("expected protected link label %q to be masked:\n%s", "Discord", masked)
|
||||
if strings.Contains(masked, "[Discord](/channels/discord)") {
|
||||
t.Fatalf("expected protected link %q to be masked:\n%s", "Discord", masked)
|
||||
}
|
||||
if strings.Contains(masked, "[Render](https://render.com/docs)") {
|
||||
t.Fatalf("expected contextual product link label %q to be masked:\n%s", "Render", masked)
|
||||
t.Fatalf("expected contextual product link %q to be masked:\n%s", "Render", masked)
|
||||
}
|
||||
if !strings.Contains(masked, "[Groups]") {
|
||||
t.Fatalf("expected ordinary link label to remain translatable:\n%s", masked)
|
||||
@@ -813,7 +814,30 @@ func TestMaskMarkdownDocSyntaxProtectsProductLinkLabelsInsideRawHTML(t *testing.
|
||||
t.Fatalf("expected contextual ordinary-word label to remain translatable:\n%s", masked)
|
||||
}
|
||||
if restored := unmaskMarkdown(masked, placeholders, mapping); restored != source {
|
||||
t.Fatalf("protected link-label round trip changed source:\n%s\nwant:\n%s", restored, source)
|
||||
t.Fatalf("protected-link round trip changed source:\n%s\nwant:\n%s", restored, source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaskMarkdownDocSyntaxKeepsProtectedLinkAssociationOpaque(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
source := "Read [Slack](/channels/slack) and nearby Slack setup notes.\n"
|
||||
state := NewPlaceholderState(source)
|
||||
placeholders := []string{}
|
||||
mapping := map[string]string{}
|
||||
masked := maskMarkdownDocSyntax(source, state.Next, &placeholders, mapping)
|
||||
|
||||
if strings.Contains(masked, "[Slack]") || strings.Contains(masked, "/channels/slack") {
|
||||
t.Fatalf("expected protected link label and destination to share one opaque placeholder:\n%s", masked)
|
||||
}
|
||||
if !strings.Contains(masked, "nearby Slack setup notes") {
|
||||
t.Fatalf("expected ordinary surrounding product prose to remain visible:\n%s", masked)
|
||||
}
|
||||
if len(placeholders) != 1 || mapping[placeholders[0]] != "[Slack](/channels/slack)" {
|
||||
t.Fatalf("unexpected protected-link placeholder mapping: placeholders=%v mapping=%v", placeholders, mapping)
|
||||
}
|
||||
if restored := unmaskMarkdown(masked, placeholders, mapping); restored != source {
|
||||
t.Fatalf("protected-link round trip changed source:\n%s\nwant:\n%s", restored, source)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2418,8 +2442,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
|
||||
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
|
||||
t.Fatalf("expected translated read_when entry in output:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "prompt_version: 30") {
|
||||
t.Fatalf("expected prompt version 30 in output metadata:\n%s", text)
|
||||
if !strings.Contains(text, fmt.Sprintf("prompt_version: %d", promptVersion)) {
|
||||
t.Fatalf("expected prompt version %d in output metadata:\n%s", promptVersion, text)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2516,6 +2540,23 @@ func TestExtractNumericValuesKeepsLowAmbiguityComposites(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractNumericValuesKeepsClockCoreBeforeMeridiemSuffix(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := strings.Join(extractNumericValues("At 5am, meet again by 6:14am."), ","); got != "6:14" {
|
||||
t.Fatalf("unexpected clock values: %q", got)
|
||||
}
|
||||
if err := validateDocChunkTranslation(
|
||||
"At 5am, meet again by 6:14am.\n",
|
||||
"सुबह 5 बजे मिलें और 6:14 बजे तक फिर मिलें।\n",
|
||||
); err != nil {
|
||||
t.Fatalf("expected detached translated clock suffix to preserve the numeric core: %v", err)
|
||||
}
|
||||
if got := extractNumericValues("Versions v6:14am and 6:14amx stay unprotected."); len(got) != 0 {
|
||||
t.Fatalf("unexpected embedded clock values: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateDocChunkTranslationRejectsDroppedDuplicateLink(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ func maskMarkdownDocSyntax(text string, nextPlaceholder func() string, placehold
|
||||
inlineRanges = append(inlineRanges, span)
|
||||
}
|
||||
}
|
||||
inlineRanges = append(inlineRanges, protectedMarkdownLinkLabelRanges(text)...)
|
||||
inlineRanges = append(inlineRanges, protectedMarkdownLinkRanges(text)...)
|
||||
masked := maskByteRanges(text, inlineRanges, nextPlaceholder, placeholders, mapping)
|
||||
|
||||
listRanges := make([][2]int, 0)
|
||||
@@ -118,7 +118,7 @@ func maskMarkdownDocSyntax(text string, nextPlaceholder func() string, placehold
|
||||
return maskByteRanges(masked, listRanges, nextPlaceholder, placeholders, mapping)
|
||||
}
|
||||
|
||||
func protectedMarkdownLinkLabelRanges(text string) [][2]int {
|
||||
func protectedMarkdownLinkRanges(text string) [][2]int {
|
||||
ranges := make([][2]int, 0)
|
||||
for _, match := range linkLabelRe.FindAllStringSubmatchIndex(text, -1) {
|
||||
if len(match) < 6 {
|
||||
@@ -127,7 +127,9 @@ func protectedMarkdownLinkLabelRanges(text string) [][2]int {
|
||||
label := text[match[2]:match[3]]
|
||||
destination := markdownInlineLinkDestination(text[match[4]:match[5]])
|
||||
if isProtectedProductLinkLabel(label, destination) {
|
||||
ranges = append(ranges, [2]int{match[2], match[3]})
|
||||
// Keep the protected label attached to its original destination even when
|
||||
// recursive chunk retries isolate or recombine the surrounding prose.
|
||||
ranges = append(ranges, [2]int{match[0], match[1]})
|
||||
}
|
||||
}
|
||||
return ranges
|
||||
@@ -154,7 +156,9 @@ func extractNumericValues(text string) []string {
|
||||
values := make([]string, 0)
|
||||
for _, span := range numericValueRe.FindAllStringIndex(text, -1) {
|
||||
candidate := [2]int{span[0], span[1]}
|
||||
if hasCompositeNumericLeadingContinuation(text, candidate[0]) || hasCompositeNumericContinuation(text, candidate[1]) || rangeOverlapsAny(candidate, protocolRanges) {
|
||||
if hasCompositeNumericLeadingContinuation(text, candidate[0]) ||
|
||||
(hasCompositeNumericContinuation(text, candidate[1]) && !hasClockMeridiemSuffix(text, candidate)) ||
|
||||
rangeOverlapsAny(candidate, protocolRanges) {
|
||||
continue
|
||||
}
|
||||
values = append(values, text[span[0]:span[1]])
|
||||
@@ -162,6 +166,17 @@ func extractNumericValues(text string) []string {
|
||||
return values
|
||||
}
|
||||
|
||||
func hasClockMeridiemSuffix(text string, span [2]int) bool {
|
||||
if !strings.Contains(text[span[0]:span[1]], ":") || span[1]+2 > len(text) {
|
||||
return false
|
||||
}
|
||||
suffix := strings.ToLower(text[span[1] : span[1]+2])
|
||||
if suffix != "am" && suffix != "pm" {
|
||||
return false
|
||||
}
|
||||
return span[1]+2 == len(text) || !isCompositeNumericWordByte(text[span[1]+2])
|
||||
}
|
||||
|
||||
func hasCompositeNumericLeadingContinuation(text string, position int) bool {
|
||||
if position == 0 {
|
||||
return false
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
const (
|
||||
workflowVersion = 16
|
||||
promptVersion = 30
|
||||
promptVersion = 31
|
||||
docsI18nEngineName = "codex"
|
||||
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
|
||||
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"
|
||||
|
||||
Reference in New Issue
Block a user