mirror of
https://github.com/turnstonelabs/turnstone.git
synced 2026-08-12 23:12:23 -06:00
fix: reduce metacognition false positives with strong/weak pattern tiers (#76)
* fix: reduce metacognition false positives with strong/weak pattern tiers
Correction detection: split "no" handling — "no," and "no." are strong
(always fire), "no <word>" uses an allowlist of correction-context words
(pronouns, demonstratives, verbs) instead of a blocklist. Phrases like
"no problem", "no worries", "no rush" are excluded automatically.
Completion detection: move most patterns to weak tier, gated by message
length (<80 chars) and absence of continuation markers ("?", "can you",
"but", "now", "please", etc.). "thanks for X" excluded at regex level.
Strong tier (always fire): "that's all", "lgtm".
* fix: align allowlist comment with implementation (include articles)
This commit is contained in:
+116
-9
@@ -14,15 +14,16 @@ from turnstone.core.metacognition import (
|
||||
|
||||
|
||||
class TestDetectCorrection:
|
||||
"""Strong patterns always fire; weak 'no <word>' uses allowlist."""
|
||||
|
||||
# -- strong patterns (always fire) --
|
||||
|
||||
def test_no_comma(self):
|
||||
assert detect_correction("no, that's wrong") is True
|
||||
|
||||
def test_no_period(self):
|
||||
assert detect_correction("no. do it differently") is True
|
||||
|
||||
def test_no_space(self):
|
||||
assert detect_correction("no I meant the other one") is True
|
||||
|
||||
def test_dont(self):
|
||||
assert detect_correction("don't use tabs") is True
|
||||
|
||||
@@ -47,6 +48,57 @@ class TestDetectCorrection:
|
||||
def test_please_dont(self):
|
||||
assert detect_correction("please don't mock the database") is True
|
||||
|
||||
# -- weak pattern: "no" + allowlisted context word --
|
||||
|
||||
def test_no_space(self):
|
||||
assert detect_correction("no I meant the other one") is True
|
||||
|
||||
def test_no_that(self):
|
||||
assert detect_correction("no that's wrong") is True
|
||||
|
||||
def test_no_it(self):
|
||||
assert detect_correction("no it should be different") is True
|
||||
|
||||
def test_no_the(self):
|
||||
assert detect_correction("no the other one") is True
|
||||
|
||||
def test_no_not(self):
|
||||
assert detect_correction("no not that file") is True
|
||||
|
||||
def test_no_you(self):
|
||||
assert detect_correction("no you should use pytest") is True
|
||||
|
||||
# -- negatives: "no <word>" not in allowlist --
|
||||
|
||||
def test_negative_no_problem(self):
|
||||
assert detect_correction("no problem") is False
|
||||
|
||||
def test_negative_no_worries(self):
|
||||
assert detect_correction("no worries") is False
|
||||
|
||||
def test_negative_no_rush(self):
|
||||
assert detect_correction("no rush") is False
|
||||
|
||||
def test_negative_no_one(self):
|
||||
assert detect_correction("no one knows") is False
|
||||
|
||||
def test_negative_no_thanks(self):
|
||||
assert detect_correction("no thanks") is False
|
||||
|
||||
def test_negative_no_doubt(self):
|
||||
assert detect_correction("no doubt about it") is False
|
||||
|
||||
def test_negative_no_idea(self):
|
||||
assert detect_correction("no idea what you mean") is False
|
||||
|
||||
def test_negative_no_kidding(self):
|
||||
assert detect_correction("no kidding") is False
|
||||
|
||||
def test_negative_no_luck(self):
|
||||
assert detect_correction("no luck finding the bug") is False
|
||||
|
||||
# -- negatives: unrelated messages --
|
||||
|
||||
def test_negative_notice(self):
|
||||
assert detect_correction("I noticed the test passes") is False
|
||||
|
||||
@@ -70,27 +122,82 @@ class TestDetectCorrection:
|
||||
|
||||
|
||||
class TestDetectCompletion:
|
||||
def test_thanks(self):
|
||||
assert detect_completion("thanks, that's perfect") is True
|
||||
"""Strong patterns always fire; weak patterns gated by length + continuation."""
|
||||
|
||||
# -- strong patterns (always fire) --
|
||||
|
||||
def test_thats_all(self):
|
||||
assert detect_completion("that's all for now") is True
|
||||
|
||||
def test_lgtm(self):
|
||||
assert detect_completion("lgtm") is True
|
||||
|
||||
# -- weak patterns: short message, no continuation --
|
||||
|
||||
def test_thanks(self):
|
||||
assert detect_completion("thanks, that's perfect") is True
|
||||
|
||||
def test_thanks_standalone(self):
|
||||
assert detect_completion("thanks") is True
|
||||
|
||||
def test_thanks_exclaim(self):
|
||||
assert detect_completion("thanks!") is True
|
||||
|
||||
def test_looks_good(self):
|
||||
assert detect_completion("looks good to me") is True
|
||||
|
||||
def test_perfect(self):
|
||||
assert detect_completion("perfect") is True
|
||||
|
||||
def test_lgtm(self):
|
||||
assert detect_completion("lgtm") is True
|
||||
|
||||
def test_done(self):
|
||||
assert detect_completion("done") is True
|
||||
|
||||
def test_negative_normal(self):
|
||||
def test_great_job(self):
|
||||
assert detect_completion("great job") is True
|
||||
|
||||
def test_that_works(self):
|
||||
assert detect_completion("that works") is True
|
||||
|
||||
# -- negatives: "thanks for" is acknowledgment --
|
||||
|
||||
def test_negative_thanks_for(self):
|
||||
assert detect_completion("thanks for the update") is False
|
||||
|
||||
def test_negative_thanks_for_looking(self):
|
||||
assert detect_completion("thanks for looking into this") is False
|
||||
|
||||
# -- negatives: continuation markers suppress weak patterns --
|
||||
|
||||
def test_negative_thanks_but(self):
|
||||
assert detect_completion("thanks but can you also add tests") is False
|
||||
|
||||
def test_negative_thanks_though(self):
|
||||
assert detect_completion("thanks though I have one more question") is False
|
||||
|
||||
def test_negative_looks_good_but(self):
|
||||
assert detect_completion("looks good but can you also add validation") is False
|
||||
|
||||
def test_negative_perfect_now(self):
|
||||
assert detect_completion("perfect, now add error handling") is False
|
||||
|
||||
def test_negative_done_can_you(self):
|
||||
assert detect_completion("done with that, can you start on the tests?") is False
|
||||
|
||||
def test_negative_question_mark(self):
|
||||
assert detect_completion("can you add error handling?") is False
|
||||
|
||||
# -- negatives: long messages suppress weak patterns --
|
||||
|
||||
def test_negative_thanks_long(self):
|
||||
msg = "thanks, this is really helpful — I was also wondering about the deployment pipeline and whether we need to update the CI config"
|
||||
assert detect_completion(msg) is False
|
||||
|
||||
def test_negative_looks_good_long(self):
|
||||
msg = "looks good overall, there are a few things I'd like to tweak though — the error messages could be more descriptive and the retry logic needs a backoff"
|
||||
assert detect_completion(msg) is False
|
||||
|
||||
# -- negatives: unrelated --
|
||||
|
||||
def test_negative_empty(self):
|
||||
assert detect_completion("") is False
|
||||
|
||||
|
||||
@@ -53,11 +53,16 @@ _NUDGE_MAP: dict[str, str] = {
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Detection heuristics
|
||||
# Detection heuristics — strong/weak tiers
|
||||
#
|
||||
# Strong patterns fire unconditionally. Weak patterns carry inherent
|
||||
# ambiguity ("no …", "thanks …") and only fire when the surrounding
|
||||
# message looks like a genuine correction/completion rather than normal
|
||||
# conversation.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_CORRECTION_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)^no[,.\s]"),
|
||||
_STRONG_CORRECTION: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)^no[,.]"), # "no," / "no." — clear rejection
|
||||
re.compile(r"(?i)\bdon'?t\b"),
|
||||
re.compile(r"(?i)^stop\b"),
|
||||
re.compile(r"(?i)^actually[,\s]"),
|
||||
@@ -71,30 +76,64 @@ _CORRECTION_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)^please don'?t\b"),
|
||||
]
|
||||
|
||||
_COMPLETION_PATTERNS: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)^thanks\b"),
|
||||
# "no <word>" is ambiguous — only match when the next word is a pronoun,
|
||||
# demonstrative, article, or verb that signals the user is redirecting,
|
||||
# not a fixed phrase like "no problem" or "no worries". Allowlist >
|
||||
# blocklist: we don't need to enumerate every benign "no X" phrase.
|
||||
_WEAK_CORRECTION: list[re.Pattern[str]] = [
|
||||
re.compile(
|
||||
r"(?i)^no\s+(?:I\b|you\b|we\b|they\b|it\b|he\b|she\b"
|
||||
r"|that\b|this\b|those\b|these\b"
|
||||
r"|the\b|a\b|an\b"
|
||||
r"|not\b|do\b|did\b|but\b)"
|
||||
),
|
||||
]
|
||||
|
||||
_STRONG_COMPLETION: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)\bthat'?s all\b"),
|
||||
re.compile(r"(?i)^lgtm\b"),
|
||||
]
|
||||
|
||||
# These patterns are common in both completion AND mid-conversation
|
||||
# acknowledgment. Only fire when the message is short and has no
|
||||
# continuation markers (question marks, follow-up requests).
|
||||
_WEAK_COMPLETION: list[re.Pattern[str]] = [
|
||||
re.compile(r"(?i)^thanks\b(?!\s+for\b)"), # "thanks for X" = acknowledgment
|
||||
re.compile(r"(?i)\blooks good\b"),
|
||||
re.compile(r"(?i)^perfect\b"),
|
||||
re.compile(r"(?i)^great job\b"),
|
||||
re.compile(r"(?i)\bthat works\b"),
|
||||
re.compile(r"(?i)^done\b"),
|
||||
re.compile(r"(?i)^lgtm\b"),
|
||||
]
|
||||
|
||||
_WEAK_MSG_CAP = 80 # weak completion patterns suppressed above this length
|
||||
|
||||
_CONTINUATION = re.compile(
|
||||
r"(?i)(?:\?|(?:can you|could you|please\s|also\s|but\s|now\s|next\s"
|
||||
r"|and\s+then|after\s+that|one\s+more|however))"
|
||||
)
|
||||
|
||||
|
||||
def detect_correction(message: str) -> bool:
|
||||
"""Return True if the message looks like a user correction."""
|
||||
if not message:
|
||||
return False
|
||||
return any(p.search(message) for p in _CORRECTION_PATTERNS)
|
||||
if any(p.search(message) for p in _STRONG_CORRECTION):
|
||||
return True
|
||||
return any(p.search(message) for p in _WEAK_CORRECTION)
|
||||
|
||||
|
||||
def detect_completion(message: str) -> bool:
|
||||
"""Return True if the message signals session completion."""
|
||||
if not message:
|
||||
return False
|
||||
return any(p.search(message) for p in _COMPLETION_PATTERNS)
|
||||
if any(p.search(message) for p in _STRONG_COMPLETION):
|
||||
return True
|
||||
if len(message) > _WEAK_MSG_CAP:
|
||||
return False
|
||||
if _CONTINUATION.search(message):
|
||||
return False
|
||||
return any(p.search(message) for p in _WEAK_COMPLETION)
|
||||
|
||||
|
||||
def should_nudge(
|
||||
|
||||
Reference in New Issue
Block a user