fix: reduce metacognition false positives with strong/weak pattern tiers (#76)

* fix: reduce metacognition false positives with strong/weak pattern tiers

Correction detection: split "no" handling — "no," and "no." are strong
(always fire), "no <word>" uses an allowlist of correction-context words
(pronouns, demonstratives, verbs) instead of a blocklist. Phrases like
"no problem", "no worries", "no rush" are excluded automatically.

Completion detection: move most patterns to weak tier, gated by message
length (<80 chars) and absence of continuation markers ("?", "can you",
"but", "now", "please", etc.). "thanks for X" excluded at regex level.
Strong tier (always fire): "that's all", "lgtm".

* fix: align allowlist comment with implementation (include articles)
This commit is contained in:
Patrick Buckley
2026-03-15 13:53:39 -07:00
committed by GitHub
parent e603a6a7d1
commit 1d36d80fe5
2 changed files with 163 additions and 17 deletions
+116 -9
View File
@@ -14,15 +14,16 @@ from turnstone.core.metacognition import (
class TestDetectCorrection:
"""Strong patterns always fire; weak 'no <word>' uses allowlist."""
# -- strong patterns (always fire) --
def test_no_comma(self):
assert detect_correction("no, that's wrong") is True
def test_no_period(self):
assert detect_correction("no. do it differently") is True
def test_no_space(self):
assert detect_correction("no I meant the other one") is True
def test_dont(self):
assert detect_correction("don't use tabs") is True
@@ -47,6 +48,57 @@ class TestDetectCorrection:
def test_please_dont(self):
assert detect_correction("please don't mock the database") is True
# -- weak pattern: "no" + allowlisted context word --
def test_no_space(self):
assert detect_correction("no I meant the other one") is True
def test_no_that(self):
assert detect_correction("no that's wrong") is True
def test_no_it(self):
assert detect_correction("no it should be different") is True
def test_no_the(self):
assert detect_correction("no the other one") is True
def test_no_not(self):
assert detect_correction("no not that file") is True
def test_no_you(self):
assert detect_correction("no you should use pytest") is True
# -- negatives: "no <word>" not in allowlist --
def test_negative_no_problem(self):
assert detect_correction("no problem") is False
def test_negative_no_worries(self):
assert detect_correction("no worries") is False
def test_negative_no_rush(self):
assert detect_correction("no rush") is False
def test_negative_no_one(self):
assert detect_correction("no one knows") is False
def test_negative_no_thanks(self):
assert detect_correction("no thanks") is False
def test_negative_no_doubt(self):
assert detect_correction("no doubt about it") is False
def test_negative_no_idea(self):
assert detect_correction("no idea what you mean") is False
def test_negative_no_kidding(self):
assert detect_correction("no kidding") is False
def test_negative_no_luck(self):
assert detect_correction("no luck finding the bug") is False
# -- negatives: unrelated messages --
def test_negative_notice(self):
assert detect_correction("I noticed the test passes") is False
@@ -70,27 +122,82 @@ class TestDetectCorrection:
class TestDetectCompletion:
def test_thanks(self):
assert detect_completion("thanks, that's perfect") is True
"""Strong patterns always fire; weak patterns gated by length + continuation."""
# -- strong patterns (always fire) --
def test_thats_all(self):
assert detect_completion("that's all for now") is True
def test_lgtm(self):
assert detect_completion("lgtm") is True
# -- weak patterns: short message, no continuation --
def test_thanks(self):
assert detect_completion("thanks, that's perfect") is True
def test_thanks_standalone(self):
assert detect_completion("thanks") is True
def test_thanks_exclaim(self):
assert detect_completion("thanks!") is True
def test_looks_good(self):
assert detect_completion("looks good to me") is True
def test_perfect(self):
assert detect_completion("perfect") is True
def test_lgtm(self):
assert detect_completion("lgtm") is True
def test_done(self):
assert detect_completion("done") is True
def test_negative_normal(self):
def test_great_job(self):
assert detect_completion("great job") is True
def test_that_works(self):
assert detect_completion("that works") is True
# -- negatives: "thanks for" is acknowledgment --
def test_negative_thanks_for(self):
assert detect_completion("thanks for the update") is False
def test_negative_thanks_for_looking(self):
assert detect_completion("thanks for looking into this") is False
# -- negatives: continuation markers suppress weak patterns --
def test_negative_thanks_but(self):
assert detect_completion("thanks but can you also add tests") is False
def test_negative_thanks_though(self):
assert detect_completion("thanks though I have one more question") is False
def test_negative_looks_good_but(self):
assert detect_completion("looks good but can you also add validation") is False
def test_negative_perfect_now(self):
assert detect_completion("perfect, now add error handling") is False
def test_negative_done_can_you(self):
assert detect_completion("done with that, can you start on the tests?") is False
def test_negative_question_mark(self):
assert detect_completion("can you add error handling?") is False
# -- negatives: long messages suppress weak patterns --
def test_negative_thanks_long(self):
msg = "thanks, this is really helpful — I was also wondering about the deployment pipeline and whether we need to update the CI config"
assert detect_completion(msg) is False
def test_negative_looks_good_long(self):
msg = "looks good overall, there are a few things I'd like to tweak though — the error messages could be more descriptive and the retry logic needs a backoff"
assert detect_completion(msg) is False
# -- negatives: unrelated --
def test_negative_empty(self):
assert detect_completion("") is False
+47 -8
View File
@@ -53,11 +53,16 @@ _NUDGE_MAP: dict[str, str] = {
}
# ---------------------------------------------------------------------------
# Detection heuristics
# Detection heuristics — strong/weak tiers
#
# Strong patterns fire unconditionally. Weak patterns carry inherent
# ambiguity ("no …", "thanks …") and only fire when the surrounding
# message looks like a genuine correction/completion rather than normal
# conversation.
# ---------------------------------------------------------------------------
_CORRECTION_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"(?i)^no[,.\s]"),
_STRONG_CORRECTION: list[re.Pattern[str]] = [
re.compile(r"(?i)^no[,.]"), # "no," / "no." — clear rejection
re.compile(r"(?i)\bdon'?t\b"),
re.compile(r"(?i)^stop\b"),
re.compile(r"(?i)^actually[,\s]"),
@@ -71,30 +76,64 @@ _CORRECTION_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"(?i)^please don'?t\b"),
]
_COMPLETION_PATTERNS: list[re.Pattern[str]] = [
re.compile(r"(?i)^thanks\b"),
# "no <word>" is ambiguous — only match when the next word is a pronoun,
# demonstrative, article, or verb that signals the user is redirecting,
# not a fixed phrase like "no problem" or "no worries". Allowlist >
# blocklist: we don't need to enumerate every benign "no X" phrase.
_WEAK_CORRECTION: list[re.Pattern[str]] = [
re.compile(
r"(?i)^no\s+(?:I\b|you\b|we\b|they\b|it\b|he\b|she\b"
r"|that\b|this\b|those\b|these\b"
r"|the\b|a\b|an\b"
r"|not\b|do\b|did\b|but\b)"
),
]
_STRONG_COMPLETION: list[re.Pattern[str]] = [
re.compile(r"(?i)\bthat'?s all\b"),
re.compile(r"(?i)^lgtm\b"),
]
# These patterns are common in both completion AND mid-conversation
# acknowledgment. Only fire when the message is short and has no
# continuation markers (question marks, follow-up requests).
_WEAK_COMPLETION: list[re.Pattern[str]] = [
re.compile(r"(?i)^thanks\b(?!\s+for\b)"), # "thanks for X" = acknowledgment
re.compile(r"(?i)\blooks good\b"),
re.compile(r"(?i)^perfect\b"),
re.compile(r"(?i)^great job\b"),
re.compile(r"(?i)\bthat works\b"),
re.compile(r"(?i)^done\b"),
re.compile(r"(?i)^lgtm\b"),
]
_WEAK_MSG_CAP = 80 # weak completion patterns suppressed above this length
_CONTINUATION = re.compile(
r"(?i)(?:\?|(?:can you|could you|please\s|also\s|but\s|now\s|next\s"
r"|and\s+then|after\s+that|one\s+more|however))"
)
def detect_correction(message: str) -> bool:
"""Return True if the message looks like a user correction."""
if not message:
return False
return any(p.search(message) for p in _CORRECTION_PATTERNS)
if any(p.search(message) for p in _STRONG_CORRECTION):
return True
return any(p.search(message) for p in _WEAK_CORRECTION)
def detect_completion(message: str) -> bool:
"""Return True if the message signals session completion."""
if not message:
return False
return any(p.search(message) for p in _COMPLETION_PATTERNS)
if any(p.search(message) for p in _STRONG_COMPLETION):
return True
if len(message) > _WEAK_MSG_CAP:
return False
if _CONTINUATION.search(message):
return False
return any(p.search(message) for p in _WEAK_COMPLETION)
def should_nudge(