diff --git a/tests/test_metacognition.py b/tests/test_metacognition.py index 2bb85cea..5df7e0dd 100644 --- a/tests/test_metacognition.py +++ b/tests/test_metacognition.py @@ -14,15 +14,16 @@ from turnstone.core.metacognition import ( class TestDetectCorrection: + """Strong patterns always fire; weak 'no ' uses allowlist.""" + + # -- strong patterns (always fire) -- + def test_no_comma(self): assert detect_correction("no, that's wrong") is True def test_no_period(self): assert detect_correction("no. do it differently") is True - def test_no_space(self): - assert detect_correction("no I meant the other one") is True - def test_dont(self): assert detect_correction("don't use tabs") is True @@ -47,6 +48,57 @@ class TestDetectCorrection: def test_please_dont(self): assert detect_correction("please don't mock the database") is True + # -- weak pattern: "no" + allowlisted context word -- + + def test_no_space(self): + assert detect_correction("no I meant the other one") is True + + def test_no_that(self): + assert detect_correction("no that's wrong") is True + + def test_no_it(self): + assert detect_correction("no it should be different") is True + + def test_no_the(self): + assert detect_correction("no the other one") is True + + def test_no_not(self): + assert detect_correction("no not that file") is True + + def test_no_you(self): + assert detect_correction("no you should use pytest") is True + + # -- negatives: "no " not in allowlist -- + + def test_negative_no_problem(self): + assert detect_correction("no problem") is False + + def test_negative_no_worries(self): + assert detect_correction("no worries") is False + + def test_negative_no_rush(self): + assert detect_correction("no rush") is False + + def test_negative_no_one(self): + assert detect_correction("no one knows") is False + + def test_negative_no_thanks(self): + assert detect_correction("no thanks") is False + + def test_negative_no_doubt(self): + assert detect_correction("no doubt about it") is False + + def test_negative_no_idea(self): + assert detect_correction("no idea what you mean") is False + + def test_negative_no_kidding(self): + assert detect_correction("no kidding") is False + + def test_negative_no_luck(self): + assert detect_correction("no luck finding the bug") is False + + # -- negatives: unrelated messages -- + def test_negative_notice(self): assert detect_correction("I noticed the test passes") is False @@ -70,27 +122,82 @@ class TestDetectCorrection: class TestDetectCompletion: - def test_thanks(self): - assert detect_completion("thanks, that's perfect") is True + """Strong patterns always fire; weak patterns gated by length + continuation.""" + + # -- strong patterns (always fire) -- def test_thats_all(self): assert detect_completion("that's all for now") is True + def test_lgtm(self): + assert detect_completion("lgtm") is True + + # -- weak patterns: short message, no continuation -- + + def test_thanks(self): + assert detect_completion("thanks, that's perfect") is True + + def test_thanks_standalone(self): + assert detect_completion("thanks") is True + + def test_thanks_exclaim(self): + assert detect_completion("thanks!") is True + def test_looks_good(self): assert detect_completion("looks good to me") is True def test_perfect(self): assert detect_completion("perfect") is True - def test_lgtm(self): - assert detect_completion("lgtm") is True - def test_done(self): assert detect_completion("done") is True - def test_negative_normal(self): + def test_great_job(self): + assert detect_completion("great job") is True + + def test_that_works(self): + assert detect_completion("that works") is True + + # -- negatives: "thanks for" is acknowledgment -- + + def test_negative_thanks_for(self): + assert detect_completion("thanks for the update") is False + + def test_negative_thanks_for_looking(self): + assert detect_completion("thanks for looking into this") is False + + # -- negatives: continuation markers suppress weak patterns -- + + def test_negative_thanks_but(self): + assert detect_completion("thanks but can you also add tests") is False + + def test_negative_thanks_though(self): + assert detect_completion("thanks though I have one more question") is False + + def test_negative_looks_good_but(self): + assert detect_completion("looks good but can you also add validation") is False + + def test_negative_perfect_now(self): + assert detect_completion("perfect, now add error handling") is False + + def test_negative_done_can_you(self): + assert detect_completion("done with that, can you start on the tests?") is False + + def test_negative_question_mark(self): assert detect_completion("can you add error handling?") is False + # -- negatives: long messages suppress weak patterns -- + + def test_negative_thanks_long(self): + msg = "thanks, this is really helpful — I was also wondering about the deployment pipeline and whether we need to update the CI config" + assert detect_completion(msg) is False + + def test_negative_looks_good_long(self): + msg = "looks good overall, there are a few things I'd like to tweak though — the error messages could be more descriptive and the retry logic needs a backoff" + assert detect_completion(msg) is False + + # -- negatives: unrelated -- + def test_negative_empty(self): assert detect_completion("") is False diff --git a/turnstone/core/metacognition.py b/turnstone/core/metacognition.py index 530b3111..7e297ed2 100644 --- a/turnstone/core/metacognition.py +++ b/turnstone/core/metacognition.py @@ -53,11 +53,16 @@ _NUDGE_MAP: dict[str, str] = { } # --------------------------------------------------------------------------- -# Detection heuristics +# Detection heuristics — strong/weak tiers +# +# Strong patterns fire unconditionally. Weak patterns carry inherent +# ambiguity ("no …", "thanks …") and only fire when the surrounding +# message looks like a genuine correction/completion rather than normal +# conversation. # --------------------------------------------------------------------------- -_CORRECTION_PATTERNS: list[re.Pattern[str]] = [ - re.compile(r"(?i)^no[,.\s]"), +_STRONG_CORRECTION: list[re.Pattern[str]] = [ + re.compile(r"(?i)^no[,.]"), # "no," / "no." — clear rejection re.compile(r"(?i)\bdon'?t\b"), re.compile(r"(?i)^stop\b"), re.compile(r"(?i)^actually[,\s]"), @@ -71,30 +76,64 @@ _CORRECTION_PATTERNS: list[re.Pattern[str]] = [ re.compile(r"(?i)^please don'?t\b"), ] -_COMPLETION_PATTERNS: list[re.Pattern[str]] = [ - re.compile(r"(?i)^thanks\b"), +# "no " is ambiguous — only match when the next word is a pronoun, +# demonstrative, article, or verb that signals the user is redirecting, +# not a fixed phrase like "no problem" or "no worries". Allowlist > +# blocklist: we don't need to enumerate every benign "no X" phrase. +_WEAK_CORRECTION: list[re.Pattern[str]] = [ + re.compile( + r"(?i)^no\s+(?:I\b|you\b|we\b|they\b|it\b|he\b|she\b" + r"|that\b|this\b|those\b|these\b" + r"|the\b|a\b|an\b" + r"|not\b|do\b|did\b|but\b)" + ), +] + +_STRONG_COMPLETION: list[re.Pattern[str]] = [ re.compile(r"(?i)\bthat'?s all\b"), + re.compile(r"(?i)^lgtm\b"), +] + +# These patterns are common in both completion AND mid-conversation +# acknowledgment. Only fire when the message is short and has no +# continuation markers (question marks, follow-up requests). +_WEAK_COMPLETION: list[re.Pattern[str]] = [ + re.compile(r"(?i)^thanks\b(?!\s+for\b)"), # "thanks for X" = acknowledgment re.compile(r"(?i)\blooks good\b"), re.compile(r"(?i)^perfect\b"), re.compile(r"(?i)^great job\b"), re.compile(r"(?i)\bthat works\b"), re.compile(r"(?i)^done\b"), - re.compile(r"(?i)^lgtm\b"), ] +_WEAK_MSG_CAP = 80 # weak completion patterns suppressed above this length + +_CONTINUATION = re.compile( + r"(?i)(?:\?|(?:can you|could you|please\s|also\s|but\s|now\s|next\s" + r"|and\s+then|after\s+that|one\s+more|however))" +) + def detect_correction(message: str) -> bool: """Return True if the message looks like a user correction.""" if not message: return False - return any(p.search(message) for p in _CORRECTION_PATTERNS) + if any(p.search(message) for p in _STRONG_CORRECTION): + return True + return any(p.search(message) for p in _WEAK_CORRECTION) def detect_completion(message: str) -> bool: """Return True if the message signals session completion.""" if not message: return False - return any(p.search(message) for p in _COMPLETION_PATTERNS) + if any(p.search(message) for p in _STRONG_COMPLETION): + return True + if len(message) > _WEAK_MSG_CAP: + return False + if _CONTINUATION.search(message): + return False + return any(p.search(message) for p in _WEAK_COMPLETION) def should_nudge(