fix(sleep): harvest pasted briefs instead of dropping them as host noise

skillopt_sleep/harvest.py

@@ -17,6 +17,7 @@
 
 import json
 import os
+import re
 from datetime import datetime
 from typing import Any, Dict, Iterable, List, Optional
 
@@ -134,13 +135,68 @@ def _detect_feedback(text: str) -> List[str]:
     return sig
 
 
+# ── Issue #318: host wrappers around user text ─────────────────────────
+#
+# The Claude Code desktop app delivers a pasted block as the user message,
+# wrapped in ``<pasted_content id="…">…</pasted_content id="…">``. The brief
+# the user pasted is typed by them and is often the whole point of the
+# session, so the wrapper is unwrapped instead of dropped.
+_PASTED_CONTENT_RE = re.compile(r"</?pasted_content[^>]*>", re.IGNORECASE)
+
+# Host-injected status notes. A message that is only a reminder is host noise,
+# and a reminder *prefixed* to a real request must not ride into the mined
+# intent as text the user never wrote. The match is anchored by the caller:
+# only the block that leads a message is removed.
+_SYSTEM_REMINDER_RE = re.compile(
+    r"<system-reminder\b[^>]*>.*?</system-reminder\s*>", re.IGNORECASE | re.DOTALL
+)
+
+# Tags the host injects as a whole user message: expanded slash-command
+# bodies, local-command output, shell-mode input and editor selections. The
+# list is closed deliberately — a wrapper this list does not know about is
+# treated as user text, so such a message keeps its request instead of losing
+# it. <system-reminder> is listed as well as stripped, so a variant carrying
+# attributes is still recognised as host noise.
+_HOST_META_TAGS = (
+    "command-name", "command-message", "command-args",
+    "local-command-stdout", "local-command-caveat",
+    "system-reminder",
+    "bash-input", "bash-stdout", "bash-stderr",
+    "ide_opened_file", "ide_selection",
+    # VS Code Copilot Chat brackets the machine context of every request
+    # (<environment_info>…</environment_info>); harvesting it leaks local
+    # environment detail and was never a user prompt.
+    "environment_info",
+)
+
+
+def _strip_host_wrappers(text: str) -> str:
+    """Unwrap pasted user content and drop the host's own leading notes.
+
+    ``<pasted_content>`` tags only ever wrap text the user typed, so both the
+    opening and the closing tag are removed wherever they appear. A
+    ``<system-reminder>`` block is removed only while it *leads* the message,
+    which is how the host prepends it. The same text inside a prompt the user
+    wrote — an inline-code sample, a fenced XML example — is their content and
+    survives, so the harvested request stays what they typed.
+    """
+    if "<" not in text:
+        return text
+    out = _PASTED_CONTENT_RE.sub("", text)
+    while True:
+        stripped = out.lstrip()
+        match = _SYSTEM_REMINDER_RE.match(stripped)
+        if match is None:
+            break
+        out = stripped[match.end():]
+    return out
+
+
 def _is_meta_prompt(text: str) -> bool:
     """Skip slash-commands / system noise that aren't real user intents."""
-    t = text.strip()
+    t = _strip_host_wrappers(text).strip()
     if not t:
         return True
-    if t.startswith("<") and t.endswith(">"):
-        return True
     if t.startswith("/") and len(t.split()) <= 3:
         return True
     if t.startswith("[Pasted text") or t.startswith("Caveat:"):
@@ -152,6 +208,14 @@ def _is_meta_prompt(text: str) -> bool:
         return True
     if t.startswith("# /"):
         return True
+    # A message that is nothing but one host tag block (`<system-reminder>`,
+    # `<local-command-stdout>`, …) carries no user intent. The tag list is
+    # closed on purpose: text that merely arrives wrapped in angle brackets
+    # is user content, and dropping it loses the session's real request
+    # (issue #318: the desktop app delivers a pasted brief that way).
+    if t.startswith("<") and t.endswith(">"):
+        head = t[1:].split(">", 1)[0].split(None, 1)
+        return bool(head) and head[0].lower() in _HOST_META_TAGS
     return False
 
 
@@ -215,6 +279,11 @@ def _is_headless_replay(digest: "SessionDigest") -> bool:
     if digest.n_user_turns > 1:
         return False
     if digest.n_user_turns == 0:
+        # Every user turn was filtered as host noise or engine input, so there
+        # is no user intent to mine. Such a record must not be admitted: it
+        # would take a slot under --max-sessions and yield no task, displacing
+        # a real session (review of #318). A pasted brief is not this case —
+        # it now survives the filter as a user turn.
         return True
     prompt = digest.user_prompts[0] if digest.user_prompts else ""
     for marker in _REPLAY_PROMPT_MARKERS:
@@ -278,7 +347,10 @@ def digest_transcript(path: str) -> Optional[SessionDigest]:
             # other sessions, usage-limit notices. The user typed none of it.
             # A body that names a known agent session is still kept, so
             # _is_agent_session can drop the whole session as before.
-            text = _text_from_content(content)
+            # Host wrappers are removed before the meta test: a pasted brief
+            # (#318) is user content, and a <system-reminder> prefix must not
+            # ride into the intent (see _strip_host_wrappers).
+            text = _strip_host_wrappers(_text_from_content(content))
             if rec.get("isMeta") is True and not any(
                 marker in text for marker in _AGENT_SESSION_MARKERS
             ):

tests/test_sleep_engine.py

@@ -82,6 +82,176 @@ def test_meta_prompt_filter(self):
         self.assertTrue(_is_meta_prompt("<system-reminder>x</system-reminder>"))
         self.assertFalse(_is_meta_prompt("please refactor the auth module"))
 
+    def _write_claude_session(self, path, turns):
+        """Write one Claude Code transcript; ``turns`` are (role, text) pairs.
+
+        Records are an hour apart so the sub-3-second heuristic in
+        _is_headless_replay never decides these fixtures.
+        """
+        self._write_jsonl(path, [
+            {"type": role,
+             "timestamp": f"2026-10-15T0{index + 1}:00:00Z",
+             "cwd": "/repo/example",
+             "sessionId": os.path.splitext(os.path.basename(path))[0],
+             "message": {"role": role, "content": text}}
+            for index, (role, text) in enumerate(turns)
+        ])
+
+    def test_meta_prompt_keeps_pasted_user_content(self):
+        # issue #318: the desktop app delivers a pasted brief inside a tag
+        # pair carrying the same id. The user typed that brief, so it is user
+        # content, and dropping it left the session's follow-up as the intent.
+        from skillopt_sleep.harvest import _strip_host_wrappers
+
+        brief = ('<pasted_content id="a1b2">\n'
+                 'Install tool X into a dedicated folder, pin the version.\n'
+                 '</pasted_content id="a1b2">')
+
+        self.assertFalse(_is_meta_prompt(brief))
+        # the helper removes the tags only; callers strip the remainder
+        self.assertEqual(
+            _strip_host_wrappers(brief),
+            "\nInstall tool X into a dedicated folder, pin the version.\n",
+        )
+
+    def test_meta_prompt_filters_only_known_host_tags(self):
+        self.assertTrue(_is_meta_prompt("<local-command-stdout>ok</local-command-stdout>"))
+        self.assertTrue(_is_meta_prompt(
+            "<command-message>skillopt-sleep</command-message>\n"
+            "<command-name>/skillopt-sleep</command-name>"
+        ))
+        # VS Code Copilot Chat brackets each request's machine context the
+        # same way; it is host detail, not a prompt, and must stay out
+        # (tests/test_copilot_harvest.py pins the privacy boundary).
+        self.assertTrue(_is_meta_prompt(
+            "<environment_info>private machine context</environment_info>"
+        ))
+        # A wrapper the host does not own is user text, not noise: it must
+        # survive the filter with its content instead of being dropped for
+        # being wrapped in angle brackets, which is what lost pasted briefs.
+        self.assertFalse(_is_meta_prompt("<release-notes>Ship 1.4</release-notes>"))
+
+    def test_pasted_brief_becomes_the_mined_intent(self):
+        brief = ('<pasted_content id="a1b2">\n'
+                 'Install tool X into a dedicated folder, pin the version, '
+                 'back up the configs first.\n'
+                 '</pasted_content id="a1b2">')
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "pasted-session.jsonl")
+            self._write_claude_session(path, [
+                ("user", brief),
+                ("assistant", "Plan: ..."),
+                ("user", "Yes, go ahead with option A."),
+                ("assistant", "Done."),
+            ])
+            digest = digest_transcript(path)
+
+        [task] = heuristic_mine([digest])
+        self.assertEqual(
+            task.intent,
+            "Install tool X into a dedicated folder, pin the version, back up the configs first.",
+        )
+        self.assertNotIn("pasted_content", task.intent)
+        # the follow-up stays available as constraint context, not as intent
+        self.assertIn("Yes, go ahead with option A.", task.context_excerpt)
+
+    def test_session_whose_only_user_turn_is_a_pasted_brief_survives_harvest(self):
+        from skillopt_sleep.harvest import harvest
+
+        brief = ('<pasted_content id="a1b2">\nInstall tool X into a dedicated folder.\n'
+                 '</pasted_content id="a1b2">')
+        with tempfile.TemporaryDirectory() as tmp:
+            project_dir = os.path.join(tmp, "projects", "-repo-example")
+            os.makedirs(project_dir)
+            self._write_claude_session(os.path.join(project_dir, "pasted-only.jsonl"), [
+                ("user", brief),
+                ("assistant", "Plan: ..."),
+                ("assistant", "Done."),
+            ])
+
+            digests = harvest(os.path.join(tmp, "projects"), scope="all")
+
+        self.assertEqual([d.session_id for d in digests], ["pasted-only"])
+        self.assertEqual(digests[0].user_prompts,
+                         ["Install tool X into a dedicated folder."])
+
+    def test_system_reminder_prefix_does_not_ride_into_the_intent(self):
+        request = "Yes: run the third review on range r2..abc1234 and update the report."
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "reminder-prefix.jsonl")
+            self._write_claude_session(path, [
+                ("user", "<system-reminder>\nStatus note injected by the host app.\n"
+                         f"</system-reminder>\n\n{request}"),
+                ("assistant", "Running the review."),
+            ])
+            digest = digest_transcript(path)
+
+        self.assertEqual(digest.user_prompts, [request])
+        [task] = heuristic_mine([digest])
+        self.assertEqual(task.intent, request)
+        self.assertNotIn("Status note", task.intent)
+
+    def test_host_only_transcript_does_not_take_a_session_slot(self):
+        from skillopt_sleep.harvest import harvest
+
+        # Review of #318: a record with no user intent must not be admitted
+        # before the cap. It mines nothing, so admitting it turns a night into
+        # zero tasks by displacing a real session with a newer host-only one.
+        with tempfile.TemporaryDirectory() as tmp:
+            project_dir = os.path.join(tmp, "projects", "-repo-example")
+            os.makedirs(project_dir)
+            host_only = os.path.join(project_dir, "host-note.jsonl")
+            self._write_jsonl(host_only, [
+                {"type": "user", "isMeta": True,
+                 "timestamp": "2026-10-15T09:00:00Z",
+                 "cwd": "/repo/example", "sessionId": "host-note",
+                 "message": {"role": "user",
+                             "content": "<local-command-stdout>ok</local-command-stdout>"}},
+                {"type": "assistant", "timestamp": "2026-10-15T09:00:30Z",
+                 "cwd": "/repo/example", "sessionId": "host-note",
+                 "message": {"role": "assistant", "content": "Noted."}},
+            ])
+            real = os.path.join(project_dir, "real-task.jsonl")
+            self._write_claude_session(real, [
+                ("user", "Fix the failing date parser test in module B."),
+                ("assistant", "Fixed."),
+            ])
+            os.utime(host_only, (2_000_000, 2_000_000))   # newer
+            os.utime(real, (1_000_000, 1_000_000))
+
+            digests = harvest(os.path.join(tmp, "projects"), scope="all", limit=1)
+
+        self.assertEqual([d.session_id for d in digests], ["real-task"])
+
+    def test_inline_code_system_reminder_example_is_user_text(self):
+        # Review of #318: the user quoting a reminder inside inline code was
+        # being deleted, which changed both the request and the mined intent.
+        request = ("How should I escape `<system-reminder>note</system-reminder>` "
+                   "inside my transcript fixture?")
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "inline-example.jsonl")
+            self._write_claude_session(path, [
+                ("user", request),
+                ("assistant", "Use html.escape()."),
+            ])
+            digest = digest_transcript(path)
+
+        self.assertEqual(digest.user_prompts, [request])
+
+    def test_fenced_system_reminder_example_is_user_text(self):
+        request = ("This is what our host writes before a real request:\n"
+                   "```xml\n<system-reminder>\ncontext\n</system-reminder>\n```\n"
+                   "Please make the parser strip it.")
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "fenced-example.jsonl")
+            self._write_claude_session(path, [
+                ("user", request),
+                ("assistant", "Will do."),
+            ])
+            digest = digest_transcript(path)
+
+        self.assertEqual(digest.user_prompts, [request])
+
     def _claude_session_with_injected_messages(self, tmp, follow_up):
         """A Claude Code session as written to disk when a skill is loaded.