diff --git a/cecli/commands/compact.py b/cecli/commands/compact.py index 0ddc5a0ae38..ae91f670900 100644 --- a/cecli/commands/compact.py +++ b/cecli/commands/compact.py @@ -12,6 +12,11 @@ async def execute(cls, io, coder, args, **kwargs): message = args.strip() if args else "" await coder.compact_context_if_needed(force=True, message=message) + # CLI-71: compaction is a command, so run_one() returns early after it. + # Set the drain flag so any queued prompts process automatically once + # compaction finishes, instead of stalling until a manual /run-queue. + coder._drain_queue_after_command = True + @classmethod def get_help(cls) -> str: return "Force compaction of the chat history context." diff --git a/cecli/commands/core.py b/cecli/commands/core.py index bda40a64df3..bb9063f5223 100644 --- a/cecli/commands/core.py +++ b/cecli/commands/core.py @@ -14,8 +14,9 @@ class Commands: scraper = None - # Commands that receive their full argument text for nested completion. - FULL_ARGS_COMPLETION_COMMANDS = ("/queue", "/insert-queue") + # Commands that receive their full argument text for completion (nested + # hosts and subcommand commands whose names take an argument). + FULL_ARGS_COMPLETION_COMMANDS = ("/queue", "/insert-queue", "/workflows") def _get_coder(self): """Return coder via weak reference, or None if collected.""" diff --git a/cecli/commands/workflows.py b/cecli/commands/workflows.py index 97b8ea649b3..e35bf795ed8 100644 --- a/cecli/commands/workflows.py +++ b/cecli/commands/workflows.py @@ -78,9 +78,19 @@ def get_help(cls) -> str: @classmethod def get_completions(cls, io, coder, args) -> list: """Complete subcommands, then workflow names for the name-taking ones.""" - partial = (args or "").strip() + raw = args or "" + partial = raw.strip() if " " not in partial: + # A trailing space means the subcommand is settled and the caller + # is now completing its argument, so offer the workflow names + # (when the subcommand takes one) instead of the subcommand list. + if raw and raw[-1].isspace(): + if partial.lower() in cls.NAME_SUBCOMMANDS: + return sorted(wf.name for wf in cls._workflows(coder)) + + return [] + return [s for s in cls.SUBCOMMANDS if s.startswith(partial)] or list(cls.SUBCOMMANDS) sub, _, rest = partial.partition(" ") diff --git a/cecli/help.py b/cecli/help.py index 30b05fc211f..0da9bd4cd27 100755 --- a/cecli/help.py +++ b/cecli/help.py @@ -1,9 +1,12 @@ +import difflib +import fnmatch import os import re import threading import warnings from collections import Counter from pathlib import Path +from urllib.parse import unquote, urlparse import importlib_resources @@ -34,6 +37,8 @@ ) _HEADING_RE = re.compile(r"^#{1,6}\s") +_HEADING_TEXT_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +_KEY_ANCHOR_RE = re.compile(r"^#\s*([a-z0-9][a-z0-9_.-]*)\s*:\s*(.*)$") _FENCE_RE = re.compile(r"^\s*(```+|~~~+)") _TOKEN_RE = re.compile(r"[a-z0-9]+") @@ -207,6 +212,186 @@ def keyword_retrieve(question, exclude_ids=None, n_results=_KEYWORD_RESULTS): return docs +def navigate_docs(question, n_files=5, n_sections=10): + """Rank documentation files and heading sections by query-term coverage. + + Unlike :func:`keyword_retrieve`, this returns navigation targets rather + than chunk text. Each file and heading section is scored by how many + unique query terms it contains, which helps locate the right page when + embedding search misses the relevant chunk. + """ + query_tokens = list(dict.fromkeys(_tokenize(question or ""))) + query_terms = set(query_tokens) + + if not query_terms: + return dict(files=[], sections=[]) + + max_files = max(1, int(n_files)) + max_sections = max(1, int(n_sections)) + file_matches = {} + section_matches = [] + + for fname in get_package_files(): + fname = Path(fname) + if any(fname.match(pattern) for pattern in exclude_website_pats): + continue + + try: + text = fname.read_text(encoding="utf-8") + except OSError: + continue + + path = _doc_display_path(fname) + url = fname_to_url(str(fname)) + anchors_by_line = {anchor["start"]: anchor["slug"] for anchor in _document_anchors(text)} + file_terms = set(_tokenize(f"{fname.parent.name} {fname.stem}")) + matched_terms = query_terms & file_terms + matched_sections = 0 + + for heading, line_number, section_text in _iter_doc_sections(text): + token_counts = Counter(_tokenize(f"{heading} {section_text}")) + section_terms = [token for token in query_tokens if token in token_counts] + + if not section_terms: + continue + + matched_terms.update(section_terms) + matched_sections += 1 + heading_terms = set(_tokenize(heading)) + heading_hits = sum(1 for token in section_terms if token in heading_terms) + frequency = sum(token_counts[token] for token in section_terms) + slug = anchors_by_line.get(line_number, "") + + section_matches.append( + dict( + path=path, + url=url, + heading=heading, + line=line_number, + slug=slug, + address=f"{path}#{slug}" if slug else path, + matched_terms=section_terms, + score=len(section_terms), + heading_hits=heading_hits, + frequency=frequency, + ) + ) + + if matched_terms: + file_matches[path] = dict( + path=path, + url=url, + filename=fname.name, + matched_terms=[token for token in query_tokens if token in matched_terms], + score=len(matched_terms), + section_count=matched_sections, + ) + + files = sorted( + file_matches.values(), + key=lambda item: (-item["score"], -item["section_count"], item["path"]), + )[:max_files] + sections = sorted( + section_matches, + key=lambda item: ( + -item["score"], + -item["heading_hits"], + -item["frequency"], + item["path"], + item["line"], + ), + )[:max_sections] + + return dict(files=files, sections=sections) + + +def get_doc(path, max_results=20): + """Read one docs file or browse matching docs paths. + + An exact docs-relative path (or docs URL) returns the whole markdown file. + A directory, glob, or missing basename returns matching paths, descriptions, + and immediate subdirectories so callers can pick the next path to read. + """ + normalized = _normalize_doc_path(path) + + if normalized is None: + return _empty_doc_result( + error=( + f"Invalid documentation path: {path}. Use a docs-relative path such as " + "config/hooks.md." + ) + ) + + doc_path, force_browse, is_glob, anchor, anchor_requested = normalized + + try: + max_results = max(1, int(max_results)) + except (TypeError, ValueError): + max_results = 20 + + entries = _doc_entries() + + if not is_glob and not force_browse: + exact = _find_exact_doc(entries, doc_path) + + if exact: + fname, display_path = exact + + try: + text = fname.read_text(encoding="utf-8") + except OSError as exc: + return _empty_doc_result(error=f"Could not read {display_path}: {exc}") + + file_entry = _doc_file_entry(fname, display_path, text) + + if anchor_requested and not anchor: + return _doc_toc_result(file_entry) + + if anchor: + return _doc_section_result(file_entry, anchor) + + file_entry["text"] = _strip_frontmatter(file_entry["text"]) + + return dict( + file=file_entry, + files=[], + dirs=[], + suggestions=[], + total=0, + path=display_path, + ) + + if anchor_requested: + return _empty_doc_result( + error=( + f'Anchors require an exact file path: "{path}" did not match one. ' + "Read the file first, then append #heading." + ) + ) + + if is_glob or force_browse or _has_doc_prefix(entries, doc_path): + result = _browse_docs(entries, doc_path, max_results, is_glob) + + if result["files"] or result["dirs"]: + return result + + return _empty_doc_result(error=f"No documentation matched path={path!r}.") + + suggestions = _suggest_docs(entries, doc_path, max_results=min(3, max_results)) + + if not suggestions: + return _empty_doc_result(error=f"No documentation matched path={path!r}.") + + return dict( + file=None, + files=[], + dirs=[], + suggestions=suggestions, + total=len(suggestions), + path=doc_path, + ) + + class Help: """Vector retriever over the bundled help docs using a local chromadb store.""" @@ -502,3 +687,787 @@ def _keyword_score(entry, tokens, phrase): score += 10.0 * matched / len(tokens) return score + + +def _doc_display_path(fname): + """Return the docs-relative path used in navigation results.""" + parts = Path(fname).parts + + try: + docs_index = [part.lower() for part in parts].index("docs") + except ValueError: + return Path(fname).name + + relative = "/".join(parts[docs_index + 1 :]) + return relative or Path(fname).name + + +def _iter_doc_sections(text): + """Yield ``(heading_path, line_number, section_text)`` for each heading. + + Heading paths include parent headings (``Parent > Child``). Content before + the first heading, including front matter, is skipped because it is not a + navigable section. + """ + sections = [] + heading_stack = [] + current_heading = "" + current_line = 0 + current = [] + fence = None + + for line_number, line in enumerate(text.splitlines(keepends=True), start=1): + was_in_fence = fence is not None + fence = _toggle_fence(line, fence) + match = None if was_in_fence else _HEADING_RE.match(line) + + if match: + if current_heading: + sections.append((current_heading, current_line, "".join(current))) + + level = len(match.group(0)) - len(match.group(0).lstrip("#")) + title = line.strip().lstrip("#").strip() + heading_stack = heading_stack[: level - 1] + [title] + current_heading = " > ".join(heading_stack) + current_line = line_number + current = [line] + elif current_heading: + current.append(line) + + if current_heading: + sections.append((current_heading, current_line, "".join(current))) + + return sections + + +def _empty_doc_result(error=None): + return dict(file=None, files=[], dirs=[], suggestions=[], total=0, path="", error=error) + + +def _doc_entries(): + """Return ``(file path, docs-relative display path)`` pairs sorted by path.""" + entries = [] + + for fname in get_package_files(): + entries.append((Path(fname), _doc_display_path(fname))) + + entries.sort(key=lambda item: item[1].lower()) + return entries + + +def _normalize_doc_path(path): + """Normalize a path, docs URL, or glob into a docs-relative lookup.""" + raw = str(path or "").strip() + anchor_requested = "#" in raw + + if not raw: + return "", True, False, "", False + + if raw in ("/", "\\"): + return "", True, False, "", False + + is_url = raw.lower().startswith(("http://", "https://")) + + if not is_url and raw.startswith("~"): + return None + + if not is_url and raw.startswith("/"): + relative = raw.lstrip("/").lower() + + if not ( + relative in ("docs", "website/docs") or relative.startswith(("docs/", "website/docs/")) + ): + return None + + anchor = "" + + if is_url: + parsed = urlparse(raw) + raw = parsed.path + anchor = unquote(parsed.fragment) + + raw, _, raw_anchor = raw.partition("#") + + if raw_anchor: + anchor = unquote(raw_anchor) + + force_browse = not is_url and raw.endswith(("/", "\\")) + parts = [] + + for part in raw.replace("\\", "/").split("/"): + if part in ("", "."): + continue + if part == "..": + return None + parts.append(part) + + if len(parts) >= 2 and parts[0].lower() == "website" and parts[1].lower() == "docs": + parts = parts[2:] + elif parts and parts[0].lower() == "docs": + parts = parts[1:] + + doc_path = "/".join(parts) + is_glob = any(char in doc_path for char in "*?[") + + if not doc_path: + force_browse = True + + return doc_path, force_browse, is_glob, anchor.strip(), anchor_requested + + +def _find_exact_doc(entries, doc_path): + targets = [doc_path] + + if not doc_path.lower().endswith(".md"): + targets.append(f"{doc_path}.md") + + for target in targets: + lowered = target.lower() + for fname, display_path in entries: + if display_path.lower() == lowered: + return fname, display_path + + return None + + +def _has_doc_prefix(entries, doc_path): + if not doc_path: + return True + + prefix = doc_path.lower().rstrip("/") + "/" + return any(display_path.lower().startswith(prefix) for _, display_path in entries) + + +def _doc_file_entry(fname, display_path, text): + return dict( + path=display_path, + url=fname_to_url(str(fname)), + text=text, + lines=len(text.splitlines()), + description=_doc_description(text), + ) + + +def _doc_browse_entry(fname, display_path): + try: + text = fname.read_text(encoding="utf-8") + except OSError: + return None + + return dict( + kind="file", + path=display_path, + url=fname_to_url(str(fname)), + lines=len(text.splitlines()), + sections=len(_doc_sections(text)), + description=_doc_description(text), + ) + + +def _browse_docs(entries, doc_path, max_results, is_glob): + if is_glob: + matches = [ + (fname, display_path) + for fname, display_path in entries + if fnmatch.fnmatchcase(display_path.lower(), doc_path.lower()) + ] + files = [ + entry for entry in (_doc_browse_entry(*match) for match in matches) if entry is not None + ] + + return dict( + file=None, + files=files[:max_results], + dirs=[], + suggestions=[], + total=len(files), + path=doc_path, + error=None, + ) + + prefix = f"{doc_path}/" if doc_path else "" + files = [] + dir_counts = {} + + for fname, display_path in entries: + if prefix and not display_path.lower().startswith(prefix.lower()): + continue + + relative = display_path[len(prefix) :] + if "/" in relative: + dir_name = relative.split("/", 1)[0] + dir_counts[dir_name] = dir_counts.get(dir_name, 0) + 1 + continue + + entry = _doc_browse_entry(fname, display_path) + if entry is not None: + files.append(entry) + + dirs = [ + dict(kind="dir", path=f"{doc_path}/{name}".strip("/"), count=count) + for name, count in sorted(dir_counts.items()) + ] + visible = [] + + for index in range(max_results): + if index < len(dirs): + visible.append(dirs[index]) + + if len(visible) >= max_results: + break + + if index < len(files): + visible.append(files[index]) + + if len(visible) >= max_results: + break + + return dict( + file=None, + files=[entry for entry in visible if entry["kind"] == "file"], + dirs=[entry for entry in visible if entry["kind"] == "dir"], + suggestions=[], + total=len(dirs) + len(files), + path=doc_path or ".", + error=None, + ) + + +def _suggest_docs(entries, doc_path, max_results=3): + if not doc_path: + return [] + + stem = ( + Path(doc_path).stem.lower() + if doc_path.lower().endswith(".md") + else Path(doc_path).name.lower() + ) + suggestions = [] + + for fname, display_path in entries: + name = Path(display_path).stem.lower() + if stem and (stem in name or name in stem): + entry = _doc_browse_entry(fname, display_path) + if entry is not None: + suggestions.append(entry) + + if len(suggestions) >= max_results: + break + + return suggestions + + +def _doc_description(text): + """Return the front-matter description, falling back to the first heading.""" + description = "" + lines = text.splitlines() + + if lines and lines[0].strip() == "---": + for line in lines[1:]: + if line.strip() == "---": + break + if line.lower().startswith("description:"): + description = line.split(":", 1)[1].strip().strip("\"'") + break + + if not description: + description = _first_doc_heading(text) + + if not description: + description = _first_doc_line(text) + + return _clip_text(description) + + +def _first_doc_heading(text): + fence = None + + for line in text.splitlines(): + was_in_fence = fence is not None + fence = _toggle_fence(line, fence) + + if not was_in_fence and _HEADING_RE.match(line): + return line.strip().lstrip("#").strip() + + return "" + + +def _first_doc_line(text): + in_frontmatter = False + + for index, line in enumerate(text.splitlines()): + stripped = line.strip() + + if not stripped: + continue + + if index == 0 and stripped == "---": + in_frontmatter = True + continue + + if in_frontmatter: + if stripped == "---": + in_frontmatter = False + continue + + return stripped + + return "" + + +def _clip_text(text, limit=100): + text = " ".join(str(text or "").split()) + + if len(text) <= limit: + return text + + return text[: limit - 3].rstrip() + "..." + + +def _doc_sections(text): + """Return ``(heading_path, level, start_line, end_line)`` per heading. + + Ranges are subtrees: a section runs until the next heading of the same or + higher level, so ``## A`` includes its ``### B`` children. Headings inside + fenced code blocks are ignored, as are front-matter lines. + """ + lines = text.splitlines(keepends=True) + headings = [] + stack = [] + fence = None + + for line_number, line in enumerate(lines, start=1): + was_in_fence = fence is not None + fence = _toggle_fence(line, fence) + + if was_in_fence: + continue + + match = _HEADING_TEXT_RE.match(line) + + if not match: + continue + + level = len(match.group(1)) + stack = stack[: level - 1] + [match.group(2).strip()] + headings.append((line_number, level, " > ".join(stack))) + + sections = [] + + for index, (start, level, heading) in enumerate(headings): + end = len(lines) + 1 + + for later_start, later_level, _ in headings[index + 1 :]: + if later_level <= level: + end = later_start + break + + sections.append((heading, level, start, end)) + + return sections + + +def _doc_slug(text): + """Return a URL-fragment slug for a heading title, key, or anchor.""" + return "-".join(re.sub(r"[^a-z0-9]+", " ", str(text or "").lower()).split()) + + +def _doc_section_slug(text): + """Normalize a heading or anchor to a space-separated comparison key.""" + return " ".join(re.sub(r"[^a-z0-9]+", " ", str(text or "").lower()).split()) + + +def _document_anchors(text): + """Return heading anchors with unique, feed-back-able slugs. + + Each entry holds ``heading``, ``level``, ``start``, ``end`` and ``slug``. + A leaf slug is used when it is unique in the document; ambiguous leaves are + qualified with their parent headings, so ``#examples`` never dangles. + """ + sections = _doc_sections(text) + leaves = Counter(_doc_slug(heading.split(" > ")[-1]) for heading, *_ in sections) + anchors = [] + used = set() + + for heading, level, start, end in sections: + leaf = _doc_slug(heading.split(" > ")[-1]) + slug = leaf if leaves[leaf] == 1 else _doc_slug(heading.replace(" > ", " ")) + base = slug or "section" + slug = base + suffix = 2 + + while slug in used: + slug = f"{base}-{suffix}" + suffix += 1 + + used.add(slug) + anchors.append(dict(heading=heading, level=level, start=start, end=end, slug=slug)) + + return anchors + + +def _doc_keys(text): + """Return sample-config key anchors from commented defaults in fences. + + Each entry holds ``key``, ``value``, ``description``, ``start`` (the + description line) and ``end`` (the key line). + """ + keys = [] + fence = None + description = "" + description_line = 0 + + for line_number, line in enumerate(text.splitlines(), start=1): + was_in_fence = fence is not None + fence = _toggle_fence(line, fence) + + if not was_in_fence: + description, description_line = "", 0 + continue + + if line.startswith("## "): + description, description_line = line[3:].strip(), line_number + continue + + match = _KEY_ANCHOR_RE.match(line) + + if not match or not description_line: + continue + + value = match.group(2).strip() + + if value.startswith(("//", "http")) or "://" in value: + continue + + keys.append( + dict( + key=match.group(1), + value=value, + description=description, + start=description_line or line_number, + end=line_number, + ) + ) + + return keys + + +def _find_doc_key(text, wanted): + """Return the sample-config key anchor matching *wanted*, or ``None``.""" + wanted_slug = _doc_slug(wanted) + + if not wanted_slug: + return None + + for entry in _doc_keys(text): + if _doc_slug(entry["key"]) == wanted_slug: + return entry + + return None + + +def _find_doc_section(text, heading): + """Return ``(heading_path, level, start, end)`` for *heading*, or ``None``. + + Resolves a generated slug first, then the full heading path, then any + single ``>``-separated segment, so ``Deduplication`` resolves inside + ``Merge Precedence`` without the caller knowing the parent heading. + """ + wanted_slug = _doc_slug(heading) + + if wanted_slug: + for anchor in _document_anchors(text): + if anchor["slug"] == wanted_slug: + return ( + anchor["heading"], + anchor["level"], + anchor["start"], + anchor["end"], + ) + + wanted = _doc_section_slug(heading) + + if not wanted: + return None + + sections = _doc_sections(text) + + for section in sections: + if _doc_section_slug(section[0]) == wanted: + return section + + for section in sections: + if wanted in [_doc_section_slug(part) for part in section[0].split(" > ")]: + return section + + return None + + +def _doc_section_result(file_entry, anchor): + """Slice one heading section or config key out of *file_entry*.""" + text = file_entry.get("text") or "" + section = _find_doc_section(text, anchor) + + if section is None: + key_entry = _find_doc_key(text, anchor) + + if key_entry is not None: + return _doc_key_result(file_entry, key_entry) + + return _empty_doc_result(error=_doc_anchor_error(file_entry, anchor)) + + heading_path, level, start, end = section + lines = text.splitlines(keepends=True) + clipped = dict(file_entry) + clipped["text"] = "".join(lines[start - 1 : end - 1]) + clipped["section"] = heading_path + clipped["level"] = level + clipped["slug"] = _anchor_at(text, start).get("slug", "") + clipped["start_line"] = start + clipped["end_line"] = end - 1 + + return _doc_result(clipped, file_entry) + + +def _doc_key_result(file_entry, key_entry): + """Return one sample-config key pair as a file-shaped result.""" + text = file_entry.get("text") or "" + lines = text.splitlines(keepends=True) + start, end = key_entry["start"], key_entry["end"] + clipped = dict(file_entry) + clipped["text"] = "".join(lines[start - 1 : end]) + clipped["key"] = key_entry["key"] + clipped["key_value"] = key_entry["value"] + clipped["slug"] = _doc_slug(key_entry["key"]) + clipped["start_line"] = start + clipped["end_line"] = end + + return _doc_result(clipped, file_entry) + + +def _doc_result(clipped, file_entry): + """Wrap one sliced file entry in the standard ``get_doc`` result shape.""" + return dict( + file=clipped, + files=[], + dirs=[], + suggestions=[], + total=0, + path=file_entry.get("path", ""), + error=None, + ) + + +def _doc_toc_result(file_entry): + """Return a heading and config-key outline for one docs file.""" + text = file_entry.get("text") or "" + anchors = _document_anchors(text) + keys = _doc_keys(text) + + return dict( + file=None, + files=[], + dirs=[], + suggestions=[], + total=len(anchors), + path=file_entry.get("path", ""), + error=None, + toc=dict( + path=file_entry.get("path", ""), + url=file_entry.get("url", ""), + lines=file_entry.get("lines", 0), + sections=anchors, + keys=[entry["key"] for entry in keys], + ), + ) + + +def _doc_anchor_error(file_entry, anchor): + """Explain an unmatched anchor, naming the closest headings and keys.""" + text = file_entry.get("text") or "" + names = [section[0] for section in _doc_sections(text)] + keys = [entry["key"] for entry in _doc_keys(text)] + path = file_entry.get("path", "") + closest_headings = difflib.get_close_matches(str(anchor), names, n=3, cutoff=0.3) + closest_keys = difflib.get_close_matches(str(anchor), keys, n=3, cutoff=0.3) + + if keys: + message = f'No heading or key matched "{anchor}" in {path}.' + else: + message = f'No heading matched "{anchor}" in {path}.' + + if closest_headings: + message += " Closest heading: " + "; ".join(closest_headings) + "." + + if closest_keys: + message += " Closest key: " + "; ".join(closest_keys) + "." + + counts = f"{len(names)} headings" + + if keys: + counts += f", {len(keys)} keys" + + return message + f" ({counts}; read the file to list them all.)" + + +def _strip_frontmatter(text): + """Drop ``parent``/``nav_order`` lines from YAML front matter. + + Whole-file reads keep the rest of the front matter, including the + ``description`` line, but the navigation-only keys are noise for an agent. + """ + lines = text.splitlines(keepends=True) + + if len(lines) < 3 or lines[0].strip() != "---": + return text + + closing = None + + for index in range(1, len(lines)): + if lines[index].strip() == "---": + closing = index + break + + if closing is None: + return text + + body = [ + line + for line in lines[1:closing] + if not line.lstrip().lower().startswith(("parent:", "nav_order:")) + ] + + return "".join([lines[0]] + body + lines[closing:]) + + +def _anchor_at(text, line): + """Return the heading anchor starting at *line*, or ``None``.""" + for anchor in _document_anchors(text): + if anchor["start"] == line: + return anchor + + return None + + +def locate_doc_chunk(chunk_id): + """Return the docs path, heading, and line range for a chunk id.""" + source, separator, index = str(chunk_id or "").rpartition("#") + + if not separator or not index.isdigit(): + return None + + fname = Path(source) + + try: + text = fname.read_text(encoding="utf-8") + except (OSError, ValueError): + return None + + chunks = _chunk_document(text) + position = int(index) + + if position >= len(chunks): + return None + + chunk = chunks[position] + start_offset = text.find(chunk) + + if start_offset < 0: + start_offset = text.find(chunk.strip()) + + if start_offset < 0: + return None + + start_line = text.count("\n", 0, start_offset) + 1 + end_offset = start_offset + len(chunk) + end_line = text.count("\n", 0, end_offset) + (0 if chunk.endswith("\n") else 1) + heading = "" + active_start = -1 + + for section_heading, _, section_start, section_end in _doc_sections(text): + if section_start <= start_line < section_end and section_start > active_start: + heading = section_heading + active_start = section_start + + return dict( + path=_doc_display_path(fname), + url=fname_to_url(str(fname)), + heading=heading, + start_line=start_line, + end_line=max(start_line, end_line), + lines=len(text.splitlines()), + ) + + +def rank_doc_addresses(query, max_results=5): + """Rank feed-back-able docs addresses for *query*, one address per file. + + Addresses are ``path#slug`` for a matched heading and ``path`` for a page + with no matching section. Ranking reuses :func:`navigate_docs`, so no + embeddings or index rebuild are involved, and each file contributes at most + one address so a large page cannot flood the results. + """ + query = str(query or "").strip() + + if not query: + return dict(addresses=[], total=0, query=query) + + try: + limit = max(1, int(max_results)) + except (TypeError, ValueError): + limit = 5 + + navigation = navigate_docs(query, n_files=max(limit * 3, 15), n_sections=max(limit * 4, 12)) + files_by_path = {display: fname for fname, display in _doc_entries()} + sections_by_file = {} + + for section in navigation.get("sections") or []: + sections_by_file.setdefault(section.get("path", ""), section) + + addresses = [] + + for item in navigation.get("files") or []: + if len(addresses) >= limit: + break + + path = item.get("path", "") + fname = files_by_path.get(path) + + if fname is None: + continue + + try: + text = fname.read_text(encoding="utf-8") + except OSError: + continue + + section = sections_by_file.get(path) + anchor = _anchor_at(text, section.get("line", 0)) if section else None + + if anchor is not None: + addresses.append( + dict( + kind="section", + path=path, + address=f"{path}#{anchor['slug']}", + heading=anchor["heading"], + lines=[anchor["start"], anchor["end"] - 1], + ) + ) + else: + addresses.append( + dict( + kind="file", + path=path, + address=path, + heading="", + description=_doc_description(text), + lines=[1, len(text.splitlines())], + ) + ) + + return dict( + addresses=addresses, + total=len(navigation.get("files") or []), + query=query, + ) diff --git a/cecli/helpers/conversation/integration.py b/cecli/helpers/conversation/integration.py index f875a4518f8..215c2dab182 100644 --- a/cecli/helpers/conversation/integration.py +++ b/cecli/helpers/conversation/integration.py @@ -1100,6 +1100,33 @@ def retrieve_creator_docs(self, coder, query, n_results=8) -> list: except Exception: return [] + def retrieve_creator_doc_addresses(self, coder, query, n_results=5) -> dict: + """Return feed-back-able docs addresses ranked by query-term coverage.""" + try: + from cecli.help import rank_doc_addresses + + return rank_doc_addresses(query, max_results=n_results) + except Exception: + return dict(addresses=[], total=0, query=query) + + def retrieve_creator_doc_navigation(self, coder, query, n_files=5, n_sections=10) -> dict: + """Return files and heading sections ranked by query-term coverage.""" + try: + from cecli.help import navigate_docs + + return navigate_docs(query, n_files=n_files, n_sections=n_sections) + except Exception: + return dict(files=[], sections=[]) + + def retrieve_creator_doc(self, coder, path, n_results=20) -> dict: + """Return a whole doc file, or matching paths for a directory or glob.""" + try: + from cecli.help import get_doc + + return get_doc(path, max_results=n_results) + except Exception: + return dict(file=None, files=[], dirs=[], suggestions=[], total=0, path="") + def defer_removal(self, file_path: str): self._deferred_removals.add(file_path) diff --git a/cecli/main.py b/cecli/main.py index ae25dd3a036..e784865bbb2 100644 --- a/cecli/main.py +++ b/cecli/main.py @@ -578,6 +578,9 @@ def custom_tracer(frame, event, arg): def main(argv=None, input=None, output=None, force_git_root=None, return_coder=False): + # Keep git from launching a pager that waits for input nobody can provide. + os.environ["GIT_PAGER"] = "cat" + from cecli.signals import ReloadProgramSignal attach_debug_thread_monitor(argv) @@ -665,6 +668,12 @@ async def main_async( from cecli.helpers.copypaste import ClipboardWatcher from cecli.helpers.file_searcher import handle_core_files from cecli.helpers.key_detection import detect_api_keys, register_api_keys + from cecli.helpers.workspaces.config import ( + find_active_workspace_name, + load_workspace_config, + resolve_workspace_config, + workspace_layout, + ) from cecli.history import ChatSummary from cecli.hooks import HookService from cecli.io import InputOutput @@ -774,20 +783,21 @@ async def main_async( args.fancy_input = False args.pretty = False - if args.workspaces or args.workspace_name: - from cecli.helpers.workspaces.config import ( - find_active_workspace_name, - load_workspace_config, - workspace_layout, - ) + if args.workspaces: + # Interpolate environment variables in the workspaces argument + args.workspaces = interpolate_env_vars(args.workspaces) + + ws_config_arg = convert_yaml_to_json_string(args.workspaces) if args.workspaces else None + + # Workspaces can also be defined by the local ``.cecli.workspaces.yml`` or + # global ``~/.cecli/workspaces.yml`` files, without any workspace flags. + workspace_conf = resolve_workspace_config(ws_config_arg) + workspace_configured = bool(args.workspaces or args.workspace_name or workspace_conf) + + if workspace_configured: from cecli.helpers.workspaces.subagents import register_workspace_subagents from cecli.helpers.workspaces.workspace import WorkspaceManager - # Interpolate environment variables in the workspaces argument - if args.workspaces: - args.workspaces = interpolate_env_vars(args.workspaces) - - ws_config_arg = convert_yaml_to_json_string(args.workspaces) if args.workspaces else None ws_name = args.workspace_name or find_active_workspace_name(ws_config_arg) if ws_name: config = load_workspace_config(ws_config_arg, name=ws_name) @@ -1294,7 +1304,7 @@ def get_io(pretty): # its repo, root, and repo_map at construction; if the repo is created # afterwards the coder keeps repo=None for the whole session and # repo-backed tools (Grep, git_*, ...) fail with "Not in a git repository." - if args.git and not (suppress_pre_init or args.workspaces or args.workspace_name): + if args.git and not (suppress_pre_init or workspace_configured): git_root = await setup_git(git_root, pre_init_io) if args.gitignore: await check_gitignore(git_root, pre_init_io) diff --git a/cecli/models.py b/cecli/models.py index 33844f0337d..052e30e8273 100644 --- a/cecli/models.py +++ b/cecli/models.py @@ -2308,11 +2308,16 @@ async def sanity_check_model(io, model): await check_for_dependencies(io, model.name) if not (model.info.get("max_input_tokens") or model.info.get("max_tokens")): show = True + possible_matches = fuzzy_match_models(model.name) + # A known model can still lack token limits and costs, so the usual + # "did you mean" suggestions would just be confusing. + model_name_lower = model.name.lower() + is_known_model = any(match.lower() == model_name_lower for match in possible_matches) io.tool_warning( - f"Warning for {model}: Unknown context window size and costs, using sane defaults." + f"Warning for {model}: The context window size and/or costs are unknown" + " for this model, using sane defaults." ) - possible_matches = fuzzy_match_models(model.name) - if possible_matches: + if not is_known_model and possible_matches: io.tool_output("Did you mean one of these?") for match in possible_matches: io.tool_output(f"- {match}") diff --git a/cecli/tools/search_docs.py b/cecli/tools/search_docs.py index 8534b657bc1..79f91ced726 100644 --- a/cecli/tools/search_docs.py +++ b/cecli/tools/search_docs.py @@ -1,9 +1,13 @@ """SearchDocs tool - on-demand cecli documentation search for the creator agent.""" +from cecli.help import locate_doc_chunk from cecli.tools.utils.base_tool import BaseTool from cecli.tools.utils.helpers import is_creator_coder from cecli.tools.utils.responses import ToolResponse +_MAX_DISTANCE = 1.45 +_FETCH_FACTOR = 3 + class Tool(BaseTool): NORM_NAME = "searchdocs" @@ -14,8 +18,10 @@ class Tool(BaseTool): "function": { "name": "SearchDocs", "description": ( - "Search cecli's bundled documentation on demand. Returns relevant " - "documentation chunks. Scoped to the creator sub-agent." + "Look up cecli's bundled documentation on demand. A query returns " + "ranked, feed-back-able addresses (path#heading-slug) to read next; " + "a path reads one file, section, config key, or listing. Scoped to " + "the creator sub-agent." ), "parameters": { "type": "object", @@ -24,7 +30,32 @@ class Tool(BaseTool): "type": "string", "description": ( "What to look up, e.g. 'custom command format', " - "'hook payload schema', or 'MCP server config'." + "'hook payload schema', or 'MCP server config'. Returns " + "ranked addresses to pass back as 'path'. Omit when " + "using 'path'." + ), + }, + "path": { + "type": "string", + "description": ( + "Docs-relative path, URL, directory, or glob. An exact " + "file returns the whole file; 'config/' or 'config/*.md' " + "lists matches; '/' lists the docs root. Append #heading " + "or #config-key to read one section or key pair, e.g. " + "'config/conf.md#Merge Precedence' or " + "'config/conf.md#dark-mode'. A bare '#' returns the " + "file's outline. Omit when using 'query'." + ), + }, + "mode": { + "type": "string", + "enum": ["addresses", "content"], + "default": "addresses", + "description": ( + "Query result style: 'addresses' returns ranked paths " + "to read next; 'content' returns documentation text " + "chunks with path, heading, and line range plus " + "navigation hints." ), }, "max_results": { @@ -32,18 +63,27 @@ class Tool(BaseTool): "default": 5, "minimum": 1, "maximum": 20, - "description": "Maximum number of documentation chunks to return.", + "description": ( + "Maximum number of addresses, chunks plus navigation " + "entries for a query, or matching files and directories " + "combined for a path." + ), }, }, - "required": ["query"], + "required": [], }, }, } @classmethod - async def execute(cls, coder, query=None, max_results=5, **kwargs): - """Return documentation chunks matching *query*. + async def execute(cls, coder, query=None, path=None, mode=None, max_results=5, **kwargs): + """Search documentation, read one file, or browse docs paths. + ``query`` returns ranked addresses unless ``mode="content"``, which + returns text chunks plus navigation hints. ``path`` returns the whole + markdown file for an exact match, a section or config key for an anchor, + an outline for a bare ``#``, or matching files and subdirectories for a + directory or glob. This is the creator's only documentation path: retrieval is explicit rather than injected automatically into the conversation. """ @@ -56,8 +96,21 @@ async def execute(cls, coder, query=None, max_results=5, **kwargs): return response query = str(query or "").strip() - if not query: - response.append_error("'query' must be a non-empty string.") + path = str(path or "").strip() + mode = str(mode or "addresses").strip().lower() + + if mode not in ("addresses", "content"): + response.append_error("'mode' must be 'addresses' or 'content'.") + return response + + if query and path: + response.append_error("Pass either 'query' (search) or 'path' (read/browse), not both.") + return response + + if not query and not path: + response.append_error( + "Provide 'query' to search or 'path' to read/browse documentation." + ) return response try: @@ -67,23 +120,422 @@ async def execute(cls, coder, query=None, max_results=5, **kwargs): try: chunks = ConversationService.get_chunks(coder) - docs = chunks.retrieve_creator_docs(coder, query, n_results=limit) except Exception as exc: response.append_error(f"SearchDocs failed: {exc}") return response - if not docs: - response.append_result("No documentation matched this query.") + if path: + _append_path_results(response, chunks, coder, path, limit) return response - for doc in docs: - url = doc.get("url", "") - document = doc.get("document", "") - url_attr = f' from_url="{url}"' if url else "" + if mode == "content": + _append_content_results(response, chunks, coder, query, limit) + else: + _append_address_results(response, chunks, coder, query, limit) + + return response + + +def _navigation_has_results(navigation): + return bool(navigation.get("files") or navigation.get("sections")) + + +def _append_path_results(response, chunks, coder, path, limit): + retrieve_doc = getattr(chunks, "retrieve_creator_doc", None) + + if not callable(retrieve_doc): + response.append_error("SearchDocs path lookup is unavailable.") + return + + try: + result = retrieve_doc(coder, path, n_results=limit) + except Exception as exc: + response.append_error(f"SearchDocs failed: {exc}") + return + + if not isinstance(result, dict): + result = {} + + if result.get("error"): + response.append_error(str(result["error"])) + return + + toc = result.get("toc") + + if isinstance(toc, dict): + response.append_result( + _format_doc_toc(toc), + metadata={ + "type": "doc_toc", + "path": toc.get("path", ""), + "sections": len(toc.get("sections") or []), + "keys": len(toc.get("keys") or []), + }, + ) + return + + file = result.get("file") + + if file: + response.append_result( + _format_doc_file(file), + metadata=_doc_file_metadata(file), + ) + return + + files = result.get("files") or [] + dirs = result.get("dirs") or [] + + if files or dirs: + response.append_result( + _format_doc_paths(result), + metadata={ + "type": "doc_paths", + "path": result.get("path", ""), + "count": len(files) + len(dirs), + "total": result.get("total", len(files) + len(dirs)), + }, + ) + return + + suggestions = result.get("suggestions") or [] + + if suggestions: + response.append_result( + _format_doc_suggestions(result), + metadata={"type": "doc_path_suggestions", "path": result.get("path", "")}, + ) + return + + response.append_result(f"No documentation matched path={path!r}.") + + +def _doc_file_metadata(file): + metadata = { + "type": "doc_file", + "path": file.get("path", ""), + "url": file.get("url", ""), + "lines": file.get("lines", 0), + } + + if file.get("key"): + metadata["key"] = file["key"] + metadata["value"] = file.get("key_value", "") + metadata["range"] = [file.get("start_line"), file.get("end_line")] + elif file.get("section"): + metadata["section"] = file["section"] + metadata["level"] = file.get("level") + metadata["range"] = [file.get("start_line"), file.get("end_line")] + + return metadata + + +def _format_doc_file(file): + path = _xml_attr(file.get("path", "")) + url = _xml_attr(file.get("url") or "") + attrs = f' path="{path}"' + section = _xml_attr(file.get("section") or "") + key = _xml_attr(file.get("key") or "") + level = _xml_attr(file.get("level", "")) + start_line = file.get("start_line") + end_line = file.get("end_line") + lines = file.get("lines", 0) + + if url: + attrs += f' url="{url}"' + + if key: + attrs += f' key="{key}"' + attrs += f' lines="{start_line}-{end_line}"' + attrs += f' of_lines="{lines}"' + elif section: + attrs += f' section="{section}" level="{level}"' + attrs += f' lines="{start_line}-{end_line}"' + attrs += f' of_lines="{lines}"' + elif lines: + attrs += f' lines="{lines}"' + + return f"\n{file.get('text', '')}\n" + + +def _doc_size_label(item): + """Return the ``N lines, M sections`` label for one path entry.""" + label = f"{item.get('lines', 0)} lines" + sections = item.get("sections") or 0 + + if sections: + label += f", {sections} sections" + + return label + + +def _format_doc_paths(result): + files = result.get("files") or [] + dirs = result.get("dirs") or [] + count = len(files) + len(dirs) + total = result.get("total", count) + browse_path = result.get("path") or "." + attrs = f' path="{browse_path}" count="{count}"' + + if total > count: + attrs += f' total="{total}"' + + lines = [f""] + + for item in dirs: + lines.append(f"- {item.get('path', '')}/ (dir, {item.get('count', 0)} files)") + + for item in files: + description = item.get("description") or "" + url = item.get("url") or "" + trailing = f" {description}" if description else "" + + if url: + trailing += f" {url}" + + lines.append(f"- {item.get('path', '')} ({_doc_size_label(item)}){trailing}") + + lines.append("") + return "\n".join(lines) + - response.append_result( - f"\n{document}\n", - metadata={"id": doc.get("id", ""), "url": url}, +def _format_doc_suggestions(result): + path = result.get("path") or "" + suggestions = result.get("suggestions") or [] + lines = [f'No exact match for path="{path}". Did you mean:'] + + for item in suggestions: + description = item.get("description") or "" + url = item.get("url") or "" + trailing = f" {description}" if description else "" + + if url: + trailing += f" {url}" + + lines.append(f"- {item.get('path', '')} ({_doc_size_label(item)}){trailing}") + + return "\n".join(lines) + + +def _format_navigation(navigation, query): + files = navigation.get("files") or [] + sections = navigation.get("sections") or [] + + if not files and not sections: + return "" + + lines = ["", f"Query terms: {query}"] + + if files: + lines.append("Files with the most query terms:") + + for item in files: + terms = ", ".join(item.get("matched_terms") or []) + url = item.get("url") or "" + url_attr = f" {url}" if url else "" + lines.append( + f"- {item.get('path', '')} ({item.get('score', 0)} terms: {terms}; " + f"{item.get('section_count', 0)} matching sections){url_attr}" ) - return response + if sections: + lines.append("Sections with the most query terms:") + + for item in sections: + terms = ", ".join(item.get("matched_terms") or []) + url = item.get("url") or "" + url_attr = f" {url}" if url else "" + address = item.get("address") or "" + address_attr = f" address={address}" if address else "" + lines.append( + f"- {item.get('path', '')}:{item.get('line', 0)} " + f"{item.get('heading', '')} ({item.get('score', 0)} terms: {terms})" + f"{url_attr}{address_attr}" + ) + + lines.append("") + + return "\n".join(lines) + + +def _append_dropped_notice(response, dropped): + """Disclose results withheld for scoring below the relevance threshold.""" + if dropped <= 0: + return + + noun = "result" if dropped == 1 else "results" + response.append_result( + f'{dropped} more {noun} scored below the ' + "relevance threshold.", + metadata={"type": "doc_notice", "dropped": dropped}, + ) + + +def _append_content_results(response, chunks, coder, query, limit): + """Append documentation chunks, a navigation block, and a floor notice.""" + try: + docs = chunks.retrieve_creator_docs(coder, query, n_results=limit * _FETCH_FACTOR) + except Exception as exc: + response.append_error(f"SearchDocs failed: {exc}") + return + + docs = docs or [] + accepted = [doc for doc in docs if doc.get("distance", 0.0) <= _MAX_DISTANCE] + kept = accepted[:limit] + dropped = len(docs) - len(accepted) + navigation = {} + retrieve_navigation = getattr(chunks, "retrieve_creator_doc_navigation", None) + + if callable(retrieve_navigation): + try: + navigation = retrieve_navigation(coder, query, n_files=limit, n_sections=limit) + except Exception: + navigation = {} + + if not isinstance(navigation, dict): + navigation = {} + + if not kept and not _navigation_has_results(navigation): + response.append_result("No documentation matched this query.") + _append_dropped_notice(response, dropped) + return + + for doc in kept: + url = doc.get("url", "") + document = doc.get("document", "") + url_attr = f' from_url="{url}"' if url else "" + location = locate_doc_chunk(doc.get("id", "")) or {} + path = _xml_attr(location.get("path", "")) + heading = _xml_attr(location.get("heading", "")) + start_line = location.get("start_line") + end_line = location.get("end_line") + location_attrs = "" + metadata = {"id": doc.get("id", ""), "url": url} + + if path: + location_attrs += f' path="{path}"' + metadata["path"] = location["path"] + + if heading: + location_attrs += f' heading="{heading}"' + metadata["heading"] = location["heading"] + + if start_line: + location_attrs += f' lines="{start_line}-{end_line}"' + metadata["range"] = [start_line, end_line] + + response.append_result( + f"\n{document}\n", + metadata=metadata, + ) + + navigation_text = _format_navigation(navigation, query) + + if navigation_text: + response.append_result( + navigation_text, + metadata={ + "type": "doc_navigation", + "query": query, + "files": len(navigation.get("files") or []), + "sections": len(navigation.get("sections") or []), + }, + ) + + _append_dropped_notice(response, dropped) + + +def _append_address_results(response, chunks, coder, query, limit): + """Append ranked, feed-back-able documentation addresses.""" + retrieve_addresses = getattr(chunks, "retrieve_creator_doc_addresses", None) + + if not callable(retrieve_addresses): + response.append_error("Documentation address ranking is unavailable.") + return + + try: + result = retrieve_addresses(coder, query, n_results=limit) + except Exception as exc: + response.append_error(f"SearchDocs failed: {exc}") + return + + if not isinstance(result, dict): + result = {} + + addresses = result.get("addresses") or [] + + if not addresses: + response.append_result("No documentation matched this query.") + return + + response.append_result( + _format_doc_addresses(result, query), + metadata={ + "type": "doc_addresses", + "query": query, + "count": len(addresses), + "total": result.get("total", len(addresses)), + }, + ) + + +def _format_doc_addresses(result, query): + """Format ranked addresses as a ```` listing.""" + addresses = result.get("addresses") or [] + total = result.get("total") or len(addresses) + label = _xml_attr(query) + attrs = f' query="{label}" count="{len(addresses)}"' + + if total > len(addresses): + attrs += f' total="{total}"' + + lines = [f""] + + for address in addresses: + target = _xml_attr(address.get("address", "")) + + if address.get("kind") == "section": + lines.append(f'- {target} {address.get("heading", "")}') + else: + description = address.get("description") or "" + suffix = f" {description}" if description else "" + lines.append(f"- {target}{suffix}") + + lines.append("") + + return "\n".join(lines) + + +def _format_doc_toc(toc): + """Format a file outline as a ```` listing of resolvable slugs.""" + path = _xml_attr(toc.get("path", "")) + url = _xml_attr(toc.get("url") or "") + sections = toc.get("sections") or [] + keys = toc.get("keys") or [] + attrs = f' path="{path}"' + + if url: + attrs += f' url="{url}"' + + attrs += f' count="{len(sections)}"' + + if keys: + attrs += f' keys="{len(keys)}"' + + lines = [f""] + + for section in sections: + heading = section.get("heading", "") + slug = section.get("slug", "") + start = section.get("start", 0) + end = max(start, section.get("end", start) - 1) + lines.append(f"- {slug} {heading} (L{start}-{end})") + + lines.append("") + + return "\n".join(lines) + + +def _xml_attr(value): + """Escape a value for a quoted XML-style attribute.""" + return str(value or "").replace("&", "&").replace("<", "<").replace('"', """) diff --git a/cecli/tui/app.py b/cecli/tui/app.py index 164b0a8316e..f685c9d46d0 100644 --- a/cecli/tui/app.py +++ b/cecli/tui/app.py @@ -161,6 +161,18 @@ def __init__(self, coder_worker, output_queue, input_queue, args): description="Main Agent", show=True, ) + self.bind( + self._encode_keys(self.get_keys_for("panel_up")), + "scroll_panel_up", + description="Scroll Panels Up", + show=False, + ) + self.bind( + self._encode_keys(self.get_keys_for("panel_down")), + "scroll_panel_down", + description="Scroll Panels Down", + show=False, + ) self.bind( self._encode_keys(self.get_keys_for("cancel")), "interrupt", @@ -299,7 +311,9 @@ def _get_config(self): "output_down": "shift+pagedown", "next_agent": "alt+ctrl+right", "prev_agent": "alt+ctrl+left", - "main_agent": "alt+ctrl+up", + "main_agent": "alt+ctrl+home", + "panel_up": "alt+ctrl+up", + "panel_down": "alt+ctrl+down", "editor": "ctrl+o", "history": "alt+shift+h", "voice": "ctrl+r", @@ -1364,6 +1378,24 @@ def action_toggle_floating_panels(self): except Exception: pass + def action_scroll_panel_up(self): + """Scroll the floating status panels up one line.""" + self._scroll_floating_panels(-1) + + def action_scroll_panel_down(self): + """Scroll the floating status panels down one line.""" + self._scroll_floating_panels(1) + + def _scroll_floating_panels(self, offset: int): + """Scroll the floating status panels by a number of lines.""" + try: + panels = self.query_one(FloatingPanels) + if not panels.visible: + return + panels.scroll_by(offset) + except Exception: + pass + def action_noop(self): pass diff --git a/cecli/tui/widgets/floating_panels.py b/cecli/tui/widgets/floating_panels.py index 3b86176800c..fedad000077 100644 --- a/cecli/tui/widgets/floating_panels.py +++ b/cecli/tui/widgets/floating_panels.py @@ -190,6 +190,14 @@ def toggle_visible(self): if target.display: self.refresh_data() + def scroll_by(self, offset: int) -> None: + """Scroll the shared item list by a number of lines.""" + try: + scroll = self.query_one("#status-scroll", VerticalScroll) + scroll.scroll_relative(y=offset, animate=False, immediate=True) + except Exception: + pass + def refresh_data(self): """Refresh the extensions columns from the active coder's state.""" coder = self._get_active_coder() diff --git a/cecli/website/docs/benchmarks.md b/cecli/website/docs/benchmarks.md index 8044ca4c1e0..6ae1b05d160 100644 --- a/cecli/website/docs/benchmarks.md +++ b/cecli/website/docs/benchmarks.md @@ -84,7 +84,7 @@ I benchmarked 4 different edit formats, described below. Each description includ ### whole -The [whole](https://github.com/cecli-dev/cecli/blob/main/cecli/coders/wholefile_prompts.py) format asks GPT to return an updated copy of the entire file, including any changes. The file should be formatted with normal markdown triple-backtick fences, inlined with the rest of its response text. +The [whole](https://github.com/cecli-dev/cecli/blob/main/cecli/prompts/wholefile.yml) format asks GPT to return an updated copy of the entire file, including any changes. The file should be formatted with normal markdown triple-backtick fences, inlined with the rest of its response text. This format is very similar to how ChatGPT returns code snippets during normal chats, except with the addition of a filename right before the opening triple-backticks. @@ -100,7 +100,7 @@ def main(): ### diff -The [diff](https://github.com/cecli-dev/cecli/blob/main/cecli/coders/editblock_prompts.py) format also asks GPT to return edits as part of the normal response text, in a simple diff format. Each edit is a fenced code block that specifies the filename and a chunk of ORIGINAL and UPDATED code. GPT provides some original lines from the file and then a new updated set of lines. +The [diff](https://github.com/cecli-dev/cecli/blob/main/cecli/prompts/editblock.yml) format also asks GPT to return edits as part of the normal response text, in a simple diff format. Each edit is a fenced code block that specifies the filename and a chunk of ORIGINAL and UPDATED code. GPT provides some original lines from the file and then a new updated set of lines. ```` Here are the changes you requested to demo.py: diff --git a/cecli/website/docs/config/api.md b/cecli/website/docs/config/api.md index 42b4b309f06..07f6e2a708a 100644 --- a/cecli/website/docs/config/api.md +++ b/cecli/website/docs/config/api.md @@ -32,7 +32,7 @@ cecli --server-config '{"host": "127.0.0.1", "port": 23254}' You can also set it via your config file: ```yaml -# ~/.cecli/config.yml +# ~/.cecli/conf.yml server-config: port: 23254 host: "127.0.0.1" diff --git a/cecli/website/docs/config/conf.md b/cecli/website/docs/config/conf.md index ddbb504ba06..629473bfcc5 100644 --- a/cecli/website/docs/config/conf.md +++ b/cecli/website/docs/config/conf.md @@ -419,7 +419,7 @@ Below is a sample of the YAML config file, which you can also [download from Git #add-gitignore-files: false ## Specify the cecli ignore file (default: .cecli.ignore in git root) -#cecliignore: .cecli.ignore +#cecli-ignore: .cecli.ignore ## Only consider files in the current subtree of the git repository #subtree-only: false diff --git a/cecli/website/docs/config/creator.md b/cecli/website/docs/config/creator.md index f247847d6a9..240a98670e6 100644 --- a/cecli/website/docs/config/creator.md +++ b/cecli/website/docs/config/creator.md @@ -30,7 +30,7 @@ With a prompt, the creator starts working immediately. Without one, it waits for The creator is a separate agent, so it does not automatically see your current conversation. Briefly include the goal and any constraints in your request. -The creator looks documentation up on demand with its `SearchDocs` tool; cecli does not inject documentation into the creator's context automatically. +The creator looks up documentation on demand with its `SearchDocs` tool. `SearchDocs(query=...)` returns ranked, feed-back-able addresses of the form `path#heading-slug`, at most one per file; pass an address straight back as `SearchDocs(path=...)` to read it. `path` also accepts a whole file (`config/hooks.md`), one section (`config/hooks.md#configuration-options`), one config key from a sample block (`config/conf.md#dark-mode`), a directory or glob (`config/`, `config/*.md`), `/` for the docs root, and `file.md#` for that file's table of contents. `SearchDocs(query=..., mode="content")` returns chunk text instead of addresses; each chunk carries its docs path, heading, and line range, and navigation hints include copy-paste addresses. cecli does not inject documentation into the creator's context automatically. When building custom tools or commands, the creator sees a `creator_requirements` context block listing the top-level modules installed in the Python environment running cecli (whether managed by pip, uv, or another installer). It uses that list to decide what can be imported; if a dependency is missing, it should tell you what to install instead of generating an extension that cannot import. diff --git a/cecli/website/docs/config/custom-system-prompts.md b/cecli/website/docs/config/custom-system-prompts.md index e93651f23e1..d94f00dfdf2 100644 --- a/cecli/website/docs/config/custom-system-prompts.md +++ b/cecli/website/docs/config/custom-system-prompts.md @@ -83,7 +83,7 @@ main_system: | ## Core Directives - **Role**: Act as an expert software engineer. - - **Act Proactively**: Autonomously use file discovery and context management tools (`ViewFilesAtGlob`, `ViewFilesMatching`, `Ls`, `ResourceManager`) to gather information and fulfill the user's request. Chain tool calls across multiple turns to continue exploration. + - **Act Proactively**: Autonomously use file discovery and context management tools (`ExploreCode`, `Grep`, `ReadFile`, `ResourceManager`) to gather information and fulfill the user's request. Chain tool calls across multiple turns to continue exploration. - **Be Decisive**: Trust that your initial findings are valid. Refrain from asking the same question or searching for the same term in multiple similar ways. - **Be Concise**: Keep all responses brief and direct (1-3 sentences). Avoid preamble, postamble, and unnecessary explanations. Do not repeat yourself. - **Be Careful**: Break updates down into smaller, more manageable chunks. Focus on one thing at a time. diff --git a/cecli/website/docs/config/hooks.md b/cecli/website/docs/config/hooks.md index fce8d1173a2..11bb8b3e4be 100644 --- a/cecli/website/docs/config/hooks.md +++ b/cecli/website/docs/config/hooks.md @@ -1,7 +1,7 @@ --- parent: Configuration nav_order: 35 -description: Create and use custom commands to extend cecli's functionality. +description: Extend cecli with command and Python hooks that run at specific points in the agent's workflow. --- # Hooks diff --git a/cecli/website/docs/config/mcp.md b/cecli/website/docs/config/mcp.md index 8af15affee2..3de821b4423 100644 --- a/cecli/website/docs/config/mcp.md +++ b/cecli/website/docs/config/mcp.md @@ -106,7 +106,7 @@ mcp-servers-file: /path/to/mcp.json These options are configurable in any of cecli's config file formats. -Also, you are able to say if you would like an mcp enabled/disabled in the config itself via `"enabled"` key By default MCP servers are enabled, so you MUST explicitly disable them in the config if you dont wish for them to be included when cecli starts up +Also, you are able to say if you would like an mcp enabled/disabled in the config itself via `"enabled"` key. By default MCP servers are enabled, so you MUST explicitly disable them in the config if you don't wish for them to be included when cecli starts up. ### Flags diff --git a/cecli/website/docs/config/model-configuration.md b/cecli/website/docs/config/model-configuration.md index a53ac710b95..ebd82b27023 100644 --- a/cecli/website/docs/config/model-configuration.md +++ b/cecli/website/docs/config/model-configuration.md @@ -50,7 +50,7 @@ When you specify a model with a suffix, CECLI splits it into the base model name ## Configuration File Example -You can define these overrides in your `config.yml` file, a `.cecli.model.overrides.yml` file, or a custom file specified via `--model-overrides-file`. +You can define these overrides in your `.cecli.conf.yml` file, a `.cecli.model.overrides.yml` file, or a custom file specified via `--model-overrides-file`. ```yaml model-overrides: diff --git a/cecli/website/docs/config/options.md b/cecli/website/docs/config/options.md index 41b35d41490..19268329662 100644 --- a/cecli/website/docs/config/options.md +++ b/cecli/website/docs/config/options.md @@ -47,7 +47,7 @@ usage: cecli [-h] [--model] [--openai-api-key] [--anthropic-api-key] [--show-diffs] [--git | --no-git] [--gitignore | --no-gitignore] [--add-gitignore-files | --no-add-gitignore-files] - [--aiderignore] [--subtree-only] + [--cecli-ignore] [--subtree-only] [--auto-commits | --no-auto-commits] [--dirty-commits | --no-dirty-commits] [--attribute-author | --no-attribute-author] diff --git a/cecli/website/docs/config/security.md b/cecli/website/docs/config/security.md index 6030f463603..53c148cf7de 100644 --- a/cecli/website/docs/config/security.md +++ b/cecli/website/docs/config/security.md @@ -167,7 +167,7 @@ When using Agent Mode, security configuration works alongside other security fea ```yaml agent: true agent-config: - tools_excludelist: ["command", "commandinteractive"] + tools_excludelist: ["command"] security-config: allowed-domains: - github.com diff --git a/cecli/website/docs/config/tui.md b/cecli/website/docs/config/tui.md index c444dd14bbf..1f5e4c23329 100644 --- a/cecli/website/docs/config/tui.md +++ b/cecli/website/docs/config/tui.md @@ -69,7 +69,9 @@ tui-config: quit: "ctrl+q" next_agent: "alt+ctrl+right" prev_agent: "alt+ctrl+left" - main_agent: "alt+ctrl+up" + main_agent: "alt+ctrl+home" + panel_up: "alt+ctrl+up" + panel_down: "alt+ctrl+down" ``` @@ -98,7 +100,9 @@ The TUI provides customizable key bindings for all major actions. The default ke | Quit | `ctrl+q` | Exit the TUI | | Next Agent | `alt+ctrl+right` | Switch to the next agent (sub-agent or primary), wrapping around | | Previous Agent | `alt+ctrl+left` | Switch to the previous agent (sub-agent or primary), wrapping around | -| Main Agent | `alt+ctrl+up` | Switch back to the main (primary) agent | +| Main Agent | `alt+ctrl+home` | Switch back to the main (primary) agent | +| Panels Up | `alt+ctrl+up` | Scroll the floating status panels up one line | +| Panels Down | `alt+ctrl+down` | Scroll the floating status panels down one line | #### Customizing Key Bindings diff --git a/cecli/website/docs/git.md b/cecli/website/docs/git.md index 2cffa9ef516..a699f4e5cb9 100644 --- a/cecli/website/docs/git.md +++ b/cecli/website/docs/git.md @@ -43,7 +43,7 @@ While it is not recommended, you can disable cecli's use of git in a few ways: Cecli sends the `--weak-model` a copy of the diffs and the chat history and asks it to produce a commit message. By default, cecli creates commit messages which follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). -You can customize the [commit prompt](https://github.com/cecli-dev/cecli/blob/main/cecli/prompts.py#L5) with the `--commit-prompt` option. You can place that on the command line, or [configure it via a config file or environment variables](config.html). +You can customize the [commit prompt](https://github.com/cecli-dev/cecli/blob/main/cecli/prompts/utils/system.py#L8) with the `--commit-prompt` option. You can place that on the command line, or [configure it via a config file or environment variables](config.html). ## Commit attribution diff --git a/cecli/website/docs/repomap.md b/cecli/website/docs/repomap.md index 108bf9359f6..b62daabff6f 100644 --- a/cecli/website/docs/repomap.md +++ b/cecli/website/docs/repomap.md @@ -13,7 +13,7 @@ Cecli uses a **concise map of your whole git repository** that includes the most Cecli sends a **repo map** to the LLM along with each change request from the user. The repo map contains a list of the files in the repo, along with the key symbols which are defined in each file. It shows how each of these symbols are defined, by including the critical lines of code for each definition. -Here's a part of the repo map of cecli's repo, for [base_coder.py](https://github.com/cecli-dev/cecli/blob/main/cecli/coders/base_coder.py) and [commands.py](https://github.com/cecli-dev/cecli/blob/main/cecli/commands.py) : +Here's a part of the repo map of cecli's repo, for [base_coder.py](https://github.com/cecli-dev/cecli/blob/main/cecli/coders/base_coder.py) and [core.py](https://github.com/cecli-dev/cecli/blob/main/cecli/commands/core.py) : ``` cecli/coders/base_coder.py: @@ -35,7 +35,7 @@ cecli/coders/base_coder.py: │ def run(self, with_message=None): ⋮... -cecli/commands.py: +cecli/commands/core.py: ⋮... │class Commands: │ voice = None diff --git a/tests/basic/test_main.py b/tests/basic/test_main.py index 3248a37c54b..ad42b261c41 100644 --- a/tests/basic/test_main.py +++ b/tests/basic/test_main.py @@ -195,6 +195,26 @@ def test_main_with_git_config_yml(dummy_io, mock_coder, git_temp_dir): assert kwargs["auto_commits"] is True +def test_main_loads_single_workspace_from_local_file(dummy_io, git_temp_dir): + from cecli.helpers.agents.service import AgentService + + project_dir = Path(git_temp_dir) / "app" + make_repo(str(project_dir)) + Path(".cecli.workspaces.yml").write_text( + f"name: file-workspace\nprojects:\n - name: app\n path: {project_dir}\n" + ) + + registry = AgentService.get_registry() + previous_registry = dict(registry) + + try: + main(["--no-git", "--exit", "--yes-always"], **dummy_io) + assert "ws:app" in AgentService.get_registry() + finally: + registry.clear() + registry.update(previous_registry) + + def test_main_with_empty_git_dir_new_subdir_file(dummy_io, git_temp_dir): make_repo() subdir = Path("subdir") diff --git a/tests/basic/test_models.py b/tests/basic/test_models.py index 888d5324a73..9b873898399 100644 --- a/tests/basic/test_models.py +++ b/tests/basic/test_models.py @@ -140,6 +140,28 @@ async def test_sanity_check_model_calls_check_dependencies(self, mock_check_deps await sanity_check_model(mock_io, model) mock_check_deps.assert_called_once_with(mock_io, "test-model") + @patch("cecli.models.check_for_dependencies") + async def test_sanity_check_model_known_model_without_context(self, mock_check_deps): + """A known model with unknown limits should not be suggested as its own typo.""" + mock_io = MagicMock() + model = MagicMock() + model.name = "openrouter/~openai/gpt-luna-latest" + model.missing_keys = [] + model.keys_in_environment = True + model.info = {} + + matches = [model.name, "openrouter/~openai/gpt-luna"] + + with patch("cecli.models.fuzzy_match_models", return_value=matches): + await sanity_check_model(mock_io, model) + + warnings = [call.args[0] for call in mock_io.tool_warning.call_args_list] + assert any( + "context window size and/or costs are unknown" in message for message in warnings + ) + mock_io.tool_output.assert_not_called() + mock_check_deps.assert_called_once_with(mock_io, "openrouter/~openai/gpt-luna-latest") + def test_model_aliases(self): # Test common aliases model = Model("4") diff --git a/tests/commands/test_workflows.py b/tests/commands/test_workflows.py index 51ee2b6d679..416ba065af5 100644 --- a/tests/commands/test_workflows.py +++ b/tests/commands/test_workflows.py @@ -7,7 +7,7 @@ import pytest -from cecli.commands import CommandRegistry +from cecli.commands import CommandRegistry, Commands from cecli.commands.workflows import WorkflowCommand from cecli.helpers.workflows.models import TaskConfig, WorkflowConfig @@ -353,8 +353,27 @@ def test_completions(mock_coder): assert WorkflowCommand.get_completions(None, mock_coder, "ru") == ["run"] - names = WorkflowCommand.get_completions(None, mock_coder, "run de") - assert names == ["demo", "dep-check"] + for sub in ("run", "show", "status"): + names = WorkflowCommand.get_completions(None, mock_coder, f"{sub} de") + assert names == ["demo", "dep-check"] + + # A trailing space means the subcommand is settled with no name prefix + # yet, so all workflow names are offered immediately. + for sub in ("run", "show", "status"): + assert WorkflowCommand.get_completions(None, mock_coder, f"{sub} ") == [ + "demo", + "dep-check", + ] + assert WorkflowCommand.get_completions(None, mock_coder, f"{sub} d") == [ + "demo", + "dep-check", + ] + + assert WorkflowCommand.get_completions(None, mock_coder, "list ") == [] + + # Subcommand commands need the full argument text so the name is not + # mistaken for a subcommand prefix by the completer. + assert "/workflows" in Commands.FULL_ARGS_COMPLETION_COMMANDS assert WorkflowCommand.get_completions(None, mock_coder, "list something") == [] diff --git a/tests/help/test_help_navigation.py b/tests/help/test_help_navigation.py new file mode 100644 index 00000000000..f46d59bd15f --- /dev/null +++ b/tests/help/test_help_navigation.py @@ -0,0 +1,45 @@ +import cecli.help as help_module +from cecli.help import navigate_docs + + +def _write_doc(root, relative_path, text): + path = root / "website" / "docs" / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def test_navigate_docs_ranks_files_and_sections(monkeypatch, tmp_path): + hooks = _write_doc( + tmp_path, + "config/hooks.md", + "# Hook config\n\nhook config body\n\n## Config\n\nconfig details\n", + ) + usage = _write_doc(tmp_path, "usage.md", "# Usage\n\nhook only\n") + monkeypatch.setattr(help_module, "get_package_files", lambda: [hooks, usage]) + + navigation = navigate_docs("hook config", n_files=1, n_sections=2) + + assert navigation["files"][0]["path"] == "config/hooks.md" + assert navigation["files"][0]["matched_terms"] == ["hook", "config"] + assert navigation["files"][0]["section_count"] == 2 + + assert navigation["sections"][0]["path"] == "config/hooks.md" + assert navigation["sections"][0]["heading"] == "Hook config" + assert navigation["sections"][0]["line"] == 1 + assert navigation["sections"][0]["slug"] == "hook-config" + assert navigation["sections"][0]["address"] == "config/hooks.md#hook-config" + assert navigation["sections"][1]["heading"] == "Hook config > Config" + + +def test_navigate_docs_returns_empty_for_stopword_query(): + assert navigate_docs("the and") == {"files": [], "sections": []} + + +def test_iter_doc_sections_skips_front_matter_and_fenced_headings(): + text = "---\nparent: Test\n---\n\n# Top\n\n```markdown\n## Not a heading\n```\n\n## Real\n" + + sections = help_module._iter_doc_sections(text) + + assert [heading for heading, _, _ in sections] == ["Top", "Top > Real"] + assert sections[0][1] == 5 diff --git a/tests/tools/test_search_docs.py b/tests/tools/test_search_docs.py index 52b2c4a6dbd..68a3f7cc5ae 100644 --- a/tests/tools/test_search_docs.py +++ b/tests/tools/test_search_docs.py @@ -33,9 +33,13 @@ async def test_search_docs_returns_retrieved_chunks(monkeypatch): monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) coder = _coder() - response = await search_docs.Tool.execute(coder, query=" hook config ", max_results=3) + response = await search_docs.Tool.execute( + coder, query=" hook config ", mode="content", max_results=3 + ) - chunks.retrieve_creator_docs.assert_called_once_with(coder, "hook config", n_results=3) + call = chunks.retrieve_creator_docs.call_args + assert call.args == (coder, "hook config") + assert call.kwargs["n_results"] == 9 result = _result(response)["result"] assert len(result) == 1 assert "https://cecli.dev/docs/config/hooks/" in result[0]["content"] @@ -48,7 +52,7 @@ async def test_search_docs_reports_no_results(monkeypatch): chunks = SimpleNamespace(retrieve_creator_docs=Mock(return_value=[])) monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) - response = await search_docs.Tool.execute(_coder(), query="hooks") + response = await search_docs.Tool.execute(_coder(), query="hooks", mode="content") result = _result(response)["result"] assert "No documentation matched" in result[0]["content"] @@ -63,7 +67,7 @@ async def test_search_docs_rejects_empty_query(monkeypatch): data = _result(response) assert data["result"] == [] - assert "'query' must be a non-empty string." in data["errors"] + assert "Provide 'query'" in data["errors"][0] chunks.retrieve_creator_docs.assert_not_called() @@ -71,7 +75,7 @@ async def test_search_docs_rejects_empty_query(monkeypatch): async def test_search_docs_is_scoped_to_creator(monkeypatch): monkeypatch.setattr(search_docs, "is_creator_coder", lambda coder: False) - response = await search_docs.Tool.execute(_coder(creator=False), query="hooks") + response = await search_docs.Tool.execute(_coder(creator=False), query="hooks", mode="content") data = _result(response) assert data["result"] == [] @@ -83,9 +87,9 @@ async def test_search_docs_clamps_max_results(monkeypatch): chunks = SimpleNamespace(retrieve_creator_docs=Mock(return_value=[])) monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) - await search_docs.Tool.execute(_coder(), query="hooks", max_results=500) + await search_docs.Tool.execute(_coder(), query="hooks", mode="content", max_results=500) - assert chunks.retrieve_creator_docs.call_args.kwargs["n_results"] == 20 + assert chunks.retrieve_creator_docs.call_args.kwargs["n_results"] == 60 @pytest.mark.asyncio @@ -95,8 +99,118 @@ async def test_search_docs_reports_retrieval_failure(monkeypatch): ) monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) - response = await search_docs.Tool.execute(_coder(), query="hooks") + response = await search_docs.Tool.execute(_coder(), query="hooks", mode="content") data = _result(response) assert data["result"] == [] assert "SearchDocs failed: index unavailable" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_search_docs_appends_ranked_navigation(monkeypatch): + chunks = SimpleNamespace( + retrieve_creator_docs=Mock( + return_value=[ + { + "id": "config/hooks.md#0", + "url": "https://cecli.dev/docs/config/hooks/", + "document": "Hook documentation body.", + } + ] + ), + retrieve_creator_doc_navigation=Mock( + return_value={ + "files": [ + { + "path": "config/hooks.md", + "url": "https://cecli.dev/docs/config/hooks/", + "matched_terms": ["hook", "config"], + "score": 2, + "section_count": 4, + } + ], + "sections": [ + { + "path": "config/hooks.md", + "url": "https://cecli.dev/docs/config/hooks/", + "heading": "Hooks > Configuration", + "line": 12, + "matched_terms": ["hook", "config"], + "score": 2, + } + ], + } + ), + ) + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) + coder = _coder() + + response = await search_docs.Tool.execute( + coder, query=" hook config ", mode="content", max_results=3 + ) + + chunks.retrieve_creator_doc_navigation.assert_called_once_with( + coder, "hook config", n_files=3, n_sections=3 + ) + result = _result(response)["result"] + assert len(result) == 2 + assert result[0]["_"]["id"] == "config/hooks.md#0" + assert result[1]["_"]["type"] == "doc_navigation" + assert "" in result[1]["content"] + assert "config/hooks.md (2 terms: hook, config; 4 matching sections)" in result[1]["content"] + assert ( + "config/hooks.md:12 Hooks > Configuration (2 terms: hook, config)" in result[1]["content"] + ) + + +@pytest.mark.asyncio +async def test_search_docs_returns_navigation_without_chunks(monkeypatch): + chunks = SimpleNamespace( + retrieve_creator_docs=Mock(return_value=[]), + retrieve_creator_doc_navigation=Mock( + return_value={ + "files": [ + { + "path": "config/hooks.md", + "url": "", + "matched_terms": ["hooks"], + "score": 1, + "section_count": 2, + } + ], + "sections": [], + } + ), + ) + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) + + response = await search_docs.Tool.execute(_coder(), query="hooks", mode="content") + + result = _result(response)["result"] + assert len(result) == 1 + assert "No documentation matched" not in result[0]["content"] + assert "config/hooks.md (1 terms: hooks; 2 matching sections)" in result[0]["content"] + + +@pytest.mark.asyncio +async def test_navigation_failure_does_not_hide_chunks(monkeypatch): + chunks = SimpleNamespace( + retrieve_creator_docs=Mock( + return_value=[ + { + "id": "config/hooks.md#0", + "url": "https://cecli.dev/docs/config/hooks/", + "document": "Hook documentation body.", + } + ] + ), + retrieve_creator_doc_navigation=Mock(side_effect=RuntimeError("navigation failed")), + ) + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) + + response = await search_docs.Tool.execute(_coder(), query="hooks", mode="content") + + data = _result(response) + assert data["errors"] == [] + assert len(data["result"]) == 1 + assert "Hook documentation body." in data["result"][0]["content"] diff --git a/tests/tools/test_search_docs_addresses.py b/tests/tools/test_search_docs_addresses.py new file mode 100644 index 00000000000..1ac17fd6168 --- /dev/null +++ b/tests/tools/test_search_docs_addresses.py @@ -0,0 +1,257 @@ +"""Golden-output tests for SearchDocs address ranking, TOC, key anchors and floors.""" + +from types import SimpleNamespace + +import pytest + +from cecli.help import ( + get_doc, + keyword_retrieve, + locate_doc_chunk, + navigate_docs, + rank_doc_addresses, +) +from cecli.helpers.conversation import ConversationService +from cecli.tools import search_docs + + +def _result(response): + return response.to_dict() + + +def _coder(): + return SimpleNamespace(agent_config={"creator_mode": True}) + + +class _Chunks: + """Chunk service backed by the real docs resolver.""" + + docs = None + + def retrieve_creator_docs(self, coder, query, n_results=8): + if self.docs is not None: + return self.docs[:n_results] + + return keyword_retrieve(query, n_results=n_results) + + def retrieve_creator_doc_navigation(self, coder, query, n_files=5, n_sections=10): + return navigate_docs(query, n_files=n_files, n_sections=n_sections) + + def retrieve_creator_doc_addresses(self, coder, query, n_results=5): + return rank_doc_addresses(query, max_results=n_results) + + def retrieve_creator_doc(self, coder, path, n_results=20): + return get_doc(path, max_results=n_results) + + +@pytest.fixture(autouse=True) +def _patch_chunks(monkeypatch): + _Chunks.docs = None + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: _Chunks())) + + +@pytest.mark.asyncio +async def test_query_returns_addresses_by_default(): + data = _result(await search_docs.Tool.execute(_coder(), query="configure mcp servers")) + entry = data["result"][0] + + assert data["errors"] == [] + assert entry["_"]["type"] == "doc_addresses" + assert entry["_"]["count"] == 5 + + content = entry["content"] + assert content.startswith('") + assert "config/mcp.md#" in content + # addresses only: no embedded documentation bodies + assert "6 more results scored below the relevance threshold.' + ] + + +@pytest.mark.asyncio +async def test_relevance_floor_keeps_relevant_hits(): + _Chunks.docs = [ + dict(id="a.md#0", url="", document="relevant", distance=0.9), + dict(id="b.md#0", url="", document="edge", distance=1.4), + dict(id="c.md#0", url="", document="noise", distance=1.6), + ] + + data = _result(await search_docs.Tool.execute(_coder(), query="anything", mode="content")) + contents = [entry["content"] for entry in data["result"]] + + assert any("relevant" in content for content in contents) + assert any("edge" in content for content in contents) + assert not any("noise" in content for content in contents) + assert any('dropped="1"' in content for content in contents) + + +@pytest.mark.asyncio +async def test_toc_slugs_round_trip(): + data = _result(await search_docs.Tool.execute(_coder(), path="config/conf.md#")) + entry = data["result"][0] + + assert entry["_"]["type"] == "doc_toc" + assert entry["_"]["sections"] == 13 + assert entry["_"]["keys"] == 129 + assert ' Merge Precedence (L28-183)" in entry["content"] + + follow_up = _result( + await search_docs.Tool.execute(_coder(), path="config/conf.md#merge-precedence") + ) + section = follow_up["result"][0] + + assert section["_"]["section"] == "YAML Config File > Merge Precedence" + assert section["_"]["range"] == [28, 183] + + +@pytest.mark.asyncio +async def test_config_key_anchor_reads_the_key_pair(): + data = _result(await search_docs.Tool.execute(_coder(), path="config/conf.md#dark-mode")) + entry = data["result"][0] + + assert entry["_"]["type"] == "doc_file" + assert entry["_"]["key"] == "dark-mode" + assert entry["_"]["value"] == "false" + assert entry["_"]["range"] == [361, 362] + + content = entry["content"] + assert 'key="dark-mode"' in content + assert 'lines="361-362"' in content + assert "## Use colors suitable for a dark terminal background" in content + assert "#dark-mode: false" in content + + +@pytest.mark.asyncio +async def test_unknown_key_fails_loud(): + data = _result(await search_docs.Tool.execute(_coder(), path="config/conf.md#dark-modes")) + + assert data["result"] == [] + error = data["errors"][0] + assert 'No heading or key matched "dark-modes"' in error + assert "Closest key: dark-mode" in error + assert "129 keys" in error + + +@pytest.mark.asyncio +async def test_root_path_lists_docs_root(): + data = _result(await search_docs.Tool.execute(_coder(), path="/")) + entry = data["result"][0] + + assert entry["_"]["type"] == "doc_paths" + assert entry["_"]["total"] == 25 + assert "- config/ (dir, " in entry["content"] + + +@pytest.mark.asyncio +async def test_whole_file_read_strips_navigation_frontmatter(): + data = _result(await search_docs.Tool.execute(_coder(), path="config/hooks.md")) + content = data["result"][0]["content"] + + assert "parent:" not in content + assert "nav_order:" not in content + assert "description:" in content + assert 'lines="344"' in content + + +@pytest.mark.asyncio +async def test_glob_support_is_kept(): + data = _result(await search_docs.Tool.execute(_coder(), path="config/*.md", max_results=3)) + entry = data["result"][0] + + assert entry["_"]["type"] == "doc_paths" + assert entry["_"]["count"] == 3 + + +@pytest.mark.asyncio +async def test_address_ranking_unavailable_is_reported(monkeypatch): + chunks = SimpleNamespace() + + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: chunks)) + data = _result(await search_docs.Tool.execute(_coder(), query="hooks")) + + assert data["result"] == [] + assert "address ranking is unavailable" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_invalid_mode_is_rejected(): + data = _result(await search_docs.Tool.execute(_coder(), query="hooks", mode="bogus")) + + assert data["result"] == [] + assert "'mode' must be 'addresses' or 'content'." in data["errors"][0] + + +@pytest.mark.asyncio +async def test_no_match_query_returns_no_addresses(): + data = _result(await search_docs.Tool.execute(_coder(), query="xylophone quux zzzzq")) + + assert data["errors"] == [] + assert data["result"][0]["content"] == "No documentation matched this query." + + +@pytest.mark.asyncio +async def test_content_chunks_report_path_heading_and_line_range(): + docs = keyword_retrieve("mcp", n_results=8) + doc = next(item for item in docs if (locate_doc_chunk(item["id"]) or {}).get("heading")) + location = locate_doc_chunk(doc["id"]) + _Chunks.docs = [dict(doc, distance=0.5)] + + data = _result( + await search_docs.Tool.execute(_coder(), query="mcp", mode="content", max_results=1) + ) + entry = next(item for item in data["result"] if "type" not in item["_"]) + + assert entry["_"]["path"] == location["path"] + assert entry["_"]["heading"] == location["heading"] + assert entry["_"]["range"] == [location["start_line"], location["end_line"]] + assert 'path="{}"'.format(location["path"]) in entry["content"] + assert 'heading="{}"'.format(location["heading"]) in entry["content"] + assert 'lines="{}-{}"'.format(location["start_line"], location["end_line"]) in entry["content"] diff --git a/tests/tools/test_search_docs_paths.py b/tests/tools/test_search_docs_paths.py new file mode 100644 index 00000000000..446be599aa7 --- /dev/null +++ b/tests/tools/test_search_docs_paths.py @@ -0,0 +1,152 @@ +"""Tests for the SearchDocs path mode (whole-file read and path browsing).""" + +from types import SimpleNamespace + +import pytest + +from cecli.help import get_doc +from cecli.helpers.conversation import ConversationService +from cecli.tools import search_docs + + +def _result(response): + return response.to_dict() + + +def _coder(creator=True): + return SimpleNamespace(agent_config={"creator_mode": creator}) + + +class _Chunks: + """Minimal chunk service backed by the real docs resolver.""" + + def retrieve_creator_docs(self, coder, query, n_results=8): + return [] + + def retrieve_creator_doc(self, coder, path, n_results=20): + return get_doc(path, max_results=n_results) + + +@pytest.fixture(autouse=True) +def _patch_chunks(monkeypatch): + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: _Chunks())) + + +@pytest.mark.asyncio +async def test_read_returns_whole_file(): + response = await search_docs.Tool.execute(_coder(), path="config/hooks.md") + + result = _result(response)["result"] + assert len(result) == 1 + + entry = result[0] + assert entry["_"]["type"] == "doc_file" + assert entry["_"]["path"] == "config/hooks.md" + assert entry["_"]["url"] == "https://cecli.dev/docs/config/hooks/" + + content = entry["content"] + assert content.startswith('") + assert "# Hooks" in content + # The whole document, not a chunk: the last heading is present too. + assert "## Managing Hooks" in content + + +@pytest.mark.asyncio +async def test_read_accepts_omitted_suffix_and_url(): + omitted = await search_docs.Tool.execute(_coder(), path="config/hooks") + url = await search_docs.Tool.execute(_coder(), path="https://cecli.dev/docs/config/hooks/") + + for response in (omitted, url): + entry = _result(response)["result"][0] + assert entry["_"]["type"] == "doc_file" + assert entry["_"]["path"] == "config/hooks.md" + + +@pytest.mark.asyncio +async def test_browse_lists_files_and_reports_total(): + response = await search_docs.Tool.execute(_coder(), path="config/", max_results=3) + + entry = _result(response)["result"][0] + content = entry["content"] + + assert entry["_"]["type"] == "doc_paths" + assert entry["_"]["count"] == 3 + assert entry["_"]["total"] > entry["_"]["count"] + assert '" in content + + +@pytest.mark.asyncio +async def test_browse_lists_subdirectories_for_root(): + response = await search_docs.Tool.execute(_coder(), path=".", max_results=3) + + entry = _result(response)["result"][0] + content = entry["content"] + + assert entry["_"]["count"] == 3 + + assert "- config/ (dir," in content + + +@pytest.mark.asyncio +async def test_bare_basename_miss_returns_suggestion(): + response = await search_docs.Tool.execute(_coder(), path="hooks") + + data = _result(response) + assert data["errors"] == [] + + entry = data["result"][0] + assert entry["_"]["type"] == "doc_path_suggestions" + assert 'No exact match for path="hooks"' in entry["content"] + assert "config/hooks.md (344 lines," in entry["content"] + + +@pytest.mark.asyncio +async def test_unknown_path_reports_error_not_empty_listing(): + for path in ("zzz", "config/nope.md", "docs/missing"): + response = await search_docs.Tool.execute(_coder(), path=path) + + data = _result(response) + assert data["result"] == [] + assert "No documentation matched" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_absolute_path_is_rejected(): + response = await search_docs.Tool.execute(_coder(), path="/etc/passwd") + + data = _result(response) + assert data["result"] == [] + assert "Invalid documentation path" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_excluded_from_search_still_readable(): + response = await search_docs.Tool.execute(_coder(), path="ctags.md") + + entry = _result(response)["result"][0] + assert entry["_"]["type"] == "doc_file" + assert entry["_"]["path"] == "ctags.md" + + +@pytest.mark.asyncio +async def test_query_and_path_are_mutually_exclusive(): + response = await search_docs.Tool.execute(_coder(), query="hooks", path="config/hooks.md") + + data = _result(response) + assert data["result"] == [] + assert "not both" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_path_mode_is_scoped_to_creator(): + response = await search_docs.Tool.execute(_coder(creator=False), path="config/hooks.md") + + data = _result(response) + assert data["result"] == [] + assert "scoped to the creator sub-agent" in data["errors"][0] diff --git a/tests/tools/test_search_docs_sections.py b/tests/tools/test_search_docs_sections.py new file mode 100644 index 00000000000..bd29c5b1cb7 --- /dev/null +++ b/tests/tools/test_search_docs_sections.py @@ -0,0 +1,158 @@ +"""Tests for heading-scoped SearchDocs reads and browse section counts.""" + +from types import SimpleNamespace + +import pytest + +from cecli.help import get_doc +from cecli.helpers.conversation import ConversationService +from cecli.tools import search_docs + +CONF_PATH = "config/conf.md" +CONF_SECTION = "YAML Config File > Merge Precedence" + + +def _result(response): + return response.to_dict() + + +def _coder(creator=True): + return SimpleNamespace(agent_config={"creator_mode": creator}) + + +class _Chunks: + """Minimal chunk service backed by the real docs resolver.""" + + def retrieve_creator_docs(self, coder, query, n_results=8): + return [] + + def retrieve_creator_doc(self, coder, path, n_results=20): + return get_doc(path, max_results=n_results) + + +@pytest.fixture(autouse=True) +def _patch_chunks(monkeypatch): + monkeypatch.setattr(ConversationService, "get_chunks", staticmethod(lambda coder: _Chunks())) + + +@pytest.mark.asyncio +async def test_section_read_returns_only_that_subtree(): + response = await search_docs.Tool.execute(_coder(), path=f"{CONF_PATH}#Merge Precedence") + + entry = _result(response)["result"][0] + content = entry["content"] + + assert entry["_"]["section"] == CONF_SECTION + assert entry["_"]["level"] == 2 + assert entry["_"]["range"] == [28, 183] + assert entry["_"]["lines"] == 646 + + assert content.startswith(f' Deduplication" + assert entry["_"]["level"] == 3 + + +@pytest.mark.asyncio +async def test_section_read_accepts_url_anchors(): + for path in ( + "https://cecli.dev/docs/config/conf/#merge-precedence", + f"{CONF_PATH}#merge-precedence", + ): + response = await search_docs.Tool.execute(_coder(), path=path) + + entry = _result(response)["result"][0] + assert entry["_"]["section"] == CONF_SECTION + + +@pytest.mark.asyncio +async def test_unknown_heading_fails_with_suggestions(): + response = await search_docs.Tool.execute(_coder(), path=f"{CONF_PATH}#Merge Precedance") + + data = _result(response) + assert data["result"] == [] + + error = data["errors"][0] + assert 'No heading or key matched "Merge Precedance"' in error + assert CONF_SECTION in error + assert "13 headings" in error + + +@pytest.mark.asyncio +async def test_anchor_requires_an_exact_file(): + for path in ("config/#hooks", "config/*.md#hooks", "config/nope.md#hooks"): + response = await search_docs.Tool.execute(_coder(), path=path) + + data = _result(response) + assert data["result"] == [] + assert "Anchors require an exact file path" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_empty_anchor_returns_the_file_outline(): + response = await search_docs.Tool.execute(_coder(), path=f"{CONF_PATH}#") + + entry = _result(response)["result"][0] + + assert entry["_"]["type"] == "doc_toc" + assert entry["_"]["path"] == CONF_PATH + assert entry["_"]["sections"] > 0 + assert " 0 for entry in result["files"]) + + response = await search_docs.Tool.execute(_coder(), path="config/", max_results=2) + content = _result(response)["result"][0]["content"] + + assert " lines, " in content + assert " sections)" in content + + +@pytest.mark.asyncio +async def test_section_read_is_scoped_to_creator(): + response = await search_docs.Tool.execute( + _coder(creator=False), path=f"{CONF_PATH}#Merge Precedence" + ) + + data = _result(response) + assert data["result"] == [] + assert "scoped to the creator sub-agent" in data["errors"][0] + + +@pytest.mark.asyncio +async def test_section_attribute_escapes_quotes_in_headings(): + response = await search_docs.Tool.execute( + _coder(), path="config/model-providers.md#provider-not-found" + ) + + entry = _result(response)["result"][0] + + assert entry["_"]["section"] == 'Model Providers > Troubleshooting > "Provider not found"' + assert ( + 'section="Model Providers > Troubleshooting > "Provider not found""' + not in (entry["content"]) + ) + assert 'section="Model Providers > Troubleshooting > "Provider not found""' in ( + entry["content"] + ) diff --git a/tests/tui/test_floating_panels.py b/tests/tui/test_floating_panels.py index 5e5c93a2eb2..983197229b3 100644 --- a/tests/tui/test_floating_panels.py +++ b/tests/tui/test_floating_panels.py @@ -167,6 +167,14 @@ async def _run_panel_checks(): await pilot.pause() assert scroll.scroll_y > 0 + panels.scroll_by(-1) + await pilot.pause() + assert scroll.scroll_y == 4 + + panels.scroll_by(1) + await pilot.pause() + assert scroll.scroll_y == 5 + panels.on_status_item_clicked(StatusItemClicked("skills", "skill-19", True)) await pilot.pause() assert scroll.scroll_y == 0 @@ -220,4 +228,9 @@ def test_floating_panels_key_registration(): config = tui._get_config() assert config["key_bindings"]["panels"] == "alt+p" + assert config["key_bindings"]["panel_up"] == "alt+ctrl+up" + assert config["key_bindings"]["panel_down"] == "alt+ctrl+down" + assert config["key_bindings"]["main_agent"] == "alt+ctrl+home" assert hasattr(TUI, "action_toggle_floating_panels") + assert hasattr(TUI, "action_scroll_panel_up") + assert hasattr(TUI, "action_scroll_panel_down")