diff --git a/.env.example b/.env.example index e0068c1f..7e99fa37 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,16 @@ LLM_API_KEY=your-key-here # --- REST API server variables (read by `openkb-api` / `python -m openkb.api`) --- # OPENKB_API_TOKEN=... # required bearer token clients send as Authorization: Bearer # OPENKB_KB_ROOT=... # optional; where REST /init creates KBs (default: ~/.config/openkb/kbs) + +# --- Remote data sources (optional extras; not read from config.yaml) --- +# Amazon S3 — pip install "openkb[s3]" +OPENKB_S3_ACCESS_KEY_ID= +OPENKB_S3_SECRET_ACCESS_KEY= +# Google Drive — OAuth client you register. The refresh token is written by `openkb source add`, not by hand. +OPENKB_GDRIVE_CLIENT_ID= +OPENKB_GDRIVE_CLIENT_SECRET= +OPENKB_GDRIVE_REFRESH_TOKEN= +# OneDrive — same rule for the refresh token. +OPENKB_ONEDRIVE_CLIENT_ID= +OPENKB_ONEDRIVE_CLIENT_SECRET= +OPENKB_ONEDRIVE_REFRESH_TOKEN= diff --git a/AGENTS.md b/AGENTS.md index 0bda2d10..59f023c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,9 @@ humans steer, agents execute. Optimize changes for agent legibility. - `config.py` — config loading/validation (LiteLLM passthrough, env). - `converter.py` — document → markdown conversion (markitdown). - `url_ingest.py` — fetch & ingest URLs (trafilatura). +- `source_cmd.py` — `openkb source` remote bindings (S3). +- `connectors/` — list, download, and sync a remote prefix through `add_single_file` (S3, Google Drive). +- `api_connectors_router.py` — REST mirror of `openkb source`. - `images.py` — figure/image extraction & handling. - `indexer.py` — PageIndex tree indexing for long docs. - `mutation.py` — crash-safe, serial KB mutations. diff --git a/README.md b/README.md index 988bebda..dcd814fa 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,38 @@ Open `http://127.0.0.1:7566/` for the Workbench. Auth is off by default (local-f > Working on the UI itself? Run the Vite dev server with `cd frontend && npm install && npm run dev` (it proxies `/api` to a running `openkb-web`), or `npm run build` to regenerate the bundled `openkb/web/`. +### Amazon S3 + +Bind a bucket prefix and compile those objects into the local wiki. The SDK is an optional extra. Credentials use the CLI order: process environment, then `/.env`, then `~/.config/openkb/.env`. + +```bash +pip install "openkb[s3]" +# /.env +# OPENKB_S3_ACCESS_KEY_ID=... +# OPENKB_S3_SECRET_ACCESS_KEY=... + +openkb source add s3 --name papers --region us-east-1 --bucket my-bucket --prefix papers/ +openkb source sync papers +``` + +An empty `--prefix` is the bucket root. `openkb source remove` deletes the binding and leaves compiled pages in place. + +### Google Drive + +Bind a folder in the user's own Drive and compile those files into the local wiki. The client library is an optional extra. Create an OAuth client of type **Desktop app** in Google Cloud. Google's installed-app flow accepts the loopback redirect `http://127.0.0.1:7567/callback` (the [desktop-app guide](https://developers.google.com/identity/protocols/oauth2/native-app) documents `http://127.0.0.1:`, and a Desktop client has no redirect-URI field to fill in). OpenKB listens on `127.0.0.1` only. + +```bash +pip install "openkb[gdrive]" +# /.env — the refresh token is written by source add, not by hand +# OPENKB_GDRIVE_CLIENT_ID=... +# OPENKB_GDRIVE_CLIENT_SECRET=... + +openkb source add gdrive --name notes --folder "https://drive.google.com/drive/folders/FOLDER_ID" +openkb source sync notes +``` + +Scope is read-only access to that user's Drive. `openkb source add gdrive --non-interactive` saves the binding only when a refresh token is already stored. OneDrive is not available yet. + # 🧩 How OpenKB Works ### Architecture diff --git a/frontend/src/components/ConnectorCards.tsx b/frontend/src/components/ConnectorCards.tsx index 577a82eb..8ffd7196 100644 --- a/frontend/src/components/ConnectorCards.tsx +++ b/frontend/src/components/ConnectorCards.tsx @@ -1,14 +1,15 @@ +import { useCallback, useEffect, useState } from "react" import { Cloud, HardDrive, ThumbsUp, type LucideIcon } from "lucide-react" import { useTranslation } from "react-i18next" +import { ApiError, apiFetch } from "@/api/client" /** - * Remote document-source connectors are NOT implemented (no OAuth/S3/sync - * backend). Rather than dead "coming soon" tiles, each card links to a GitHub - * issue search where users 👍 to signal demand — a telemetry-free, OSS-native - * way to prioritize. The copy MUST never imply a connector is connected. + * Remote document-source connectors. * - * `voteUrl` is an interim per-connector label search on the canonical repo; swap - * to a specific `.../issues/` once a tracking issue exists (one-line change). + * A provider with no backend keeps the GitHub vote link (`label:connector:`). + * Amazon S3 and Google Drive are shipped: with a knowledge base selected, zero + * bindings show a connect form and one or more bindings show Sync / Remove / + * Add another. OneDrive stays a vote link. */ export interface Connector { id: string @@ -21,38 +22,261 @@ const REPO = "https://github.com/VectifyAI/OpenKB" const voteSearch = (label: string) => `${REPO}/issues?q=${encodeURIComponent(`is:issue label:connector:${label}`)}` +const SHIPPED = new Set(["s3", "gdrive"]) + +interface FieldSpec { + key: string + label: string + hint?: string +} + +const FORMS: Record = { + s3: { + needKb: "connector.needKb", + fields: [ + { key: "name", label: "connector.name" }, + { key: "region", label: "connector.region" }, + { key: "bucket", label: "connector.bucket" }, + { key: "prefix", label: "connector.prefix", hint: "connector.prefixHint" }, + ], + }, + gdrive: { + needKb: "connector.needKbDrive", + hint: "connector.gdriveTokenHint", + fields: [ + { key: "name", label: "connector.name" }, + { key: "folder", label: "connector.folder", hint: "connector.folderHint" }, + ], + }, +} + +function ProviderCard({ kb, connector }: { kb?: string; connector: Connector }) { + const { t } = useTranslation("common") + const spec = FORMS[connector.id] + const Icon = connector.icon + const [bindings, setBindings] = useState([]) + const [adding, setAdding] = useState(false) + const [pending, setPending] = useState<"save" | "sync" | "remove" | null>(null) + const [error, setError] = useState(null) + const [values, setValues] = useState>({}) + + const reload = useCallback(async () => { + if (!kb) return + const body = await apiFetch<{ connectors: BindingView[] }>( + `/api/v1/connectors?kb=${encodeURIComponent(kb)}`, + ) + setBindings(body.connectors.filter((item) => item.provider === connector.id)) + }, [kb, connector.id]) + + useEffect(() => { + if (!kb) return + let cancelled = false + reload().catch((exc: unknown) => { + if (!cancelled) setError(exc instanceof Error ? exc.message : String(exc)) + }) + return () => { + cancelled = true + } + }, [kb, reload]) + + const save = async () => { + if (!kb) return + setPending("save") + setError(null) + try { + await apiFetch("/api/v1/connectors", { + method: "POST", + body: { kb, provider: connector.id, ...values }, + }) + setValues({}) + setAdding(false) + await reload() + } catch (exc) { + const message = exc instanceof ApiError ? exc.message : String(exc) + setError(t("connector.formError", { error: message })) + } finally { + setPending(null) + } + } + + const syncOne = async (bindingName: string) => { + if (!kb) return + setPending("sync") + setError(null) + try { + await apiFetch("/api/v1/connectors/sync", { + method: "POST", + body: { kb, name: bindingName, non_interactive: true }, + }) + await reload() + } catch (exc) { + setError(exc instanceof Error ? exc.message : String(exc)) + } finally { + setPending(null) + } + } + + const removeOne = async (bindingName: string) => { + if (!kb) return + setPending("remove") + setError(null) + try { + await apiFetch("/api/v1/connectors/remove", { + method: "POST", + body: { kb, name: bindingName }, + }) + await reload() + } catch (exc) { + setError(exc instanceof Error ? exc.message : String(exc)) + } finally { + setPending(null) + } + } + + const showForm = Boolean(kb) && (bindings.length === 0 || adding) + + return ( +
+
+ + + +
{connector.label}
+
+ {!kb &&

{t(spec.needKb)}

} + {kb && spec.hint && bindings.length === 0 && ( +

{t(spec.hint)}

+ )} + {kb && bindings.length > 0 && ( +
    + {bindings.map((binding) => ( +
  • +
    {binding.name}
    +
    {binding.description}
    +
    + {binding.stale ? t("connector.notSynced") : t("connector.synced")} +
    + {binding.error &&
    {binding.error}
    } +
    + + +
    +
  • + ))} +
+ )} + {kb && bindings.length > 0 && !adding && ( + + )} + {showForm && ( +
{ + event.preventDefault() + void save() + }} + > + {spec.fields.map((field) => ( + + ))} + +
+ )} + {error &&

{error}

} +
+ ) +} + +/** Grid of connector cards. Pass `kb` on a knowledge-base page so a shipped provider can bind. */ +export default function ConnectorCards({ kb }: { kb?: string }) { + return ( +
+ {CONNECTORS.map((connector) => + SHIPPED.has(connector.id) ? ( + + ) : ( + + ), + )} +
+ ) +} + export const CONNECTORS: Connector[] = [ { id: "gdrive", label: "Google Drive", icon: Cloud, voteUrl: voteSearch("gdrive") }, { id: "s3", label: "Amazon S3", icon: HardDrive, voteUrl: voteSearch("s3") }, { id: "onedrive", label: "OneDrive", icon: Cloud, voteUrl: voteSearch("onedrive") }, ] -/** Grid of connector "want this?" cards linking to GitHub demand-vote searches. */ -export default function ConnectorCards() { +interface BindingView { + name: string + provider: string + description: string + stale: boolean + error: string | null +} + +const inputCls = + "mt-1 w-full h-8 rounded-md border border-input bg-transparent px-2 text-[12px] font-mono2 outline-none focus-visible:ring-2 focus-visible:ring-ring" + +function VoteCard({ connector }: { connector: Connector }) { const { t } = useTranslation("common") + const Icon = connector.icon return ( -
- {CONNECTORS.map((c) => ( - - - - -
-
{c.label}
-
- - {t("connector.vote")} -
-
-
- ))} -
+ + + + +
+
{connector.label}
+
+ + {t("connector.vote")} +
+
+
) } + diff --git a/frontend/src/locales/en/common.json b/frontend/src/locales/en/common.json index 91430075..44274597 100644 --- a/frontend/src/locales/en/common.json +++ b/frontend/src/locales/en/common.json @@ -54,8 +54,28 @@ "baseLabel": "API address (optional; leave blank to use the current origin)", "connect": "Connect" }, - "connector": { + "connector": { "tooltip": "Cloud sync is in development · vote for it on GitHub", - "vote": "Want it? Vote" + "vote": "Want it? Vote", + "connect": "Connect", + "addAnother": "Add another", + "sync": "Sync", + "remove": "Remove", + "name": "Name", + "region": "Region", + "bucket": "Bucket", + "prefix": "Prefix", + "prefixHint": "Empty prefix is the whole bucket", + "notSynced": "Not synced since this change", + "synced": "Synced", + "save": "Save binding", + "saving": "Saving…", + "needKb": "Open a knowledge base to connect Amazon S3.", + "needKbDrive": "Open a knowledge base to connect Google Drive.", + "folder": "Folder", + "folderHint": "A folder URL or id from your own Drive", + "gdriveTokenHint": "Sign in once with openkb source add gdrive if this knowledge base has no refresh token yet.", + "missingExtra": "Install the s3 extra on the server: pip install \"openkb[s3]\"", + "formError": "Could not save the binding: {{error}}" } } diff --git a/frontend/src/locales/en/kb.json b/frontend/src/locales/en/kb.json index d3543146..c8f4360e 100644 --- a/frontend/src/locales/en/kb.json +++ b/frontend/src/locales/en/kb.json @@ -82,7 +82,7 @@ }, "remote": { "heading": "Remote data sources", - "note": "Cloud connectors are in development and not yet available · want one? Click a card to vote on GitHub" + "note": "Amazon S3 and Google Drive connect on this page. OneDrive is not available yet — click a card to vote on GitHub." }, "overview": { "index": { "label": "Index", "caption": "wiki root" }, diff --git a/frontend/src/locales/en/settings.json b/frontend/src/locales/en/settings.json index 30ccd99b..757bcaa9 100644 --- a/frontend/src/locales/en/settings.json +++ b/frontend/src/locales/en/settings.json @@ -29,7 +29,7 @@ "kbRootEnvPinned": "Pinned by the OPENKB_KB_ROOT environment variable; edits here won't take effect until it's unset." }, "conn": { - "note": "Cloud data-source connectors (OAuth / S3) are in development and not yet available; for now, upload local files from the KB detail page. Want a connector? Click a card below to vote on GitHub and help us prioritize." + "note": "Amazon S3 and Google Drive are connected from a knowledge base page. OneDrive is not available yet — click a card to vote on GitHub." }, "about": { "links": { diff --git a/frontend/src/locales/zh/common.json b/frontend/src/locales/zh/common.json index 09edf636..c0b60b77 100644 --- a/frontend/src/locales/zh/common.json +++ b/frontend/src/locales/zh/common.json @@ -56,6 +56,26 @@ }, "connector": { "tooltip": "云端同步开发中 · 去 GitHub 投票表达需求", - "vote": "想要它?去投票" + "vote": "想要它?去投票", + "connect": "连接", + "addAnother": "再添加一个", + "sync": "同步", + "remove": "移除", + "name": "名称", + "region": "区域", + "bucket": "存储桶", + "prefix": "前缀", + "prefixHint": "前缀留空表示整个存储桶", + "notSynced": "更改后尚未同步", + "synced": "已同步", + "save": "保存绑定", + "saving": "正在保存…", + "needKb": "请打开一个知识库以连接 Amazon S3。", + "needKbDrive": "请打开一个知识库以连接 Google Drive。", + "folder": "文件夹", + "folderHint": "你自己的云端硬盘文件夹链接或 ID", + "gdriveTokenHint": "如果这个知识库还没有刷新令牌,请先运行一次 openkb source add gdrive 完成登录。", + "missingExtra": "请在服务器安装 s3 扩展:pip install \"openkb[s3]\"", + "formError": "无法保存绑定:{{error}}" } } diff --git a/frontend/src/locales/zh/kb.json b/frontend/src/locales/zh/kb.json index f93280cd..8df8e749 100644 --- a/frontend/src/locales/zh/kb.json +++ b/frontend/src/locales/zh/kb.json @@ -82,7 +82,7 @@ }, "remote": { "heading": "远程数据源", - "note": "云端连接器开发中,尚不可用 · 想要它?点卡片去 GitHub 投票" + "note": "Amazon S3 与 Google Drive 可在本页连接。OneDrive 尚不可用 — 点卡片去 GitHub 投票。" }, "overview": { "index": { "label": "索引", "caption": "wiki 根" }, diff --git a/frontend/src/locales/zh/settings.json b/frontend/src/locales/zh/settings.json index 5d60a145..a971d51d 100644 --- a/frontend/src/locales/zh/settings.json +++ b/frontend/src/locales/zh/settings.json @@ -29,7 +29,7 @@ "kbRootEnvPinned": "已由环境变量 OPENKB_KB_ROOT 固定,此处的修改在取消该环境变量前不会生效。" }, "conn": { - "note": "云端数据源连接器(OAuth / S3)开发中,尚不可用;当前请在知识库详情页手动上传本地文件。想要某个连接器?点下方卡片去 GitHub 投票,帮我们排优先级。" + "note": "Amazon S3 与 Google Drive 在知识库页面连接。OneDrive 尚不可用 — 点下方卡片去 GitHub 投票。" }, "about": { "links": { diff --git a/frontend/src/pages/KbDetail.tsx b/frontend/src/pages/KbDetail.tsx index c4c5935e..00af3087 100644 --- a/frontend/src/pages/KbDetail.tsx +++ b/frontend/src/pages/KbDetail.tsx @@ -1266,13 +1266,13 @@ function DocumentsPane({ ))} - {/* Remote connectors: no backend. Reframed as GitHub feature-request voting; never fake a connected state. */} + {/* S3 binds on this page. Drive and OneDrive stay vote links until they ship. */}

{t('kb:remote.heading')}

{t('kb:remote.note')}

- +
diff --git a/openkb/api.py b/openkb/api.py index 16b5db87..ab3cc2d7 100644 --- a/openkb/api.py +++ b/openkb/api.py @@ -31,6 +31,7 @@ from openkb.agent.query import build_run_config_from_bundle, run_query from openkb.api_config import apply_kb_config_patch, read_kb_config from openkb.api_config_router import config_router +from openkb.api_connectors_router import connectors_router from openkb.api_documents_router import documents_router from openkb.api_graph import graph_router from openkb.api_helpers import ( @@ -169,6 +170,7 @@ async def lifespan(app: FastAPI): app.include_router(kbs_router) app.include_router(pages_router) app.include_router(documents_router) + app.include_router(connectors_router) @app.get("/api/v1/kbs", response_model=KbListResponse) async def list_kbs_endpoint( diff --git a/openkb/api_config.py b/openkb/api_config.py index 6491731b..2f21c66c 100644 --- a/openkb/api_config.py +++ b/openkb/api_config.py @@ -37,7 +37,7 @@ resolve_entity_types, save_config, ) -from openkb.locks import atomic_write_text +from openkb.envfile import merge_env_file logger = logging.getLogger(__name__) @@ -93,46 +93,10 @@ def _reject_credential_newlines( def _merge_patch_env(env_path: Path, updates: dict[str, str | None]) -> None: """Securely merge-patch a ``.env`` credential file (shared KB/global writer). - Parses existing ``KEY=VALUE`` lines, tolerating a leading ``export `` so the - key normalizes: ``export LLM_API_KEY=x`` is stored under ``LLM_API_KEY``, - exactly what python-dotenv strips on read-back. Without this an ``export ``- - prefixed line would be keyed ``"export LLM_API_KEY"``, so a later - ``pop("LLM_API_KEY")`` (RFC 7386 clear) would MISS it and leave a live - credential on disk. Applies ``updates`` (a ``None`` value REMOVES the key, - else sets it), then writes atomically. - - The file is tightened to 0o600 BEFORE the atomic write so the LLM key is - never world-readable — not even briefly: atomic_write_text copies the - target's *current* mode onto its private temp file before the rename (see - locks._target_mode), so touch(0o600)+chmod(0o600) makes os.replace land a - 0o600 ``.env`` with no widen-then-chmod gap. Callers MUST hold the relevant - write lock (the per-KB mutation lock or the global-config lock): this is a - read-modify-write. Credential VALUES are never logged. + Callers MUST hold the relevant write lock. The bytes and the ``0600`` mode + are :func:`openkb.envfile.merge_env_file`. """ - env_lines: dict[str, str] = {} - if env_path.exists(): - for raw in env_path.read_text(encoding="utf-8").splitlines(): - line = raw.strip() - if not line or line.startswith("#"): - continue - # Strip a leading ``export `` (shell-style .env) BEFORE partition so - # the key normalizes and a later removal matches; python-dotenv - # strips it too when it reads the file back. - if line.startswith("export ") or line.startswith("export\t"): - line = line[len("export") :].lstrip() - if "=" not in line: - continue - k, _eq, v = line.partition("=") - env_lines[k.strip()] = v - for key, value in updates.items(): - if value is None: - env_lines.pop(key, None) - else: - env_lines[key] = value - env_path.parent.mkdir(parents=True, exist_ok=True) - env_path.touch(mode=0o600, exist_ok=True) - env_path.chmod(0o600) - atomic_write_text(env_path, "".join(f"{k}={v}\n" for k, v in env_lines.items())) + merge_env_file(env_path, updates) def read_kb_config(kb_dir: Path) -> KbConfigResponse: diff --git a/openkb/api_connectors_router.py b/openkb/api_connectors_router.py new file mode 100644 index 00000000..137ecf90 --- /dev/null +++ b/openkb/api_connectors_router.py @@ -0,0 +1,187 @@ +"""REST mirror of ``openkb source``. + +Lock-holding work runs in a worker thread. ``kb_lock`` reentrancy is +per-thread, so these handlers must not take the lock on the event loop. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field +from starlette.concurrency import run_in_threadpool + +from openkb.api_helpers import _resolve_kb, require_bearer_token +from openkb.connectors.backend import ProviderError +from openkb.connectors.credentials import s3_credentials +from openkb.connectors.gdrive import prepare_gdrive_binding +from openkb.connectors.model import parse_s3_binding +from openkb.connectors.state import KbBusyError +from openkb.connectors.store import remove_binding, upsert_binding +from openkb.connectors.sync import binding_views, sync_bindings + +connectors_router = APIRouter() + + +class ConnectorView(BaseModel): + name: str + provider: str + location: dict[str, str] + description: str + stale: bool + last_sync_at: str | None = None + added: int = 0 + skipped: int = 0 + updated: int = 0 + failed: int = 0 + error: str | None = None + + +class ConnectorListResponse(BaseModel): + connectors: list[ConnectorView] + + +class ConnectorAddRequest(BaseModel): + kb: str = Field(..., min_length=1) + name: str + provider: str = "s3" + region: str = "" + bucket: str = "" + prefix: str = "" + folder: str = "" + + +class ConnectorRemoveRequest(BaseModel): + kb: str = Field(..., min_length=1) + name: str + + +class ConnectorSyncRequest(BaseModel): + kb: str = Field(..., min_length=1) + name: str | None = None + non_interactive: bool = True + + +class BindingSyncView(BaseModel): + name: str + provider: str + ok: bool + added: int + skipped: int + updated: int + failed: int + error: str | None = None + + +class ConnectorSyncResponse(BaseModel): + exit_code: int + results: list[BindingSyncView] + + +class ConnectorMessage(BaseModel): + message: str + + +def _add(kb_dir, request: ConnectorAddRequest) -> str: + if request.provider == "gdrive": + return prepare_gdrive_binding( + kb_dir, + name=request.name, + folder=request.folder, + non_interactive=True, + ) + if request.provider != "s3": + raise ValueError("Only the s3 and gdrive providers are available.") + if s3_credentials(kb_dir) is None: + raise ValueError( + "S3 credentials are missing. Set OPENKB_S3_ACCESS_KEY_ID and " + "OPENKB_S3_SECRET_ACCESS_KEY in the environment or /.env." + ) + binding = parse_s3_binding( + name=request.name, + region=request.region, + bucket=request.bucket, + prefix=request.prefix, + ) + return upsert_binding(kb_dir, binding) + + +@connectors_router.get("/api/v1/connectors", response_model=ConnectorListResponse) +async def list_connectors_endpoint( + kb: str = Query(...), + _: None = Depends(require_bearer_token), +) -> ConnectorListResponse: + kb_dir = _resolve_kb(kb) + views = await run_in_threadpool(binding_views, kb_dir) + return ConnectorListResponse(connectors=[ConnectorView(**view) for view in views]) + + +@connectors_router.post("/api/v1/connectors", response_model=ConnectorMessage) +async def add_connector_endpoint( + request: ConnectorAddRequest, + _: None = Depends(require_bearer_token), +) -> ConnectorMessage: + kb_dir = _resolve_kb(request.kb) + try: + message = await run_in_threadpool(_add, kb_dir, request) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ProviderError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except KbBusyError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return ConnectorMessage(message=message) + + +@connectors_router.post("/api/v1/connectors/remove", response_model=ConnectorMessage) +async def remove_connector_endpoint( + request: ConnectorRemoveRequest, + _: None = Depends(require_bearer_token), +) -> ConnectorMessage: + kb_dir = _resolve_kb(request.kb) + + def _remove(): + return remove_binding(kb_dir, request.name) + + try: + message = await run_in_threadpool(_remove) + except KeyError as exc: + raise HTTPException(status_code=404, detail=f"No binding named {request.name!r}.") from exc + except KbBusyError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return ConnectorMessage(message=message) + + +@connectors_router.post("/api/v1/connectors/sync", response_model=ConnectorSyncResponse) +async def sync_connectors_endpoint( + request: ConnectorSyncRequest, + _: None = Depends(require_bearer_token), +) -> ConnectorSyncResponse: + kb_dir = _resolve_kb(request.kb) + + def _sync(): + return sync_bindings( + kb_dir, + name=request.name, + non_interactive=request.non_interactive, + ) + + try: + outcome = await run_in_threadpool(_sync) + except KbBusyError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return ConnectorSyncResponse( + exit_code=outcome.exit_code, + results=[ + BindingSyncView( + name=item.name, + provider=item.provider, + ok=item.ok, + added=item.added, + skipped=item.skipped, + updated=item.updated, + failed=item.failed, + error=item.error, + ) + for item in outcome.reports + ], + ) diff --git a/openkb/cli.py b/openkb/cli.py index c9e54318..8b50ca47 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -445,15 +445,31 @@ def _run_compile_with_retry(coro_factory, label: str) -> None: def add_single_file( - file_path: Path, kb_dir: Path, *, stage: bool = True, bundle=None + file_path: Path, + kb_dir: Path, + *, + stage: bool = True, + bundle=None, + identity: str | None = None, ) -> Literal["added", "skipped", "failed"]: - """Convert, index, and compile a single document under the KB mutation lock.""" + """Convert, index, and compile a single document under the KB mutation lock. + + ``identity`` is a stable registry key for a remote object (for example + ``s3://bucket/key``). Local adds leave it unset and keep using the file path. + """ with kb_ingest_lock(kb_dir / ".openkb"): - return _add_single_file_locked(file_path, kb_dir, stage=stage, bundle=bundle) + return _add_single_file_locked( + file_path, kb_dir, stage=stage, bundle=bundle, identity=identity + ) def _add_single_file_locked( - file_path: Path, kb_dir: Path, *, stage: bool = True, bundle=None + file_path: Path, + kb_dir: Path, + *, + stage: bool = True, + bundle=None, + identity: str | None = None, ) -> Literal["added", "skipped", "failed"]: """Convert, index, and compile a single document into the knowledge base. @@ -487,7 +503,11 @@ def _add_single_file_locked( # 2. Convert document into staging when possible. click.echo(f"Adding: {file_path.name}") try: - result = convert_document(file_path, kb_dir, staging_dir=staging_dir) + # Omit the keyword when unset so local adds keep the historical call. + if identity is None: + result = convert_document(file_path, kb_dir, staging_dir=staging_dir) + else: + result = convert_document(file_path, kb_dir, staging_dir=staging_dir, identity=identity) except Exception as exc: click.echo(f" [ERROR] Conversion failed: {exc}") logger.debug("Conversion traceback:", exc_info=True) @@ -587,7 +607,7 @@ def commit_body(snapshot) -> None: "name": file_path.name, "doc_name": doc_name, "type": doc_type, - "path": _registry_path(file_path, kb_dir), + "path": identity if identity else _registry_path(file_path, kb_dir), } if result.raw_path is not None: meta["raw_path"] = _registry_path(result.raw_path, kb_dir) @@ -3806,3 +3826,13 @@ async def run_lint_report( "lint_files_changed": lint_files_changed, "lint_ghosts_removed": lint_ghosts_removed, } + + +def _register_source_command() -> None: + """Attach ``openkb source`` without importing it while this module loads.""" + from openkb.source_cmd import source as source_command + + cli.add_command(source_command) + + +_register_source_command() diff --git a/openkb/connectors/__init__.py b/openkb/connectors/__init__.py new file mode 100644 index 00000000..9f6c96a9 --- /dev/null +++ b/openkb/connectors/__init__.py @@ -0,0 +1,4 @@ +"""Remote data-source connectors. + +S3 is the first backend. Drive and OneDrive reuse this package later. +""" diff --git a/openkb/connectors/backend.py b/openkb/connectors/backend.py new file mode 100644 index 00000000..4113e8c5 --- /dev/null +++ b/openkb/connectors/backend.py @@ -0,0 +1,38 @@ +"""List-and-download interface shared by remote backends.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + + +class TransientError(Exception): + """A retryable HTTP failure (throttle, timeout, 5xx).""" + + +class ProviderError(Exception): + """A failure of one provider. Other providers still run.""" + + +@dataclass(frozen=True) +class RemoteObject: + """One remote file selected for ingest. + + ``locator`` is the provider's download handle (an S3 key is already in + ``identity``; Drive stores a file id here). + """ + + identity: str + name: str + locator: str = "" + + +class ObjectBackend(Protocol): + """List objects under a binding and download one of them.""" + + def list_objects(self) -> list[RemoteObject]: + """Return every object under the binding.""" + + def download(self, obj: RemoteObject, dest: Path) -> None: + """Write ``obj`` to ``dest``.""" diff --git a/openkb/connectors/credentials.py b/openkb/connectors/credentials.py new file mode 100644 index 00000000..09f258cb --- /dev/null +++ b/openkb/connectors/credentials.py @@ -0,0 +1,87 @@ +"""Connector credentials in CLI order. + +Process environment, then ``/.env``, then ``~/.config/openkb/.env``. +This is intentionally not :func:`openkb.config.resolve_credential_bundle`, +which lets the KB file win over the process environment. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from openkb.config import GLOBAL_CONFIG_DIR + +S3_ACCESS_KEY_ID = "OPENKB_S3_ACCESS_KEY_ID" +S3_SECRET_ACCESS_KEY = "OPENKB_S3_SECRET_ACCESS_KEY" +GDRIVE_CLIENT_ID = "OPENKB_GDRIVE_CLIENT_ID" +GDRIVE_CLIENT_SECRET = "OPENKB_GDRIVE_CLIENT_SECRET" +GDRIVE_REFRESH_TOKEN = "OPENKB_GDRIVE_REFRESH_TOKEN" +ONEDRIVE_CLIENT_ID = "OPENKB_ONEDRIVE_CLIENT_ID" +ONEDRIVE_CLIENT_SECRET = "OPENKB_ONEDRIVE_CLIENT_SECRET" +ONEDRIVE_REFRESH_TOKEN = "OPENKB_ONEDRIVE_REFRESH_TOKEN" + +CONNECTOR_ENV_KEYS = ( + S3_ACCESS_KEY_ID, + S3_SECRET_ACCESS_KEY, + GDRIVE_CLIENT_ID, + GDRIVE_CLIENT_SECRET, + GDRIVE_REFRESH_TOKEN, + ONEDRIVE_CLIENT_ID, + ONEDRIVE_CLIENT_SECRET, + ONEDRIVE_REFRESH_TOKEN, +) + + +def _dotenv_get(path: Path, key: str) -> str | None: + if not path.is_file(): + return None + from dotenv import dotenv_values + + value = dotenv_values(path).get(key) + if value is None or value == "": + return None + return value + + +def resolve_connector_credential(kb_dir: Path, key: str) -> str | None: + """Return the first non-empty value for ``key`` in CLI order.""" + process = os.environ.get(key) + if process: + return process + kb_value = _dotenv_get(kb_dir / ".env", key) + if kb_value: + return kb_value + return _dotenv_get(GLOBAL_CONFIG_DIR / ".env", key) + + +def s3_credentials(kb_dir: Path) -> tuple[str, str] | None: + """Return the S3 key pair, or ``None`` when either key is absent.""" + access = resolve_connector_credential(kb_dir, S3_ACCESS_KEY_ID) + secret = resolve_connector_credential(kb_dir, S3_SECRET_ACCESS_KEY) + if not access or not secret: + return None + return access, secret + + +def gdrive_client(kb_dir: Path) -> tuple[str, str] | None: + """Return the OAuth client id and secret, or ``None`` when either is absent.""" + client_id = resolve_connector_credential(kb_dir, GDRIVE_CLIENT_ID) + secret = resolve_connector_credential(kb_dir, GDRIVE_CLIENT_SECRET) + if not client_id or not secret: + return None + return client_id, secret + + +def gdrive_refresh_token(kb_dir: Path) -> str | None: + return resolve_connector_credential(kb_dir, GDRIVE_REFRESH_TOKEN) + + +def resolved_secrets(kb_dir: Path) -> list[str]: + """Credential values currently visible, used to keep them out of state files.""" + found: list[str] = [] + for key in CONNECTOR_ENV_KEYS: + value = resolve_connector_credential(kb_dir, key) + if value: + found.append(value) + return found diff --git a/openkb/connectors/gdrive.py b/openkb/connectors/gdrive.py new file mode 100644 index 00000000..5b659f07 --- /dev/null +++ b/openkb/connectors/gdrive.py @@ -0,0 +1,284 @@ +"""Google Drive list and download. + +The API client is imported only when a real backend is built. Tests pass a +fake client and never import the SDK. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import NoReturn, Protocol + +from openkb.connectors.backend import ProviderError, RemoteObject, TransientError +from openkb.connectors.credentials import GDRIVE_REFRESH_TOKEN, gdrive_client, gdrive_refresh_token +from openkb.connectors.model import GDriveBinding, gdrive_object_identity, parse_gdrive_binding +from openkb.connectors.oauth import ( + GDRIVE_REDIRECT_URI, + AuthRejected, + refresh_access_token, + save_refresh_token, + sign_in, +) +from openkb.connectors.store import upsert_binding +from openkb.connectors.sync import recover_auth + +GDRIVE_AUTHORIZE_URL = "https://accounts.google.com/o/oauth2/v2/auth" +GDRIVE_TOKEN_URL = "https://oauth2.googleapis.com/token" +GDRIVE_SCOPE = "https://www.googleapis.com/auth/drive.readonly" +_FOLDER_MIME = "application/vnd.google-apps.folder" +_TRANSIENT_STATUS = {429, 500, 502, 503, 504} + +Announce = Callable[[str], None] +RefreshFn = Callable[[str], tuple[str, str | None]] +SignInFn = Callable[[], object] + + +class DriveClient(Protocol): + """The slice of the Drive API this backend uses.""" + + def list_folder(self, folder_id: str) -> list[dict[str, object]]: + """Return children. Each item has ``id``, ``name``, and ``folder``.""" + + def download(self, file_id: str, dest: Path) -> None: + """Write one file to ``dest``.""" + + +def _load_google(): + try: + from google.oauth2.credentials import Credentials # type: ignore[import-not-found] + from googleapiclient.discovery import build # type: ignore[import-not-found] + from googleapiclient.http import MediaIoBaseDownload # type: ignore[import-not-found] + except ImportError as exc: + raise ProviderError( + 'Google Drive support is not installed. Install it with: pip install "openkb[gdrive]"' + ) from exc + return Credentials, build, MediaIoBaseDownload + + +def _translate(exc: Exception) -> NoReturn: + status = getattr(getattr(exc, "resp", None), "status", None) + if status in _TRANSIENT_STATUS: + raise TransientError(f"Drive transient error (HTTP {status})") from exc + name = type(exc).__name__ + if "Timeout" in name or "Connection" in name: + raise TransientError(f"Drive transient error ({name})") from exc + raise ProviderError(f"Drive error ({name})") from exc + + +class _GoogleClient: + def __init__(self, access_token: str) -> None: + credentials_cls, build, self._download_cls = _load_google() + credentials = credentials_cls(token=access_token) + self._service = build("drive", "v3", credentials=credentials, cache_discovery=False) + + def list_folder(self, folder_id: str) -> list[dict[str, object]]: + found: list[dict[str, object]] = [] + page_token: str | None = None + while True: + try: + response = ( + self._service.files() + .list( + q=f"'{folder_id}' in parents and trashed = false", + fields="nextPageToken, files(id, name, mimeType)", + pageSize=1000, + pageToken=page_token, + supportsAllDrives=False, + ) + .execute() + ) + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _translate(exc) + for item in response.get("files") or []: + name = item.get("name") + file_id = item.get("id") + if not isinstance(name, str) or not isinstance(file_id, str): + continue + found.append( + { + "id": file_id, + "name": name, + "folder": item.get("mimeType") == _FOLDER_MIME, + } + ) + page_token = response.get("nextPageToken") + if not page_token: + return found + + def download(self, file_id: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + request = self._service.files().get_media(fileId=file_id) + with dest.open("wb") as handle: + downloader = self._download_cls(handle, request) + done = False + while not done: + try: + _status, done = downloader.next_chunk() + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _translate(exc) + + +def _walk(client: DriveClient, root_id: str, folder_id: str, relative: str) -> list[RemoteObject]: + found: list[RemoteObject] = [] + for item in client.list_folder(folder_id): + name = item.get("name") + file_id = item.get("id") + if not isinstance(name, str) or not name or not isinstance(file_id, str): + continue + path = name if not relative else f"{relative}/{name}" + if item.get("folder") is True: + found.extend(_walk(client, root_id, file_id, path)) + continue + found.append( + RemoteObject( + identity=gdrive_object_identity(root_id, path), + name=name, + locator=file_id, + ) + ) + return found + + +class GDriveBackend: + """List a folder recursively and download one file.""" + + def __init__( + self, binding: GDriveBinding, access_token: str, *, client: DriveClient | None = None + ): + self._binding = binding + self._client = client if client is not None else _GoogleClient(access_token) + + def list_objects(self) -> list[RemoteObject]: + try: + return _walk(self._client, self._binding.folder_id, self._binding.folder_id, "") + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _translate(exc) + + def download(self, obj: RemoteObject, dest: Path) -> None: + if not obj.locator: + raise ProviderError(f"Drive file id is missing for {obj.identity}.") + try: + self._client.download(obj.locator, dest) + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _translate(exc) + + +def run_gdrive_sign_in(kb_dir: Path, *, announce: Announce | None = None) -> str: + """Browser sign-in. Returns the refresh token and does not write it.""" + client = gdrive_client(kb_dir) + if client is None: + raise ProviderError(_missing_client_message()) + client_id, secret = client + return sign_in( + authorize_url=GDRIVE_AUTHORIZE_URL, + token_url=GDRIVE_TOKEN_URL, + client_id=client_id, + client_secret=secret, + scopes=GDRIVE_SCOPE, + redirect_uri=GDRIVE_REDIRECT_URI, + extra_params={"access_type": "offline", "prompt": "consent"}, + announce=announce, + ) + + +def _missing_client_message() -> str: + return ( + "Google Drive credentials are missing. Set OPENKB_GDRIVE_CLIENT_ID and " + "OPENKB_GDRIVE_CLIENT_SECRET in the environment or /.env." + ) + + +def _default_refresh(kb_dir: Path, refresh_token: str) -> tuple[str, str | None]: + client = gdrive_client(kb_dir) + if client is None: + raise ProviderError(_missing_client_message()) + client_id, secret = client + return refresh_access_token( + token_url=GDRIVE_TOKEN_URL, + client_id=client_id, + client_secret=secret, + refresh_token=refresh_token, + scopes=GDRIVE_SCOPE, + ) + + +def access_token_for( + kb_dir: Path, + *, + non_interactive: bool, + sign_in_hook: SignInFn | None = None, + refresh: RefreshFn | None = None, + announce: Announce | None = None, +) -> str: + """Return an access token, signing in when the refresh token is missing or rejected. + + ``--non-interactive`` never calls the sign-in hook. + """ + if gdrive_client(kb_dir) is None: + raise ProviderError(_missing_client_message()) + refresh_fn = refresh or (lambda token: _default_refresh(kb_dir, token)) + + def browser_sign_in() -> object: + if sign_in_hook is not None: + return sign_in_hook() + return run_gdrive_sign_in(kb_dir, announce=announce) + + def use(token: str) -> str: + last: TransientError | None = None + for _attempt in range(3): + try: + access, rotated = refresh_fn(token) + except TransientError as exc: + last = exc + continue + if rotated and rotated != token: + save_refresh_token(kb_dir, GDRIVE_REFRESH_TOKEN, rotated) + return access + assert last is not None + raise ProviderError(str(last)) from last + + stored = gdrive_refresh_token(kb_dir) + if stored: + try: + return use(stored) + except AuthRejected: + if non_interactive: + raise ProviderError( + "Sign-in is required. Re-run without --non-interactive to open a browser." + ) from None + token = recover_auth(non_interactive=non_interactive, sign_in=browser_sign_in) + if not isinstance(token, str) or not token: + raise ProviderError("Sign-in did not return a refresh token.") + save_refresh_token(kb_dir, GDRIVE_REFRESH_TOKEN, token) + return use(token) + + +def prepare_gdrive_binding( + kb_dir: Path, + *, + name: str, + folder: str, + non_interactive: bool, + announce: Announce | None = None, + sign_in_hook: SignInFn | None = None, + refresh: RefreshFn | None = None, +) -> str: + """Validate the token, then save the binding. A failure writes no binding.""" + binding = parse_gdrive_binding(name=name, folder=folder) + access_token_for( + kb_dir, + non_interactive=non_interactive, + sign_in_hook=sign_in_hook, + refresh=refresh, + announce=announce, + ) + return upsert_binding(kb_dir, binding) diff --git a/openkb/connectors/model.py b/openkb/connectors/model.py new file mode 100644 index 00000000..8e0fd17e --- /dev/null +++ b/openkb/connectors/model.py @@ -0,0 +1,113 @@ +"""Binding records and remote object identities.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}") + + +@dataclass(frozen=True) +class S3Binding: + """One Amazon S3 prefix bound to a knowledge base.""" + + name: str + region: str + bucket: str + prefix: str = "" + + provider: str = "s3" + + def location(self) -> dict[str, str]: + return {"region": self.region, "bucket": self.bucket, "prefix": self.prefix} + + def describe(self) -> str: + prefix = self.prefix or "" + return f"s3://{self.bucket}/{prefix}" + + def page_prefix(self) -> str: + """Registry prefix for pages compiled from this binding.""" + return f"s3://{self.bucket}/{self.prefix}" + + +def s3_object_identity(bucket: str, key: str) -> str: + """Registry identity for one S3 object. A rename is a different key.""" + return f"s3://{bucket}/{key}" + + +def validate_binding_name(name: str) -> str: + cleaned = name.strip() + if not _NAME_RE.fullmatch(cleaned): + raise ValueError( + "Binding name must be 1-64 characters of letters, digits, '_' or '-', " + "and must start with a letter or digit." + ) + return cleaned + + +def parse_s3_binding( + *, + name: str, + region: str, + bucket: str, + prefix: str = "", +) -> S3Binding: + cleaned_name = validate_binding_name(name) + cleaned_region = region.strip() + cleaned_bucket = bucket.strip() + if not cleaned_region: + raise ValueError("S3 region is required.") + if not cleaned_bucket or "/" in cleaned_bucket: + raise ValueError("S3 bucket is required and cannot contain '/'.") + cleaned_prefix = prefix.strip() + if cleaned_prefix.startswith("/"): + raise ValueError("S3 prefix must not start with '/'. An empty prefix is the bucket root.") + return S3Binding( + name=cleaned_name, + region=cleaned_region, + bucket=cleaned_bucket, + prefix=cleaned_prefix, + ) + + +_FOLDER_IN_URL = re.compile(r"(?:/folders/|[?&]id=)([A-Za-z0-9_-]+)") +_FOLDER_ID = re.compile(r"[A-Za-z0-9_-]{10,}") + + +@dataclass(frozen=True) +class GDriveBinding: + """One folder in the user's own Google Drive.""" + + name: str + folder_id: str + provider: str = "gdrive" + + def location(self) -> dict[str, str]: + return {"folder_id": self.folder_id} + + def describe(self) -> str: + return f"gdrive://{self.folder_id}/" + + def page_prefix(self) -> str: + return f"gdrive://{self.folder_id}/" + + +def gdrive_object_identity(folder_id: str, relative_path: str) -> str: + """Registry identity. ``relative_path`` is the path under the bound folder.""" + return f"gdrive://{folder_id}/{relative_path}" + + +def parse_gdrive_folder(folder: str) -> str: + """Accept a folder URL or a raw folder id.""" + text = folder.strip() + found = _FOLDER_IN_URL.search(text) + if found: + return found.group(1) + if _FOLDER_ID.fullmatch(text): + return text + raise ValueError("Drive folder must be a folder URL or a folder id.") + + +def parse_gdrive_binding(*, name: str, folder: str) -> GDriveBinding: + return GDriveBinding(name=validate_binding_name(name), folder_id=parse_gdrive_folder(folder)) diff --git a/openkb/connectors/oauth.py b/openkb/connectors/oauth.py new file mode 100644 index 00000000..671d0482 --- /dev/null +++ b/openkb/connectors/oauth.py @@ -0,0 +1,272 @@ +"""Loopback OAuth sign-in shared by Drive and, later, OneDrive. + +The listener binds ``127.0.0.1`` only. Google's desktop client uses +``http://127.0.0.1:7567/callback``. Microsoft's portal text box accepts +``http://localhost`` rather than an ``http://127.0.0.1`` URI, so OneDrive +sends that redirect string; the browser still reaches this listener because +``localhost`` resolves here. ``--non-interactive`` never calls :func:`sign_in`. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import secrets +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Callable +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any + +import portalocker + +from openkb.connectors.backend import ProviderError, TransientError +from openkb.connectors.state import BUSY_MESSAGE, LOCK_TIMEOUT_SECONDS, KbBusyError +from openkb.envfile import merge_env_file +from openkb.locks import kb_lock + +LOOPBACK_HOST = "127.0.0.1" +LOOPBACK_PORT = 7567 +CALLBACK_PATH = "/callback" +SIGN_IN_TIMEOUT_SECONDS = 300 +GDRIVE_REDIRECT_URI = "http://127.0.0.1:7567/callback" + +_AUTH_LOCK = threading.Lock() +Announce = Callable[[str], None] + + +class AuthRejected(ProviderError): + """The token endpoint rejected the refresh token or the client.""" + + +class _CallbackResult: + def __init__(self) -> None: + self.query: dict[str, list[str]] | None = None + + +class _CallbackServer(HTTPServer): + def __init__(self, address: tuple[str, int], result: _CallbackResult) -> None: + self.result = result + super().__init__(address, _CallbackHandler) + + +class _CallbackHandler(BaseHTTPRequestHandler): + server: _CallbackServer + + def do_GET(self) -> None: # noqa: N802 + parsed = urllib.parse.urlparse(self.path) + if parsed.path != CALLBACK_PATH: + self.send_error(404) + return + self.server.result.query = urllib.parse.parse_qs(parsed.query) + body = b"OpenKB sign-in complete. You can close this window." + self.send_response(200) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt: str, *args: object) -> None: + return + + +def _pkce() -> tuple[str, str]: + verifier = secrets.token_urlsafe(64) + digest = hashlib.sha256(verifier.encode("ascii")).digest() + challenge = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + return verifier, challenge + + +def _post_form(url: str, fields: dict[str, str]) -> dict[str, Any]: + payload = urllib.parse.urlencode(fields).encode("utf-8") + request = urllib.request.Request( + url, + data=payload, + method="POST", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + body = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + if exc.code in {429, 500, 502, 503, 504}: + raise TransientError(f"Token endpoint returned HTTP {exc.code}") from exc + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = {} + if isinstance(parsed, dict): + _raise_oauth_error(parsed, exc.code) + raise ProviderError(f"Token endpoint returned HTTP {exc.code}") from exc + except urllib.error.URLError as exc: + raise TransientError("Token endpoint could not be reached") from exc + if not isinstance(body, dict): + raise ProviderError("Token endpoint returned a response that is not an object.") + _raise_oauth_error(body, 200) + return body + + +def _raise_oauth_error(body: dict[str, Any], status: int) -> None: + error = body.get("error") + if not error: + return + code = str(error) + if code in {"invalid_grant", "invalid_client", "interaction_required", "consent_required"}: + raise AuthRejected(f"Sign-in was rejected ({code}).") + raise ProviderError(f"Token endpoint returned {code} (HTTP {status}).") + + +def refresh_access_token( + *, + token_url: str, + client_id: str, + client_secret: str, + refresh_token: str, + scopes: str, +) -> tuple[str, str | None]: + """Exchange a refresh token. Returns ``(access_token, new_refresh_or_none)``.""" + body = _post_form( + token_url, + { + "grant_type": "refresh_token", + "refresh_token": refresh_token, + "client_id": client_id, + "client_secret": client_secret, + "scope": scopes, + }, + ) + access = body.get("access_token") + if not isinstance(access, str) or not access: + raise ProviderError("Token endpoint did not return an access token.") + rotated = body.get("refresh_token") + new_refresh = rotated if isinstance(rotated, str) and rotated else None + return access, new_refresh + + +def save_refresh_token(kb_dir: Path, key: str, token: str) -> None: + """Write one refresh token into ``/.env`` at mode ``0600``.""" + try: + with kb_lock(kb_dir / ".openkb", exclusive=True, timeout=LOCK_TIMEOUT_SECONDS): + merge_env_file(kb_dir / ".env", {key: token}) + except TimeoutError as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + except portalocker.exceptions.LockException as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + + +def sign_in( + *, + authorize_url: str, + token_url: str, + client_id: str, + client_secret: str, + scopes: str, + redirect_uri: str, + extra_params: dict[str, str] | None = None, + announce: Announce | None = None, + host: str = LOOPBACK_HOST, + port: int = LOOPBACK_PORT, + timeout: float = SIGN_IN_TIMEOUT_SECONDS, +) -> str: + """Open a loopback sign-in and return the refresh token. + + ``announce`` receives the authorize URL before the wait, so a headless + session can print it. The browser wait does not hold the ingest lock. + A second call waits until the first listener is done. + """ + if host != LOOPBACK_HOST: + raise ProviderError("Sign-in listens on 127.0.0.1 only.") + if not _AUTH_LOCK.acquire(blocking=True): # pragma: no cover - blocking acquire waits + raise ProviderError("Another sign-in is already waiting for the browser.") + try: + return _sign_in_locked( + authorize_url=authorize_url, + token_url=token_url, + client_id=client_id, + client_secret=client_secret, + scopes=scopes, + redirect_uri=redirect_uri, + extra_params=extra_params or {}, + announce=announce, + host=host, + port=port, + timeout=timeout, + ) + finally: + _AUTH_LOCK.release() + + +def _sign_in_locked( + *, + authorize_url: str, + token_url: str, + client_id: str, + client_secret: str, + scopes: str, + redirect_uri: str, + extra_params: dict[str, str], + announce: Announce | None, + host: str, + port: int, + timeout: float, +) -> str: + verifier, challenge = _pkce() + state = secrets.token_urlsafe(32) + query = { + "client_id": client_id, + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": scopes, + "state": state, + "code_challenge": challenge, + "code_challenge_method": "S256", + } + query.update(extra_params) + url = authorize_url + "?" + urllib.parse.urlencode(query) + result = _CallbackResult() + try: + server = _CallbackServer((host, port), result) + except OSError as exc: + raise ProviderError( + f"Sign-in port {port} is already in use. Close the other OpenKB sign-in and retry." + ) from exc + server.timeout = 0.25 + if announce is not None: + announce(url) + deadline = time.monotonic() + timeout + try: + while result.query is None and time.monotonic() < deadline: + server.handle_request() + finally: + server.server_close() + if result.query is None: + raise ProviderError("Sign-in timed out.") + returned_state = (result.query.get("state") or [""])[0] + if returned_state != state: + raise ProviderError("Sign-in failed because the returned state did not match.") + if result.query.get("error"): + raise ProviderError(f"Sign-in failed ({result.query['error'][0]}).") + code = (result.query.get("code") or [""])[0] + if not code: + raise ProviderError("Sign-in failed because the provider returned no code.") + body = _post_form( + token_url, + { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": redirect_uri, + "client_id": client_id, + "client_secret": client_secret, + "code_verifier": verifier, + }, + ) + refresh = body.get("refresh_token") + if not isinstance(refresh, str) or not refresh: + raise ProviderError("The provider did not return a refresh token.") + return refresh diff --git a/openkb/connectors/s3.py b/openkb/connectors/s3.py new file mode 100644 index 00000000..a8e546c2 --- /dev/null +++ b/openkb/connectors/s3.py @@ -0,0 +1,109 @@ +"""Amazon S3 list and download. boto3 is imported only when a client is built.""" + +from __future__ import annotations + +from pathlib import Path + +from openkb.connectors.backend import ProviderError, RemoteObject, TransientError +from openkb.connectors.model import S3Binding, s3_object_identity + +_TRANSIENT_STATUS = {429, 500, 502, 503, 504} +_TRANSIENT_CODES = {"SlowDown", "RequestTimeout", "Throttling", "RequestTimeoutException"} + + +def _load_boto3(): + try: + import boto3 # type: ignore[import-not-found] + except ImportError as exc: + raise ProviderError( + 'Amazon S3 support is not installed. Install it with: pip install "openkb[s3]"' + ) from exc + return boto3 + + +def _raise_translated(exc: Exception) -> None: + try: + from botocore.exceptions import BotoCoreError, ClientError # type: ignore[import-not-found] + except ImportError: + raise ProviderError(str(exc)) from exc + if isinstance(exc, ClientError): + error = exc.response.get("Error", {}) if isinstance(exc.response, dict) else {} + meta = exc.response.get("ResponseMetadata", {}) if isinstance(exc.response, dict) else {} + status = meta.get("HTTPStatusCode") + code = str(error.get("Code", "")) + if status in _TRANSIENT_STATUS or code in _TRANSIENT_CODES: + raise TransientError(f"S3 transient error ({code or status})") from exc + raise ProviderError(f"S3 error ({code or status})") from exc + if isinstance(exc, BotoCoreError): + name = type(exc).__name__ + if "Timeout" in name or "Connection" in name: + raise TransientError(f"S3 transient error ({name})") from exc + raise ProviderError(f"S3 error ({name})") from exc + raise exc + + +class S3Backend: + """List and download one bucket prefix with caller-supplied keys.""" + + def __init__( + self, + binding: S3Binding, + access_key: str, + secret_key: str, + *, + client=None, + ) -> None: + self._binding = binding + self._access_key = access_key + self._secret_key = secret_key + self._client = client + + def _s3(self): + if self._client is not None: + return self._client + boto3 = _load_boto3() + return boto3.client( + "s3", + region_name=self._binding.region, + aws_access_key_id=self._access_key, + aws_secret_access_key=self._secret_key, + ) + + def list_objects(self) -> list[RemoteObject]: + client = self._s3() + kwargs: dict[str, str] = {"Bucket": self._binding.bucket} + if self._binding.prefix: + kwargs["Prefix"] = self._binding.prefix + found: list[RemoteObject] = [] + try: + paginator = client.get_paginator("list_objects_v2") + for page in paginator.paginate(**kwargs): + for item in page.get("Contents") or []: + key = item.get("Key") + if not isinstance(key, str) or not key or key.endswith("/"): + continue + name = key.rsplit("/", 1)[-1] + found.append( + RemoteObject( + identity=s3_object_identity(self._binding.bucket, key), + name=name, + ) + ) + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _raise_translated(exc) + return found + + def download(self, obj: RemoteObject, dest: Path) -> None: + prefix = f"s3://{self._binding.bucket}/" + if not obj.identity.startswith(prefix): + raise ProviderError(f"Object {obj.identity} is outside bucket {self._binding.bucket}.") + key = obj.identity[len(prefix) :] + dest.parent.mkdir(parents=True, exist_ok=True) + try: + self._s3().download_file(self._binding.bucket, key, str(dest)) + except Exception as exc: + if isinstance(exc, (ProviderError, TransientError)): + raise + _raise_translated(exc) diff --git a/openkb/connectors/state.py b/openkb/connectors/state.py new file mode 100644 index 00000000..69a39cd7 --- /dev/null +++ b/openkb/connectors/state.py @@ -0,0 +1,117 @@ +"""Derived sync status in ``.openkb/connector-state.json``. + +The file is not the wiki. Writes go through :func:`atomic_write_json` while +the exclusive ingest lock is held, and they do not open a mutation journal +of their own (taking the lock does run the existing journal recovery). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import portalocker + +from openkb.locks import atomic_write_json, kb_lock + +LOCK_TIMEOUT_SECONDS = 30 +BUSY_MESSAGE = ( + "knowledge base is busy (another `openkb add` or sync is running); retry when it finishes" +) + +_COUNT_KEYS = ("added", "skipped", "updated", "failed") + + +class SecretInStateError(ValueError): + """Refused to persist a state document that contains a credential.""" + + +class KbBusyError(TimeoutError): + """The ingest lock was not free within the connector wait.""" + + +def state_path(kb_dir: Path) -> Path: + return kb_dir / ".openkb" / "connector-state.json" + + +def read_state(kb_dir: Path) -> dict[str, Any]: + path = state_path(kb_dir) + if not path.is_file(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + if not isinstance(data, dict): + return {} + return data + + +def is_stale(provider: str, location: dict[str, str], entry: dict[str, Any] | None) -> bool: + """True when the saved result describes a different binding than ``location``.""" + if not entry or not entry.get("last_sync_at"): + return True + return entry.get("provider") != provider or entry.get("location") != location + + +def _reject_secrets(document: dict[str, Any], secrets: list[str]) -> None: + payload = json.dumps(document) + for secret in secrets: + if secret and secret in payload: + raise SecretInStateError("refusing to write connector state that contains a credential") + + +def update_state( + kb_dir: Path, + name: str, + entry: dict[str, Any] | None, + *, + secrets: list[str] | None = None, + timeout: float | None = None, +) -> None: + """Insert, replace, or delete one binding's status under the ingest lock. + + ``entry is None`` deletes the key. ``timeout`` defaults to 30 seconds. + """ + + def mutate(data: dict[str, Any]) -> None: + if entry is None: + data.pop(name, None) + else: + data[name] = entry + + _mutate(kb_dir, mutate, secrets=secrets or [], timeout=timeout) + + +def _mutate( + kb_dir: Path, + mutate, + *, + secrets: list[str], + timeout: float | None, +) -> None: + wait = LOCK_TIMEOUT_SECONDS if timeout is None else timeout + try: + with kb_lock(kb_dir / ".openkb", exclusive=True, timeout=wait): + data = read_state(kb_dir) + mutate(data) + _reject_secrets(data, secrets) + atomic_write_json(state_path(kb_dir), data) + except TimeoutError as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + except portalocker.exceptions.LockException as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + + +def fresh_entry(provider: str, location: dict[str, str]) -> dict[str, Any]: + """A status row with zero counts, used when a sync produced no file results.""" + entry: dict[str, Any] = { + "provider": provider, + "location": location, + "last_sync_at": None, + "error": None, + } + for key in _COUNT_KEYS: + entry[key] = 0 + return entry diff --git a/openkb/connectors/store.py b/openkb/connectors/store.py new file mode 100644 index 00000000..f8a80371 --- /dev/null +++ b/openkb/connectors/store.py @@ -0,0 +1,164 @@ +"""The ``connectors`` list in ``.openkb/config.yaml``. + +This is desired state. Sync status lives in :mod:`openkb.connectors.state`. +Config and state are updated in one ingest-lock hold so a replace cannot +leave the previous location's counts on the new binding. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import portalocker +import yaml + +from openkb.config import save_config +from openkb.connectors.model import ( + GDriveBinding, + S3Binding, + parse_gdrive_binding, + parse_s3_binding, +) +from openkb.connectors.state import ( + BUSY_MESSAGE, + LOCK_TIMEOUT_SECONDS, + KbBusyError, + read_state, + state_path, +) +from openkb.locks import atomic_write_json, kb_lock +from openkb.state import HashRegistry + + +def config_path(kb_dir: Path) -> Path: + return kb_dir / ".openkb" / "config.yaml" + + +def _read_raw(path: Path) -> dict[str, Any]: + if not path.is_file(): + return {} + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + raise ValueError("config.yaml must be a mapping.") + return data + + +Binding = S3Binding | GDriveBinding + + +def _parse_item(item: object) -> Binding | None: + if not isinstance(item, dict): + return None + provider = item.get("provider") + try: + if provider == "s3": + return parse_s3_binding( + name=str(item.get("name", "")), + region=str(item.get("region", "")), + bucket=str(item.get("bucket", "")), + prefix=str(item.get("prefix", "") or ""), + ) + if provider == "gdrive": + return parse_gdrive_binding( + name=str(item.get("name", "")), + folder=str(item.get("folder_id", "") or ""), + ) + except ValueError: + return None + return None + + +def load_bindings(kb_dir: Path) -> list[Binding]: + raw = _read_raw(config_path(kb_dir)) + items = raw.get("connectors") or [] + if not isinstance(items, list): + return [] + bindings: list[Binding] = [] + for item in items: + parsed = _parse_item(item) + if parsed is not None: + bindings.append(parsed) + return bindings + + +def _count_remote_pages(kb_dir: Path, binding: Binding) -> int: + registry = HashRegistry(kb_dir / ".openkb" / "hashes.json") + needle = binding.page_prefix() + count = 0 + for meta in registry.all_entries().values(): + path = meta.get("path") + if isinstance(path, str) and path.startswith(needle): + count += 1 + return count + + +def upsert_binding(kb_dir: Path, binding: Binding) -> str: + """Insert or replace ``binding``. Returns ``created`` or ``replaced``. + + A replaced binding's sync status is cleared. Compiled pages stay. + """ + message = "created" + try: + with kb_lock(kb_dir / ".openkb", exclusive=True, timeout=LOCK_TIMEOUT_SECONDS): + path = config_path(kb_dir) + raw = _read_raw(path) + items = raw.get("connectors") + if not isinstance(items, list): + items = [] + kept: list[object] = [] + previous: Binding | None = None + for item in items: + parsed = _parse_item(item) + if parsed is not None and parsed.name == binding.name: + previous = parsed + continue + if isinstance(item, dict) and item.get("name") == binding.name: + previous = parsed + continue + kept.append(item) + kept.append({"name": binding.name, "provider": binding.provider, **binding.location()}) + raw["connectors"] = kept + save_config(path, raw) + if previous is not None: + data = read_state(kb_dir) + data.pop(binding.name, None) + atomic_write_json(state_path(kb_dir), data) + count = _count_remote_pages(kb_dir, previous) + message = ( + f"Binding `{binding.name}` now points to {binding.describe()}. " + f"{count} documents compiled from the previous location remain in the wiki." + ) + else: + message = f"Binding `{binding.name}` saved ({binding.describe()})." + except TimeoutError as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + except portalocker.exceptions.LockException as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + return message + + +def remove_binding(kb_dir: Path, name: str) -> str: + """Delete the binding record and its status. Wiki pages and ``raw/`` stay.""" + try: + with kb_lock(kb_dir / ".openkb", exclusive=True, timeout=LOCK_TIMEOUT_SECONDS): + path = config_path(kb_dir) + raw = _read_raw(path) + items = raw.get("connectors") + if not isinstance(items, list): + items = [] + kept = [ + item for item in items if not (isinstance(item, dict) and item.get("name") == name) + ] + if len(kept) == len(items): + raise KeyError(name) + raw["connectors"] = kept + save_config(path, raw) + data = read_state(kb_dir) + data.pop(name, None) + atomic_write_json(state_path(kb_dir), data) + except TimeoutError as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + except portalocker.exceptions.LockException as exc: + raise KbBusyError(BUSY_MESSAGE) from exc + return f"Removed binding `{name}`. Documents compiled from it remain in the wiki." diff --git a/openkb/connectors/sync.py b/openkb/connectors/sync.py new file mode 100644 index 00000000..429ceac6 --- /dev/null +++ b/openkb/connectors/sync.py @@ -0,0 +1,387 @@ +"""Sync saved bindings into the knowledge base through ``add_single_file``.""" + +from __future__ import annotations + +import tempfile +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +from openkb.connectors.backend import ObjectBackend, ProviderError, RemoteObject, TransientError +from openkb.connectors.credentials import resolved_secrets, s3_credentials +from openkb.connectors.model import S3Binding +from openkb.connectors.s3 import S3Backend +from openkb.connectors.state import ( + SecretInStateError, + fresh_entry, + is_stale, + read_state, + update_state, +) +from openkb.connectors.store import Binding, load_bindings +from openkb.state import HashRegistry + +RETRY_ATTEMPTS = 3 + +AddFn = Callable[..., str] +SignInFn = Callable[[], object] + + +@dataclass +class BindingReport: + name: str + provider: str + ok: bool + added: int = 0 + skipped: int = 0 + updated: int = 0 + failed: int = 0 + error: str | None = None + stale: bool = False + + +@dataclass +class SyncOutcome: + reports: list[BindingReport] = field(default_factory=list) + + @property + def exit_code(self) -> int: + for report in self.reports: + if not report.ok or report.failed: + return 1 + return 0 + + +def recover_auth(*, non_interactive: bool, sign_in: SignInFn | None) -> object: + """Run loopback sign-in, or fail the provider when that is not allowed. + + S3 does not call this. Drive does, when a refresh token is missing or + rejected. ``--non-interactive`` never invokes ``sign_in``. + """ + if non_interactive or sign_in is None: + raise ProviderError( + "Sign-in is required. Re-run without --non-interactive to open a browser." + ) + return sign_in() + + +def _extensions() -> set[str]: + from openkb.cli import SUPPORTED_EXTENSIONS + + return SUPPORTED_EXTENSIONS + + +def _add_file(path: Path, kb_dir: Path, identity: str) -> str: + from openkb.cli import add_single_file + from openkb.locks import kb_ingest_lock_held + + if kb_ingest_lock_held(kb_dir / ".openkb"): + raise RuntimeError("sync called add_single_file while holding the ingest lock") + return add_single_file(path, kb_dir, identity=identity) + + +def _retry(action: Callable[[], object]) -> object: + last: TransientError | None = None + for _attempt in range(RETRY_ATTEMPTS): + try: + return action() + except TransientError as exc: + last = exc + assert last is not None + raise last + + +def _record( + kb_dir: Path, + binding: Binding, + *, + added: int, + skipped: int, + updated: int, + failed: int, + error: str | None, + synced: bool, +) -> None: + entry = fresh_entry(binding.provider, binding.location()) + entry["added"] = added + entry["skipped"] = skipped + entry["updated"] = updated + entry["failed"] = failed + entry["error"] = error + if synced: + entry["last_sync_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + secrets = resolved_secrets(kb_dir) + try: + update_state(kb_dir, binding.name, entry, secrets=secrets) + except SecretInStateError: + entry["error"] = "sync failed" + update_state(kb_dir, binding.name, entry, secrets=secrets) + + +def _sync_one( + kb_dir: Path, + binding: Binding, + backend: ObjectBackend, + add: AddFn, +) -> BindingReport: + added = skipped = updated = failed = 0 + error: str | None = None + try: + objects = _retry(backend.list_objects) + except (TransientError, ProviderError) as exc: + _record( + kb_dir, + binding, + added=0, + skipped=0, + updated=0, + failed=0, + error=str(exc), + synced=False, + ) + return BindingReport( + name=binding.name, + provider=binding.provider, + ok=False, + error=str(exc), + ) + assert isinstance(objects, list) + registry = HashRegistry(kb_dir / ".openkb" / "hashes.json") + allowed = _extensions() + for obj in objects: + if not isinstance(obj, RemoteObject): + continue + suffix = Path(obj.name).suffix.lower() + if suffix not in allowed: + skipped += 1 + continue + prior = registry.get_by_path(obj.identity) + try: + with tempfile.TemporaryDirectory(prefix="openkb-src-") as tmp: + dest = Path(tmp) / (Path(obj.name).name or "object") + + def download(obj: RemoteObject = obj, dest: Path = dest) -> None: + backend.download(obj, dest) + + _retry(download) + status = add(dest, kb_dir, identity=obj.identity) + except TransientError as exc: + failed += 1 + error = str(exc) + continue + except Exception as exc: + failed += 1 + error = str(exc) + continue + if status == "skipped": + skipped += 1 + elif status == "failed": + failed += 1 + error = f"Failed to compile {obj.name}." + elif prior is not None: + updated += 1 + else: + added += 1 + ok = failed == 0 + _record( + kb_dir, + binding, + added=added, + skipped=skipped, + updated=updated, + failed=failed, + error=error, + synced=True, + ) + return BindingReport( + name=binding.name, + provider=binding.provider, + ok=ok, + added=added, + skipped=skipped, + updated=updated, + failed=failed, + error=error, + ) + + +def _fail_group(kb_dir: Path, bindings: list[Binding], message: str) -> list[BindingReport]: + reports: list[BindingReport] = [] + for binding in bindings: + _record( + kb_dir, + binding, + added=0, + skipped=0, + updated=0, + failed=0, + error=message, + synced=False, + ) + reports.append( + BindingReport( + name=binding.name, + provider=binding.provider, + ok=False, + error=message, + ) + ) + return reports + + +def _sync_s3_group( + kb_dir: Path, + bindings: list[S3Binding], + *, + add: AddFn | None, + backend_for: Callable[[S3Binding, str, str], ObjectBackend] | None, +) -> list[BindingReport]: + creds = s3_credentials(kb_dir) + if creds is None: + return _fail_group( + kb_dir, + list(bindings), + "S3 credentials are missing. Set OPENKB_S3_ACCESS_KEY_ID and " + "OPENKB_S3_SECRET_ACCESS_KEY in the environment or /.env.", + ) + access, secret = creds + add_fn = add or _add_file + reports: list[BindingReport] = [] + for binding in bindings: + backend = ( + backend_for(binding, access, secret) + if backend_for is not None + else S3Backend(binding, access, secret) + ) + reports.append(_sync_one(kb_dir, binding, backend, add_fn)) + return reports + + +def _sync_gdrive_group( + kb_dir: Path, + bindings: list[Binding], + *, + non_interactive: bool, + sign_in: SignInFn | None, + announce: Callable[[str], None] | None, + add: AddFn | None, + backend_for: Callable[[Binding, str], ObjectBackend] | None, +) -> list[BindingReport]: + from openkb.connectors.gdrive import GDriveBackend, access_token_for + from openkb.connectors.model import GDriveBinding + + try: + token = access_token_for( + kb_dir, + non_interactive=non_interactive, + sign_in_hook=sign_in if sign_in is not None else None, + announce=announce, + ) + except ProviderError as exc: + return _fail_group(kb_dir, bindings, str(exc)) + add_fn = add or _add_file + reports: list[BindingReport] = [] + for binding in bindings: + if not isinstance(binding, GDriveBinding): + continue + backend = ( + backend_for(binding, token) + if backend_for is not None + else GDriveBackend(binding, token) + ) + reports.append(_sync_one(kb_dir, binding, backend, add_fn)) + return reports + + +def _grouped(bindings: list[Binding]) -> list[tuple[str, list[Binding]]]: + order: list[str] = [] + groups: dict[str, list[Binding]] = {} + for binding in bindings: + if binding.provider not in groups: + order.append(binding.provider) + groups[binding.provider] = [] + groups[binding.provider].append(binding) + return [(provider, groups[provider]) for provider in order] + + +def sync_bindings( + kb_dir: Path, + *, + name: str | None = None, + non_interactive: bool = False, + sign_in: SignInFn | None = None, + announce: Callable[[str], None] | None = None, + add: AddFn | None = None, + backend_for: Callable[[S3Binding, str, str], ObjectBackend] | None = None, + gdrive_backend_for: Callable[[Binding, str], ObjectBackend] | None = None, +) -> SyncOutcome: + """Sync every binding, or the one named ``name``. + + ``sign_in`` and ``announce`` are for Drive. ``--non-interactive`` drops + both, so a missing or rejected token fails that provider and no browser + opens. ``add`` and the backend hooks exist so tests skip the network and + the compiler. S3 never calls ``sign_in``. + """ + if non_interactive: + sign_in = None + announce = None + bindings = load_bindings(kb_dir) + if name is not None: + bindings = [item for item in bindings if item.name == name] + if not bindings: + return SyncOutcome( + reports=[ + BindingReport( + name=name, + provider="", + ok=False, + error=f"No binding named {name!r}.", + ) + ] + ) + reports: list[BindingReport] = [] + for provider, group in _grouped(bindings): + if provider == "s3": + s3_group = [item for item in group if isinstance(item, S3Binding)] + reports.extend(_sync_s3_group(kb_dir, s3_group, add=add, backend_for=backend_for)) + elif provider == "gdrive": + reports.extend( + _sync_gdrive_group( + kb_dir, + group, + non_interactive=non_interactive, + sign_in=sign_in, + announce=announce, + add=add, + backend_for=gdrive_backend_for, + ) + ) + else: + reports.extend(_fail_group(kb_dir, group, f"Provider {provider} is not available.")) + return SyncOutcome(reports=reports) + + +def binding_views(kb_dir: Path) -> list[dict[str, object]]: + """Bindings plus the last sync row, for the CLI and the Workbench.""" + state = read_state(kb_dir) + views: list[dict[str, object]] = [] + for binding in load_bindings(kb_dir): + entry = state.get(binding.name) + row = entry if isinstance(entry, dict) else None + views.append( + { + "name": binding.name, + "provider": binding.provider, + "location": binding.location(), + "description": binding.describe(), + "stale": is_stale(binding.provider, binding.location(), row), + "last_sync_at": None if row is None else row.get("last_sync_at"), + "added": 0 if row is None else int(row.get("added") or 0), + "skipped": 0 if row is None else int(row.get("skipped") or 0), + "updated": 0 if row is None else int(row.get("updated") or 0), + "failed": 0 if row is None else int(row.get("failed") or 0), + "error": None if row is None else row.get("error"), + } + ) + return views diff --git a/openkb/converter.py b/openkb/converter.py index 08929e01..4e0b9eb0 100644 --- a/openkb/converter.py +++ b/openkb/converter.py @@ -77,6 +77,7 @@ def resolve_doc_name( registry: HashRegistry, *, persist_legacy: bool = True, + path_key: str | None = None, ) -> str: """Resolve the stable wiki name for ``src`` (Scheme A). @@ -87,7 +88,13 @@ def resolve_doc_name( sanitized stem unless another document already owns that name, in which case it gets a deterministic ``-{sha256(path)[:8]}`` suffix. """ - path_key = _registry_path(src, kb_dir) + explicit_identity = path_key is not None + if path_key is None: + path_key = _registry_path(src, kb_dir) + if explicit_identity: + # Remote objects have no filesystem path. Skip the legacy stem backfill + # so a cloud file cannot attach itself to an unrelated local document. + return resolve_doc_name_from_key(src.stem, path_key, registry) known = registry.get_by_path(path_key) if known is not None: @@ -143,6 +150,7 @@ def convert_document( kb_dir: Path, *, staging_dir: Path | None = None, + identity: str | None = None, ) -> ConvertResult: """Convert a document and integrate it into the knowledge base. @@ -181,6 +189,7 @@ def convert_document( kb_dir, registry, persist_legacy=staging_dir is None, + path_key=identity, ) # ------------------------------------------------------------------ diff --git a/openkb/envfile.py b/openkb/envfile.py new file mode 100644 index 00000000..152c18e5 --- /dev/null +++ b/openkb/envfile.py @@ -0,0 +1,46 @@ +"""`.env` merge used by credential writers. + +One implementation of the mode ``0600`` write. :mod:`openkb.api_config` calls +it for the REST config patch. Connector sign-in calls it too, so the CLI does +not import FastAPI. +""" + +from __future__ import annotations + +from pathlib import Path + +from openkb.locks import atomic_write_text + + +def merge_env_file(env_path: Path, updates: dict[str, str | None]) -> None: + """Merge ``updates`` into ``env_path`` and leave the file at mode ``0600``. + + A ``None`` value removes the key. A leading ``export `` is stripped so the + key matches what python-dotenv reads back. Callers hold the KB lock: this + is a read-modify-write. Credential values are never logged. + + The file is tightened to ``0o600`` before the replace. ``atomic_write_text`` + copies the target's current mode onto its temp file, so the renamed result + is already ``0600``. + """ + env_lines: dict[str, str] = {} + if env_path.exists(): + for raw in env_path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export ") or line.startswith("export\t"): + line = line[len("export") :].lstrip() + if "=" not in line: + continue + key, _eq, stored = line.partition("=") + env_lines[key.strip()] = stored + for key, value in updates.items(): + if value is None: + env_lines.pop(key, None) + else: + env_lines[key] = value + env_path.parent.mkdir(parents=True, exist_ok=True) + env_path.touch(mode=0o600, exist_ok=True) + env_path.chmod(0o600) + atomic_write_text(env_path, "".join(f"{k}={v}\n" for k, v in env_lines.items())) diff --git a/openkb/locks.py b/openkb/locks.py index a085da75..37f40e53 100644 --- a/openkb/locks.py +++ b/openkb/locks.py @@ -14,13 +14,14 @@ import os import tempfile import threading +import time from pathlib import Path from typing import IO, Iterator import portalocker -def flock(fh: IO, *, exclusive: bool) -> None: +def flock(fh: IO, *, exclusive: bool, timeout: float | None = None) -> None: """Acquire an advisory lock on an open file handle (cross-platform). Delegates to :mod:`portalocker`: @@ -31,10 +32,29 @@ def flock(fh: IO, *, exclusive: bool) -> None: readers are honoured; exclusive locks use ``msvcrt.locking``, which retries for ~10s and then raises rather than blocking indefinitely. - On failure portalocker raises :class:`portalocker.LockException` — note this - is *not* an ``OSError`` (e.g. on filesystems without working lock support). + ``timeout`` bounds the wait. ``None`` keeps the historical blocking + acquire used by ``add``. On failure portalocker raises + :class:`portalocker.LockException` — note this is *not* an ``OSError`` + (e.g. on filesystems without working lock support, or a timed-out wait). """ - portalocker.lock(fh, portalocker.LOCK_EX if exclusive else portalocker.LOCK_SH) + flags = portalocker.LOCK_EX if exclusive else portalocker.LOCK_SH + if timeout is None: + portalocker.lock(fh, flags) + return + # portalocker 3.2's lock() has no timeout argument. Poll the non-blocking + # flag until the deadline, then raise the same LockException a contended + # non-blocking acquire raises. + deadline = time.monotonic() + timeout + nonblocking = flags | portalocker.LOCK_NB + while True: + try: + portalocker.lock(fh, nonblocking) + return + except portalocker.LockException: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise + time.sleep(min(0.05, remaining)) def funlock(fh: IO) -> None: @@ -68,10 +88,16 @@ def read(self) -> Iterator[None]: self._condition.notify_all() @contextlib.contextmanager - def write(self) -> Iterator[None]: + def write(self, timeout: float | None = None) -> Iterator[None]: + deadline = None if timeout is None else time.monotonic() + timeout with self._condition: while self._writer or self._readers: - self._condition.wait() + if deadline is None: + self._condition.wait() + continue + remaining = deadline - time.monotonic() + if remaining <= 0 or not self._condition.wait(timeout=remaining): + raise TimeoutError("timed out waiting for the knowledge base lock") self._writer = True try: yield @@ -126,8 +152,13 @@ def _drain_pending_journals(openkb_dir: Path) -> None: @contextlib.contextmanager -def kb_lock(openkb_dir: Path, *, exclusive: bool) -> Iterator[None]: - """Hold a KB-level advisory lock.""" +def kb_lock(openkb_dir: Path, *, exclusive: bool, timeout: float | None = None) -> Iterator[None]: + """Hold a KB-level advisory lock. + + ``timeout`` applies only to a fresh exclusive acquire. ``None`` blocks, which + is what ``add`` uses. A thread that already holds the lock reenters without + waiting. + """ lock_path = openkb_dir / "ingest.lock" lock_path.parent.mkdir(parents=True, exist_ok=True) resolved = lock_path.resolve() @@ -156,10 +187,10 @@ def kb_lock(openkb_dir: Path, *, exclusive: bool) -> Iterator[None]: return local_lock = _local_lock(lock_path) - local_context = local_lock.write() if exclusive else local_lock.read() + local_context = local_lock.write(timeout=timeout) if exclusive else local_lock.read() with local_context: with lock_path.open("a+", encoding="utf-8") as fh: - flock(fh, exclusive=exclusive) + flock(fh, exclusive=exclusive, timeout=timeout) held[resolved] = (1, 0) if exclusive else (0, 1) try: if exclusive: diff --git a/openkb/source_cmd.py b/openkb/source_cmd.py new file mode 100644 index 00000000..584634a7 --- /dev/null +++ b/openkb/source_cmd.py @@ -0,0 +1,146 @@ +"""``openkb source`` — bind a remote prefix and sync it into the knowledge base.""" + +from __future__ import annotations + +from pathlib import Path + +import click + +from openkb.connectors.backend import ProviderError +from openkb.connectors.credentials import s3_credentials +from openkb.connectors.gdrive import prepare_gdrive_binding +from openkb.connectors.model import parse_s3_binding +from openkb.connectors.state import KbBusyError +from openkb.connectors.store import remove_binding, upsert_binding +from openkb.connectors.sync import binding_views, sync_bindings + + +def _kb_dir(ctx: click.Context) -> Path: + from openkb.cli import _find_kb_dir + + root = ctx.find_root() + override = None + if isinstance(root.obj, dict): + override = root.obj.get("kb_dir_override") + found = _find_kb_dir(override) + if found is None: + raise click.ClickException("No knowledge base found. Run `openkb init` or pass --kb-dir.") + return found + + +@click.group() +def source() -> None: + """Bind a remote data source and sync it into this knowledge base.""" + + +def _announce(url: str) -> None: + """Print the sign-in URL and open a browser. A headless session still waits.""" + click.echo(f"Open this URL to sign in:\n{url}") + import webbrowser + + try: + webbrowser.open(url) + except Exception: + return + + +@source.command("add") +@click.argument("provider", type=click.Choice(["s3", "gdrive"])) +@click.option("--name", required=True, help="Local name for this binding.") +@click.option("--region", default="", help="AWS region, for example us-east-1.") +@click.option("--bucket", default="", help="S3 bucket name.") +@click.option("--prefix", default="", help="Key prefix. Empty means the bucket root.") +@click.option("--folder", default="", help="Google Drive folder URL or folder id.") +@click.option( + "--non-interactive", + is_flag=True, + default=False, + help="Never open a browser. Drive then requires a refresh token already on disk.", +) +@click.pass_context +def add_cmd(ctx, provider, name, region, bucket, prefix, folder, non_interactive) -> None: + """Save a binding. Credentials must already be available.""" + kb_dir = _kb_dir(ctx) + try: + if provider == "s3": + if s3_credentials(kb_dir) is None: + raise click.ClickException( + "S3 credentials are missing. Set OPENKB_S3_ACCESS_KEY_ID and " + "OPENKB_S3_SECRET_ACCESS_KEY in the environment or /.env." + ) + binding = parse_s3_binding(name=name, region=region, bucket=bucket, prefix=prefix) + message = upsert_binding(kb_dir, binding) + else: + message = prepare_gdrive_binding( + kb_dir, + name=name, + folder=folder, + non_interactive=non_interactive, + announce=None if non_interactive else _announce, + ) + except (ValueError, KbBusyError, ProviderError) as exc: + raise click.ClickException(str(exc)) from exc + click.echo(message) + + +@source.command("list") +@click.pass_context +def list_cmd(ctx) -> None: + """List bindings and the last sync status.""" + kb_dir = _kb_dir(ctx) + views = binding_views(kb_dir) + if not views: + click.echo("No remote bindings.") + return + for view in views: + status = "not synced since this change" if view["stale"] else "synced" + click.echo(f"{view['name']}\t{view['provider']}\t{view['description']}\t{status}") + + +@source.command("remove") +@click.argument("name") +@click.pass_context +def remove_cmd(ctx, name) -> None: + """Remove a binding. Compiled documents stay in the wiki.""" + kb_dir = _kb_dir(ctx) + try: + message = remove_binding(kb_dir, name) + except KeyError as exc: + raise click.ClickException(f"No binding named {name!r}.") from exc + except KbBusyError as exc: + raise click.ClickException(str(exc)) from exc + click.echo(message) + + +@source.command("sync") +@click.argument("name", required=False) +@click.option( + "--non-interactive", + is_flag=True, + default=False, + help="Never open a browser. A missing credential fails that provider.", +) +@click.pass_context +def sync_cmd(ctx, name, non_interactive) -> None: + """Download new and changed objects and compile them.""" + kb_dir = _kb_dir(ctx) + try: + outcome = sync_bindings( + kb_dir, + name=name, + non_interactive=non_interactive, + announce=None if non_interactive else _announce, + ) + except KbBusyError as exc: + raise click.ClickException(str(exc)) from exc + for report in outcome.reports: + if report.error and not report.ok and report.added == 0 and report.failed == 0: + click.echo(f"{report.name}: {report.error}", err=True) + continue + click.echo( + f"{report.name}: added {report.added}, skipped {report.skipped}, " + f"updated {report.updated}, failed {report.failed}" + ) + if report.error: + click.echo(f" {report.error}", err=True) + ctx.exit(outcome.exit_code) diff --git a/pyproject.toml b/pyproject.toml index 389a8b2b..7cee2e73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,11 @@ dev = [ # `pip install "openkb[api]"` keeps working. web = ["fastapi", "uvicorn", "python-multipart"] api = ["openkb[web]"] +# Cloud SDKs stay out of the core install. Tests mock the client and do not +# import boto3, so CI's `uv sync --locked --extra dev --extra api` is unchanged. +s3 = ["boto3==1.43.124"] +# Tests mock the Drive client and do not import it, so CI stays on dev+api. +gdrive = ["google-api-python-client==2.201.0", "google-auth==2.62.1"] [tool.hatch.version] source = "vcs" diff --git a/tests/test_api_connectors.py b/tests/test_api_connectors.py new file mode 100644 index 00000000..80193be2 --- /dev/null +++ b/tests/test_api_connectors.py @@ -0,0 +1,94 @@ +"""REST connectors call the same functions as the CLI, off the event loop.""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from openkb.api import create_app +from openkb.connectors.sync import BindingReport, SyncOutcome + + +def _client(monkeypatch, token: str | None = "secret") -> TestClient: + if token is None: + monkeypatch.delenv("OPENKB_API_TOKEN", raising=False) + else: + monkeypatch.setenv("OPENKB_API_TOKEN", token) + return TestClient(create_app()) + + +def _auth(token: str = "secret") -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +def _kb(tmp_path): + kb = tmp_path / "kb" + (kb / ".openkb").mkdir(parents=True) + (kb / "wiki").mkdir() + (kb / ".env").write_text( + "OPENKB_S3_ACCESS_KEY_ID=AKIA_TEST\nOPENKB_S3_SECRET_ACCESS_KEY=secret-test\n", + encoding="utf-8", + ) + return kb + + +def _use_kb(monkeypatch, kb_dir, name: str = "test-kb") -> str: + def resolve(kb): + assert kb == name + return kb_dir + + monkeypatch.setattr("openkb.api_helpers.resolve_kb_alias", resolve) + return name + + +def test_connector_round_trip(tmp_path, monkeypatch): + kb = _kb(tmp_path) + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + name = _use_kb(monkeypatch, kb) + client = _client(monkeypatch) + created = client.post( + "/api/v1/connectors", + headers=_auth(), + json={ + "kb": name, + "name": "papers", + "provider": "s3", + "region": "us-east-1", + "bucket": "acme", + "prefix": "docs/", + }, + ) + assert created.status_code == 200, created.text + listed = client.get("/api/v1/connectors", headers=_auth(), params={"kb": name}) + assert listed.status_code == 200 + body = listed.json()["connectors"] + assert body[0]["name"] == "papers" + assert body[0]["location"]["bucket"] == "acme" + assert body[0]["stale"] is True + + def fake_sync(kb_dir, *, name=None, non_interactive=False, **kwargs): + assert non_interactive is True + return SyncOutcome( + reports=[ + BindingReport(name="papers", provider="s3", ok=True, added=1), + ] + ) + + monkeypatch.setattr("openkb.api_connectors_router.sync_bindings", fake_sync) + synced = client.post( + "/api/v1/connectors/sync", + headers=_auth(), + json={"kb": name, "name": "papers"}, + ) + assert synced.status_code == 200 + assert synced.json()["exit_code"] == 0 + removed = client.post( + "/api/v1/connectors/remove", + headers=_auth(), + json={"kb": name, "name": "papers"}, + ) + assert removed.status_code == 200 + assert "remain" in removed.json()["message"] + page = kb / "wiki" / "keep.md" + page.write_text("stay", encoding="utf-8") + assert page.read_text(encoding="utf-8") == "stay" diff --git a/tests/test_connectors.py b/tests/test_connectors.py new file mode 100644 index 00000000..c3fb1298 --- /dev/null +++ b/tests/test_connectors.py @@ -0,0 +1,299 @@ +"""Connector store, credentials, sync, and the S3 backend. No live AWS calls.""" + +from __future__ import annotations + +import threading +from pathlib import Path + +import pytest + +from openkb.connectors.backend import ProviderError, RemoteObject, TransientError +from openkb.connectors.credentials import resolve_connector_credential +from openkb.connectors.model import S3Binding, parse_s3_binding, s3_object_identity +from openkb.connectors.s3 import S3Backend, _load_boto3 +from openkb.connectors.state import ( + LOCK_TIMEOUT_SECONDS, + KbBusyError, + SecretInStateError, + read_state, + update_state, +) +from openkb.connectors.store import load_bindings, remove_binding, upsert_binding +from openkb.connectors.sync import recover_auth, sync_bindings +from openkb.locks import kb_ingest_lock, kb_ingest_lock_held +from openkb.state import HashRegistry + + +def _kb(tmp_path: Path) -> Path: + kb = tmp_path / "kb" + (kb / ".openkb").mkdir(parents=True) + (kb / "wiki").mkdir() + (kb / "raw").mkdir() + (kb / "wiki" / "summaries").mkdir() + return kb + + +def _binding(**kwargs) -> S3Binding: + values = {"name": "papers", "region": "us-east-1", "bucket": "acme", "prefix": "docs/"} + values.update(kwargs) + return parse_s3_binding(**values) + + +def _creds(kb: Path) -> None: + (kb / ".env").write_text( + "OPENKB_S3_ACCESS_KEY_ID=AKIA_TEST\nOPENKB_S3_SECRET_ACCESS_KEY=secret-test\n", + encoding="utf-8", + ) + + +class _Backend: + def __init__(self, objects: list[RemoteObject], *, download_failures: int = 0) -> None: + self.objects = objects + self.download_failures = download_failures + self.downloads = 0 + + def list_objects(self) -> list[RemoteObject]: + return list(self.objects) + + def download(self, obj: RemoteObject, dest: Path) -> None: + self.downloads += 1 + if self.downloads <= self.download_failures: + raise TransientError("slow down") + dest.write_bytes(b"bytes:" + obj.identity.encode()) + + +def test_credential_order_is_process_then_kb_then_global(tmp_path, monkeypatch): + kb = _kb(tmp_path) + global_dir = tmp_path / "global" + global_dir.mkdir() + monkeypatch.setattr("openkb.connectors.credentials.GLOBAL_CONFIG_DIR", global_dir) + (global_dir / ".env").write_text("OPENKB_S3_ACCESS_KEY_ID=from-global\n", encoding="utf-8") + (kb / ".env").write_text("OPENKB_S3_ACCESS_KEY_ID=from-kb\n", encoding="utf-8") + monkeypatch.setenv("OPENKB_S3_ACCESS_KEY_ID", "from-env") + + assert resolve_connector_credential(kb, "OPENKB_S3_ACCESS_KEY_ID") == "from-env" + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID") + assert resolve_connector_credential(kb, "OPENKB_S3_ACCESS_KEY_ID") == "from-kb" + (kb / ".env").write_text("", encoding="utf-8") + assert resolve_connector_credential(kb, "OPENKB_S3_ACCESS_KEY_ID") == "from-global" + + +def test_replace_clears_state_and_remove_keeps_wiki(tmp_path): + kb = _kb(tmp_path) + message = upsert_binding(kb, _binding()) + assert message.startswith("Binding `papers` saved") + update_state( + kb, + "papers", + { + "provider": "s3", + "location": _binding().location(), + "last_sync_at": "2026-11-05T00:00:00Z", + "added": 2, + "skipped": 0, + "updated": 0, + "failed": 0, + "error": None, + }, + ) + page = kb / "wiki" / "summaries" / "kept.md" + raw = kb / "raw" / "kept.pdf" + page.write_text("stay", encoding="utf-8") + raw.write_bytes(b"stay") + + replaced = upsert_binding(kb, _binding(bucket="other")) + assert "remain in the wiki" in replaced + assert "papers" not in read_state(kb) + + removed = remove_binding(kb, "papers") + assert "remain in the wiki" in removed + assert load_bindings(kb) == [] + assert page.read_text(encoding="utf-8") == "stay" + assert raw.read_bytes() == b"stay" + + +def test_state_refuses_a_credential(tmp_path): + kb = _kb(tmp_path) + with pytest.raises(SecretInStateError): + update_state( + kb, + "papers", + {"error": "leaked secret-test"}, + secrets=["secret-test"], + ) + assert read_state(kb) == {} + + +def test_state_write_times_out_while_add_holds_the_lock(tmp_path): + assert LOCK_TIMEOUT_SECONDS == 30 + kb = _kb(tmp_path) + started = threading.Event() + release = threading.Event() + + def holder() -> None: + with kb_ingest_lock(kb / ".openkb"): + started.set() + assert release.wait(timeout=2) + + thread = threading.Thread(target=holder) + thread.start() + assert started.wait(timeout=2) + try: + with pytest.raises(KbBusyError, match="knowledge base is busy"): + update_state( + kb, + "papers", + {"provider": "s3", "location": {}, "error": None}, + timeout=0.2, + ) + finally: + release.set() + thread.join(timeout=2) + + +def test_sync_skips_known_hash_and_updates_same_key(tmp_path, monkeypatch): + kb = _kb(tmp_path) + _creds(kb) + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + upsert_binding(kb, _binding(prefix="")) + identity = s3_object_identity("acme", "docs/a.txt") + HashRegistry(kb / ".openkb" / "hashes.json").add( + "old", + {"doc_name": "a", "path": identity, "name": "a.txt"}, + ) + calls: list[str] = [] + + def add(path: Path, kb_dir: Path, identity: str) -> str: + assert not kb_ingest_lock_held(kb_dir / ".openkb") + calls.append(identity) + if path.read_bytes() == b"same": + return "skipped" + return "added" + + same = RemoteObject(identity=identity, name="a.txt") + backend = _Backend([same]) + backend.download = lambda obj, dest: dest.write_bytes(b"same") # type: ignore[method-assign] + outcome = sync_bindings( + kb, + add=add, + backend_for=lambda binding, access, secret: backend, + ) + assert outcome.exit_code == 0 + assert outcome.reports[0].skipped == 1 + assert calls == [identity] + + backend.download = lambda obj, dest: dest.write_bytes(b"new") # type: ignore[method-assign] + outcome = sync_bindings( + kb, + add=add, + backend_for=lambda binding, access, secret: backend, + ) + assert outcome.reports[0].updated == 1 + assert outcome.reports[0].added == 0 + + +def test_new_key_adds_and_one_failure_keeps_the_rest(tmp_path, monkeypatch): + kb = _kb(tmp_path) + _creds(kb) + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + upsert_binding(kb, _binding(prefix="")) + good = RemoteObject(identity="s3://acme/a.txt", name="a.txt") + bad = RemoteObject(identity="s3://acme/b.txt", name="b.txt") + noise = RemoteObject(identity="s3://acme/c.bin", name="c.bin") + + def add(path: Path, kb_dir: Path, identity: str) -> str: + if identity.endswith("b.txt"): + raise ProviderError("compile failed") + return "added" + + backend = _Backend([good, bad, noise]) + outcome = sync_bindings(kb, add=add, backend_for=lambda *_args: backend) + report = outcome.reports[0] + assert report.added == 1 + assert report.failed == 1 + assert report.skipped == 1 + assert outcome.exit_code == 1 + + +def test_transient_download_retries_then_fails(tmp_path, monkeypatch): + kb = _kb(tmp_path) + _creds(kb) + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + upsert_binding(kb, _binding(prefix="")) + obj = RemoteObject(identity="s3://acme/a.txt", name="a.txt") + backend = _Backend([obj], download_failures=2) + outcome = sync_bindings( + kb, + add=lambda path, kb_dir, identity: "added", + backend_for=lambda *_args: backend, + ) + assert outcome.exit_code == 0 + assert backend.downloads == 3 + assert outcome.reports[0].added == 1 + + backend = _Backend([obj], download_failures=3) + outcome = sync_bindings( + kb, + add=lambda path, kb_dir, identity: "added", + backend_for=lambda *_args: backend, + ) + assert outcome.reports[0].failed == 1 + assert outcome.exit_code == 1 + + +def test_non_interactive_does_not_call_sign_in(): + called: list[str] = [] + with pytest.raises(ProviderError): + recover_auth(non_interactive=True, sign_in=lambda: called.append("opened")) + assert called == [] + + +def test_s3_lists_pages_and_empty_prefix_is_the_bucket_root(tmp_path): + class _Paginator: + def __init__(self) -> None: + self.kwargs: dict[str, str] = {} + + def paginate(self, **kwargs): + self.kwargs = kwargs + yield {"Contents": [{"Key": "notes.txt"}, {"Key": "dir/"}]} + yield {"Contents": [{"Key": "sub/paper.pdf"}]} + + class _Client: + def __init__(self) -> None: + self.paginator = _Paginator() + self.downloaded: list[str] = [] + + def get_paginator(self, name: str): + assert name == "list_objects_v2" + return self.paginator + + def download_file(self, bucket: str, key: str, dest: str) -> None: + self.downloaded.append(f"{bucket}/{key}") + Path(dest).write_text(key, encoding="utf-8") + + client = _Client() + binding = _binding(prefix="") + backend = S3Backend(binding, "a", "b", client=client) + found = backend.list_objects() + assert "Prefix" not in client.paginator.kwargs + assert [item.identity for item in found] == [ + "s3://acme/notes.txt", + "s3://acme/sub/paper.pdf", + ] + out = tmp_path / found[0].name + backend.download(found[0], out) + assert out.read_text(encoding="utf-8") == "notes.txt" + assert client.downloaded == ["acme/notes.txt"] + + +def test_missing_boto3_names_the_extra(): + try: + import boto3 # noqa: F401 + except ImportError: + with pytest.raises(ProviderError, match=r"openkb\[s3\]"): + _load_boto3() + else: + pytest.skip("boto3 is installed in this environment") diff --git a/tests/test_gdrive.py b/tests/test_gdrive.py new file mode 100644 index 00000000..85d48beb --- /dev/null +++ b/tests/test_gdrive.py @@ -0,0 +1,486 @@ +"""Google Drive binding, loopback sign-in, and sync. No live Google account.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +import pytest + +from openkb.connectors.backend import ProviderError, RemoteObject, TransientError +from openkb.connectors.gdrive import ( + GDriveBackend, + _load_google, + access_token_for, + prepare_gdrive_binding, +) +from openkb.connectors.model import parse_gdrive_binding, parse_gdrive_folder +from openkb.connectors.oauth import ( + LOOPBACK_HOST, + LOOPBACK_PORT, + SIGN_IN_TIMEOUT_SECONDS, + AuthRejected, + save_refresh_token, + sign_in, +) +from openkb.connectors.sync import sync_bindings +from openkb.locks import kb_ingest_lock_held, kb_lock + + +def _kb(tmp_path: Path) -> Path: + kb = tmp_path / "kb" + (kb / ".openkb").mkdir(parents=True) + (kb / "wiki").mkdir() + (kb / "raw").mkdir() + return kb + + +def _isolate(monkeypatch, tmp_path: Path) -> None: + global_dir = tmp_path / "global-config" + global_dir.mkdir(exist_ok=True) + monkeypatch.setattr("openkb.connectors.credentials.GLOBAL_CONFIG_DIR", global_dir) + for key in ( + "OPENKB_GDRIVE_CLIENT_ID", + "OPENKB_GDRIVE_CLIENT_SECRET", + "OPENKB_GDRIVE_REFRESH_TOKEN", + "OPENKB_S3_ACCESS_KEY_ID", + "OPENKB_S3_SECRET_ACCESS_KEY", + ): + monkeypatch.delenv(key, raising=False) + + +def _client_env(kb: Path, refresh: str = "") -> None: + lines = "OPENKB_GDRIVE_CLIENT_ID=client\nOPENKB_GDRIVE_CLIENT_SECRET=secret\n" + if refresh: + lines += f"OPENKB_GDRIVE_REFRESH_TOKEN={refresh}\n" + (kb / ".env").write_text(lines, encoding="utf-8") + + +def _free_port() -> int: + server = HTTPServer((LOOPBACK_HOST, 0), BaseHTTPRequestHandler) + port = int(server.server_address[1]) + server.server_close() + return port + + +def test_folder_url_and_raw_id(): + url = "https://drive.google.com/drive/folders/1AbCdefghij?usp=sharing" + assert parse_gdrive_folder(url) == "1AbCdefghij" + assert parse_gdrive_folder("1AbCdefghij") == "1AbCdefghij" + with pytest.raises(ValueError): + parse_gdrive_folder("not a folder") + + +def test_sign_in_constants(): + assert LOOPBACK_HOST == "127.0.0.1" + assert LOOPBACK_PORT == 7567 + assert SIGN_IN_TIMEOUT_SECONDS == 300 + + +def test_sign_in_refuses_a_public_bind(): + with pytest.raises(ProviderError, match="127.0.0.1"): + sign_in( + authorize_url="https://example.invalid/auth", + token_url="https://example.invalid/token", + client_id="client", + client_secret="secret", + scopes="scope", + redirect_uri="http://127.0.0.1:9/callback", + host="0.0.0.0", + port=9, + timeout=0.1, + ) + + +def test_sign_in_port_in_use(): + port = _free_port() + holder = HTTPServer((LOOPBACK_HOST, port), BaseHTTPRequestHandler) + try: + with pytest.raises(ProviderError, match=f"port {port} is already in use"): + sign_in( + authorize_url="https://example.invalid/auth", + token_url="https://example.invalid/token", + client_id="client", + client_secret="secret", + scopes="scope", + redirect_uri=f"http://127.0.0.1:{port}/callback", + port=port, + timeout=0.2, + ) + finally: + holder.server_close() + + +def test_sign_in_times_out(): + port = _free_port() + with pytest.raises(ProviderError, match="timed out"): + sign_in( + authorize_url="https://example.invalid/auth", + token_url="https://example.invalid/token", + client_id="client", + client_secret="secret", + scopes="scope", + redirect_uri=f"http://127.0.0.1:{port}/callback", + port=port, + timeout=0.2, + ) + + +def test_sign_in_checks_state_and_pkce(): + callback_port = _free_port() + token_box: dict[str, str] = {} + + class _Token(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + parsed = urllib.parse.parse_qs(self.rfile.read(length).decode("utf-8")) + token_box.update({key: values[0] for key, values in parsed.items()}) + body = json.dumps({"refresh_token": "refresh-1", "access_token": "access-1"}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt: str, *args: object) -> None: + return + + token_server = HTTPServer((LOOPBACK_HOST, 0), _Token) + token_port = int(token_server.server_address[1]) + thread = threading.Thread(target=token_server.serve_forever, daemon=True) + thread.start() + + def announce(url: str) -> None: + query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query) + state = query["state"][0] + challenge = query["code_challenge"][0] + token_box["challenge"] = challenge + + def hit() -> None: + urllib.request.urlopen( + f"http://localhost:{callback_port}/callback?code=the-code&state={state}", + timeout=2, + ) + + threading.Thread(target=hit, daemon=True).start() + + try: + refresh = sign_in( + authorize_url="https://example.invalid/auth", + token_url=f"http://127.0.0.1:{token_port}/token", + client_id="client", + client_secret="secret", + scopes="scope", + redirect_uri=f"http://localhost:{callback_port}/callback", + port=callback_port, + timeout=3, + announce=announce, + ) + finally: + token_server.shutdown() + token_server.server_close() + + assert refresh == "refresh-1" + assert token_box["code"] == "the-code" + assert token_box["code_verifier"] + import base64 + import hashlib + + digest = hashlib.sha256(token_box["code_verifier"].encode("ascii")).digest() + expected = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + assert token_box["challenge"] == expected + + +def test_refresh_token_is_written_only_to_the_kb_env(tmp_path, monkeypatch): + kb = _kb(tmp_path) + global_dir = tmp_path / "global" + global_dir.mkdir() + monkeypatch.setattr("openkb.connectors.credentials.GLOBAL_CONFIG_DIR", global_dir) + save_refresh_token(kb, "OPENKB_GDRIVE_REFRESH_TOKEN", "refresh-1") + text = (kb / ".env").read_text(encoding="utf-8") + assert "OPENKB_GDRIVE_REFRESH_TOKEN=refresh-1" in text + assert (kb / ".env").stat().st_mode & 0o777 == 0o600 + assert not (global_dir / ".env").exists() + + +def test_non_interactive_add_does_not_open_sign_in_or_save(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb) + monkeypatch.delenv("OPENKB_GDRIVE_REFRESH_TOKEN", raising=False) + called: list[str] = [] + with pytest.raises(ProviderError, match="Sign-in is required"): + prepare_gdrive_binding( + kb, + name="notes", + folder="1AbCdefghij", + non_interactive=True, + sign_in_hook=lambda: called.append("opened") or "nope", + refresh=lambda _token: ("access", None), + ) + assert called == [] + assert not (kb / ".openkb" / "config.yaml").exists() + + +def test_rejected_refresh_signs_in_once_then_saves(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb, refresh="old-token") + monkeypatch.delenv("OPENKB_GDRIVE_REFRESH_TOKEN", raising=False) + calls: list[str] = [] + + def refresh(token: str) -> tuple[str, str | None]: + if token == "old-token": + raise AuthRejected("rejected") + return "access", None + + def hook() -> str: + calls.append("opened") + assert not kb_ingest_lock_held(kb / ".openkb") + return "new-token" + + message = prepare_gdrive_binding( + kb, + name="notes", + folder="https://drive.google.com/drive/folders/1AbCdefghij", + non_interactive=False, + sign_in_hook=hook, + refresh=refresh, + ) + assert "saved" in message + assert calls == ["opened"] + saved = (kb / ".env").read_text(encoding="utf-8") + assert "OPENKB_GDRIVE_REFRESH_TOKEN=new-token" in saved + assert "secret" not in (kb / ".openkb" / "config.yaml").read_text(encoding="utf-8") + + +def test_sign_in_does_not_hold_the_ingest_lock(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb) + monkeypatch.delenv("OPENKB_GDRIVE_REFRESH_TOKEN", raising=False) + started = threading.Event() + release = threading.Event() + + def hook() -> str: + started.set() + assert release.wait(2) + return "new-token" + + def run() -> None: + access_token_for( + kb, + non_interactive=False, + sign_in_hook=hook, + refresh=lambda _token: ("access", None), + ) + + worker = threading.Thread(target=run) + worker.start() + assert started.wait(2) + with kb_lock(kb / ".openkb", exclusive=True, timeout=1): + release.set() + worker.join(2) + assert not worker.is_alive() + + +def test_drive_sync_keeps_s3_when_sign_in_fails(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb) + monkeypatch.delenv("OPENKB_GDRIVE_REFRESH_TOKEN", raising=False) + (kb / ".env").write_text( + (kb / ".env").read_text(encoding="utf-8") + + "OPENKB_S3_ACCESS_KEY_ID=AKIA\nOPENKB_S3_SECRET_ACCESS_KEY=secret-s3\n", + encoding="utf-8", + ) + from openkb.connectors.model import parse_s3_binding + from openkb.connectors.store import upsert_binding + + upsert_binding( + kb, parse_s3_binding(name="papers", region="us-east-1", bucket="acme", prefix="") + ) + upsert_binding(kb, parse_gdrive_binding(name="notes", folder="1AbCdefghij")) + + def fail() -> str: + raise ProviderError("browser closed") + + outcome = sync_bindings( + kb, + sign_in=fail, + add=lambda path, kb_dir, identity: "added", + backend_for=lambda binding, access, secret: _EmptyBackend(), + gdrive_backend_for=lambda binding, token: _EmptyBackend(), + ) + by_name = {item.name: item for item in outcome.reports} + assert by_name["papers"].ok + assert by_name["notes"].ok is False + assert "browser closed" in (by_name["notes"].error or "") + + +class _EmptyBackend: + def list_objects(self) -> list[RemoteObject]: + return [] + + def download(self, obj: RemoteObject, dest: Path) -> None: + dest.write_bytes(b"") + + +def test_drive_lists_recursively_and_skips_unsupported(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb, refresh="refresh-1") + monkeypatch.delenv("OPENKB_GDRIVE_REFRESH_TOKEN", raising=False) + from openkb.connectors.store import upsert_binding + + upsert_binding(kb, parse_gdrive_binding(name="notes", folder="1AbCdefghij")) + monkeypatch.setattr( + "openkb.connectors.gdrive.access_token_for", + lambda *_args, **_kwargs: "access-token", + ) + + class _Files: + def list_objects(self) -> list[RemoteObject]: + return [ + RemoteObject("gdrive://1AbCdefghij/a.txt", "a.txt", "a"), + RemoteObject("gdrive://1AbCdefghij/c.bin", "c.bin", "bin"), + RemoteObject("gdrive://1AbCdefghij/sub/b.pdf", "b.pdf", "b"), + ] + + def download(self, obj: RemoteObject, dest: Path) -> None: + dest.write_text(obj.locator, encoding="utf-8") + + seen: list[str] = [] + + def add(path, kb_dir, identity) -> str: + seen.append(identity) + assert not kb_ingest_lock_held(kb_dir / ".openkb") + return "added" + + outcome = sync_bindings( + kb, + add=add, + gdrive_backend_for=lambda binding, token: _Files(), + ) + assert outcome.exit_code == 0 + report = outcome.reports[0] + assert report.added == 2 + assert report.skipped == 1 + assert seen == ["gdrive://1AbCdefghij/a.txt", "gdrive://1AbCdefghij/sub/b.pdf"] + + +def test_listing_shape_uses_the_bound_folder_id(): + class _Files: + def list_folder(self, folder_id: str) -> list[dict[str, object]]: + if folder_id == "root-folder": + return [{"id": "sub", "name": "sub", "folder": True}] + return [{"id": "file", "name": "notes.txt", "folder": False}] + + def download(self, file_id: str, dest: Path) -> None: + dest.write_text(file_id, encoding="utf-8") + + binding = parse_gdrive_binding(name="notes", folder="root-folder") + backend = GDriveBackend(binding, "access", client=_Files()) # type: ignore[arg-type] + assert backend.list_objects() == [ + RemoteObject( + identity="gdrive://root-folder/sub/notes.txt", + name="notes.txt", + locator="file", + ) + ] + + +def test_refresh_gives_up_after_three_transient_failures(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb, refresh="refresh-1") + calls = {"n": 0} + + def refresh(_token: str) -> tuple[str, str | None]: + calls["n"] += 1 + raise TransientError("HTTP 503") + + with pytest.raises(ProviderError, match="503"): + access_token_for(kb, non_interactive=True, refresh=refresh) + assert calls["n"] == 3 + + +def test_missing_google_names_the_extra(): + try: + import googleapiclient # noqa: F401 + except ImportError: + with pytest.raises(ProviderError, match=r"openkb\[gdrive\]"): + _load_google() + else: + pytest.skip("google-api-python-client is installed in this environment") + + +def test_bad_folder_does_not_sign_in(tmp_path, monkeypatch): + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb, refresh="refresh-1") + called: list[str] = [] + with pytest.raises(ValueError, match="folder"): + prepare_gdrive_binding( + kb, + name="notes", + folder="bad id", + non_interactive=False, + sign_in_hook=lambda: called.append("opened") or "token", + refresh=lambda _token: ("access", None), + ) + assert called == [] + assert not (kb / ".openkb" / "config.yaml").exists() + + +def test_cli_non_interactive_add_does_not_save(tmp_path, monkeypatch): + from click.testing import CliRunner + + from openkb.cli import cli + + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb) + monkeypatch.chdir(kb) + result = CliRunner().invoke( + cli, + [ + "source", + "add", + "gdrive", + "--name", + "notes", + "--folder", + "1AbCdefghij", + "--non-interactive", + ], + ) + assert result.exit_code != 0 + assert "Sign-in is required" in result.output + assert not (kb / ".openkb" / "config.yaml").exists() + + +def test_api_add_is_non_interactive(tmp_path, monkeypatch): + from fastapi.testclient import TestClient + + from openkb.api import create_app + + _isolate(monkeypatch, tmp_path) + kb = _kb(tmp_path) + _client_env(kb) + monkeypatch.setattr("openkb.api_helpers.resolve_kb_alias", lambda name: kb) + monkeypatch.setenv("OPENKB_API_TOKEN", "secret") + client = TestClient(create_app()) + response = client.post( + "/api/v1/connectors", + headers={"Authorization": "Bearer secret"}, + json={"kb": "demo", "name": "notes", "provider": "gdrive", "folder": "1AbCdefghij"}, + ) + assert response.status_code == 400 + assert "Sign-in is required" in response.text + assert not (kb / ".openkb" / "config.yaml").exists() diff --git a/tests/test_source_cli.py b/tests/test_source_cli.py new file mode 100644 index 00000000..043cae11 --- /dev/null +++ b/tests/test_source_cli.py @@ -0,0 +1,99 @@ +"""CLI and REST coverage for ``openkb source``.""" + +from __future__ import annotations + +from click.testing import CliRunner + +from openkb.cli import cli +from openkb.connectors.sync import BindingReport, SyncOutcome + + +def _kb(tmp_path): + kb = tmp_path / "kb" + (kb / ".openkb").mkdir(parents=True) + (kb / "wiki").mkdir() + (kb / ".env").write_text( + "OPENKB_S3_ACCESS_KEY_ID=AKIA_TEST\nOPENKB_S3_SECRET_ACCESS_KEY=secret-test\n", + encoding="utf-8", + ) + return kb + + +def test_source_add_requires_credentials_before_saving(tmp_path, monkeypatch): + kb = _kb(tmp_path) + (kb / ".env").write_text("", encoding="utf-8") + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + monkeypatch.chdir(kb) + result = CliRunner().invoke( + cli, + ["source", "add", "s3", "--name", "papers", "--region", "us-east-1", "--bucket", "acme"], + ) + assert result.exit_code != 0 + assert "credentials are missing" in result.output + assert ( + "connectors" not in (kb / ".openkb" / "config.yaml").read_text(encoding="utf-8") + if (kb / ".openkb" / "config.yaml").exists() + else True + ) + + +def test_source_add_list_remove(tmp_path, monkeypatch): + kb = _kb(tmp_path) + monkeypatch.delenv("OPENKB_S3_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("OPENKB_S3_SECRET_ACCESS_KEY", raising=False) + monkeypatch.chdir(kb) + runner = CliRunner() + added = runner.invoke( + cli, + [ + "source", + "add", + "s3", + "--name", + "papers", + "--region", + "us-east-1", + "--bucket", + "acme", + "--prefix", + "docs/", + "--non-interactive", + ], + ) + assert added.exit_code == 0, added.output + listed = runner.invoke(cli, ["source", "list"]) + assert "papers" in listed.output + assert "s3://acme/docs/" in listed.output + removed = runner.invoke(cli, ["source", "remove", "papers"]) + assert removed.exit_code == 0 + assert "remain in the wiki" in removed.output + + +def test_source_sync_exit_code(tmp_path, monkeypatch): + kb = _kb(tmp_path) + monkeypatch.chdir(kb) + (kb / ".openkb" / "config.yaml").write_text( + "connectors:\n- name: papers\n provider: s3\n region: us-east-1\n" + " bucket: acme\n prefix: ''\n", + encoding="utf-8", + ) + + def fake_sync(kb_dir, *, name=None, non_interactive=False, **kwargs): + assert non_interactive is True + return SyncOutcome( + reports=[ + BindingReport( + name="papers", + provider="s3", + ok=False, + failed=1, + error="one file failed", + ) + ] + ) + + monkeypatch.setattr("openkb.source_cmd.sync_bindings", fake_sync) + result = CliRunner().invoke(cli, ["source", "sync", "--non-interactive"]) + assert result.exit_code == 1 + assert "failed 1" in result.output diff --git a/uv.lock b/uv.lock index b9c12c73..ce2f5c55 100644 --- a/uv.lock +++ b/uv.lock @@ -237,6 +237,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "boto3" +version = "1.43.124" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/fc/71d0afe1c7bad85524c292bcfb53d89acd5d05a944992a10066eb8e7ecd3/boto3-1.43.124.tar.gz", hash = "sha256:7f330e53e274a0b8e5982ed3dd4cec9e7e932cf48306c08db8b9e880d09c5c63", size = 112680, upload-time = "2026-11-04T21:22:36.001Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/04/7a7c75e465b063d40d63b6fc5217e312793400fb43411d3064a5ba9174cc/boto3-1.43.124-py3-none-any.whl", hash = "sha256:62edd40fdeef6523ee2f73f4ccbe754fde79200ad0763bd86f4422e8125b2735", size = 140042, upload-time = "2026-11-04T21:22:33.566Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.124" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d3/1e/c0abe23b9495414dc79e3faeb6e96e2336053fd8cae7d80cc6bbe4bd02f3/botocore-1.43.124.tar.gz", hash = "sha256:6df80235995e2926e0768e7ad270e774adcd6ff0307da7ab87ffb8c384edb578", size = 16509340, upload-time = "2026-11-04T21:22:30.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/a4/77409b07ba0b5520dbc591c33b305c5fe4c186036d881ca894a597c0dfba/botocore-1.43.124-py3-none-any.whl", hash = "sha256:4651bac68d1aef78d42c8aefb514a6cd86fda07971b8348652a9d24f21b07965", size = 16209102, upload-time = "2026-11-04T21:22:22.159Z" }, +] + [[package]] name = "certifi" version = "2026.4.22" @@ -596,7 +624,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -829,6 +857,77 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] +[[package]] +name = "google-api-core" +version = "2.43.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "proto-plus" }, + { name = "protobuf" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/74/307919972c4dc84598f4128a8d6ae05ac3d9e6ca73fe60a7bb93c670931c/google_api_core-2.43.2.tar.gz", hash = "sha256:5dfc5e9d52982bcc03bd6af380a893365b5fdf907dc527f63d3a372833c33329", size = 286250, upload-time = "2026-10-29T17:20:40.193Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/1d/a0120c7d937597293fce74510c80fe0cec8e45b1d702a816b567b6754c25/google_api_core-2.43.2-py3-none-any.whl", hash = "sha256:5a9d21baab97fd3c0547ccfc186a9e64f4227d8d7026f393835316400b29a6d4", size = 224304, upload-time = "2026-10-29T17:20:17.733Z" }, +] + +[[package]] +name = "google-api-python-client" +version = "2.201.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/47/077e889d3c618d971d3bce4ab198c645a607276b8d4bb151cfe78d2c92bb/google_api_python_client-2.201.0.tar.gz", hash = "sha256:d5691982abd7287f53cb0b0e0c6a9984d4103cf864ea0a88cb6e4347bbaf70de", size = 16211808, upload-time = "2026-09-30T22:06:21.223Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/c0/221313acdee33b8e5d3fe8a06ba398fc18cfd14627816764bb00318a6065/google_api_python_client-2.201.0-py3-none-any.whl", hash = "sha256:2d9bf1ba3f12eee8ed3d0f1791ce0605d163432f496baa72d3677faa2cf097d6", size = 16812656, upload-time = "2026-09-30T22:06:18.574Z" }, +] + +[[package]] +name = "google-auth" +version = "2.62.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/38/d79ab09dcfc1e1638d887c8f65657619d3ebc454165c558ef79355971ce3/google_auth-2.62.1.tar.gz", hash = "sha256:3c8e1ce2553784ae17fd220e6f88df3281d86ab98019278f436d06dffab17a9e", size = 398519, upload-time = "2026-11-03T03:06:03.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/dd/2e11f0976ecb3ce48ef3f324837460e9ea50071fa4eb01f1e49df87d3cee/google_auth-2.62.1-py3-none-any.whl", hash = "sha256:015399a5abde39033c69404cc1c079e6d4c2de2993e164c9f4eac6e5376ad72b", size = 267975, upload-time = "2026-11-03T03:06:01.422Z" }, +] + +[[package]] +name = "google-auth-httplib2" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/6d/a511ca64d5412850e351bdec6bb224e5090749bd85c186135e8fdb4fd85a/google_auth_httplib2-0.4.4.tar.gz", hash = "sha256:b931de392c20cfaa351cd789274922bd8cdc001e0e9e96de31b39d71347f8e16", size = 11221, upload-time = "2026-10-01T18:14:25.38Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/ea/27cf5d623efdc1cdb2632b4c872ac5092cd3895671ba1d5004e73c3f1a7c/google_auth_httplib2-0.4.4-py3-none-any.whl", hash = "sha256:bbe5d7b2401bb3a4017f4720e1e91bd273ab9a2bb60b84e65edbc0de127852da", size = 9542, upload-time = "2026-10-01T18:07:31.699Z" }, +] + +[[package]] +name = "googleapis-common-protos" +version = "1.76.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/b6/bc095b23a3e0d978e75c8c18cabe4c11f791837410670e4486f0d539bfba/googleapis_common_protos-1.76.0.tar.gz", hash = "sha256:71db0e44a1d4c2dd024ebad44ca67129cc1e2892e4861df466764749d5ef8642", size = 156250, upload-time = "2026-10-22T23:43:36.263Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6d/78dc67a9e779e594b49bf62604034d3a6aa428a55afc729cafda4b7333e1/googleapis_common_protos-1.76.0-py3-none-any.whl", hash = "sha256:59e13344a9f3651aa201a03e2bee815a3bf43a4baf1f9ff8a9b62a840ce25fac", size = 307707, upload-time = "2026-10-22T23:43:10.445Z" }, +] + [[package]] name = "griffelib" version = "2.0.2" @@ -908,6 +1007,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httplib2" +version = "0.32.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/da/affa4b2adc11468c7d88d9b4258cac432479046edcf212284b55d4528593/httplib2-0.32.1.tar.gz", hash = "sha256:37900eb8a74576bd99fc39ae1dc8409f95f965d45b56cbcb2194bb55f58a90e1", size = 255012, upload-time = "2026-10-28T10:22:02.139Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/47/1d02221cde942fdd7a4ce4a940682e0b054534215ab02ffdee4955e4d66c/httplib2-0.32.1-py3-none-any.whl", hash = "sha256:82e7a6ab08fcb27879864ddf63dbb529fb032ca336c331cb1597d6dd8b47c92e", size = 92277, upload-time = "2026-10-28T10:21:57.1Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -1114,6 +1225,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/8d/302cb2057b7513327b4d575cff6b1d066ee6431a5357fc3f8867cd684406/jiter-0.15.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d5d6090cdc1b7c9e780dfb04949a990adb1e301a2fc0bbcee7de4638d33f9a", size = 344469, upload-time = "2026-05-19T10:09:46.864Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "json-repair" version = "0.59.10" @@ -1989,6 +2109,13 @@ dev = [ { name = "ruff" }, { name = "types-pyyaml" }, ] +gdrive = [ + { name = "google-api-python-client" }, + { name = "google-auth" }, +] +s3 = [ + { name = "boto3" }, +] web = [ { name = "fastapi" }, { name = "python-multipart" }, @@ -1997,9 +2124,12 @@ web = [ [package.metadata] requires-dist = [ + { name = "boto3", marker = "extra == 's3'", specifier = "==1.43.124" }, { name = "click", specifier = "==8.4.0" }, { name = "fastapi", marker = "extra == 'api'" }, { name = "fastapi", marker = "extra == 'web'" }, + { name = "google-api-python-client", marker = "extra == 'gdrive'", specifier = "==2.201.0" }, + { name = "google-auth", marker = "extra == 'gdrive'", specifier = "==2.62.1" }, { name = "httpx", marker = "extra == 'dev'" }, { name = "json-repair", specifier = "==0.59.10" }, { name = "litellm", specifier = "==1.87.2" }, @@ -2024,7 +2154,7 @@ requires-dist = [ { name = "uvicorn", marker = "extra == 'web'" }, { name = "watchdog", specifier = "==6.0.0" }, ] -provides-extras = ["api", "dev", "web"] +provides-extras = ["api", "dev", "gdrive", "s3", "web"] [[package]] name = "openpyxl" @@ -2038,6 +2168,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -2078,10 +2220,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2149,9 +2291,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -2463,6 +2605,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "proto-plus" +version = "1.29.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/70/783e33ffbb4466cc154a94f79b869b92a451e2bd45605054e68ff68b7af6/proto_plus-1.29.0.tar.gz", hash = "sha256:cfb4e62ad7e13dd18f346cabbda00cab39930d36a05791fd81ddb074d6ee884f", size = 58592, upload-time = "2026-09-29T19:26:15.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/05/a3ef5b1161498e7b5a2a60288a9cf41b9149e94555423a43ec536737ac1a/proto_plus-1.29.0-py3-none-any.whl", hash = "sha256:8acd070469a7aaf43f440b022ef9757c8cac1a9f866e933f59ae98669ddc6c8b", size = 50786, upload-time = "2026-09-29T19:25:50.409Z" }, +] + [[package]] name = "protobuf" version = "7.35.0" @@ -2478,6 +2632,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -2674,6 +2849,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/a4/b9e91aac82293f9c954654c85581ee8212b5b05efadc534b581141241e6f/pymupdf-1.27.2.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:77691604c5d1d0233827139bbcdea61fd57879c84712b8e49b1f45520f7ab9c2", size = 25000393, upload-time = "2026-04-24T14:11:01.669Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/11/b213bebff182584360cb8d17c72c1677fec5c5c228de439e63bcf8ab1c8f/pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36", size = 1050487, upload-time = "2026-09-20T20:59:05.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, +] + [[package]] name = "pypdf2" version = "3.0.1" @@ -3174,6 +3358,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/6a/aca01554949f3a401991dc32fe22837baeaccb8a0d868256cbb26a029778/ruff-0.9.7-py3-none-win_arm64.whl", hash = "sha256:b075a700b2533feb7a01130ff656a4ec0d5f340bb540ad98759b8401c32c2037", size = 10177763, upload-time = "2025-02-20T13:26:48.92Z" }, ] +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" @@ -3516,6 +3712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, ] +[[package]] +name = "uritemplate" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, +] + [[package]] name = "urllib3" version = "2.7.0"