Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,16 @@ LLM_API_KEY=your-key-here
# --- REST API server variables (read by `openkb-api` / `python -m openkb.api`) ---
# OPENKB_API_TOKEN=... # required bearer token clients send as Authorization: Bearer <token>
# OPENKB_KB_ROOT=... # optional; where REST /init creates KBs (default: ~/.config/openkb/kbs)

# --- Remote data sources (optional extras; not read from config.yaml) ---
# Amazon S3 — pip install "openkb[s3]"
OPENKB_S3_ACCESS_KEY_ID=
OPENKB_S3_SECRET_ACCESS_KEY=
# Google Drive — OAuth client you register. The refresh token is written by `openkb source add`, not by hand.
OPENKB_GDRIVE_CLIENT_ID=
OPENKB_GDRIVE_CLIENT_SECRET=
OPENKB_GDRIVE_REFRESH_TOKEN=
# OneDrive — same rule for the refresh token.
OPENKB_ONEDRIVE_CLIENT_ID=
OPENKB_ONEDRIVE_CLIENT_SECRET=
OPENKB_ONEDRIVE_REFRESH_TOKEN=
3 changes: 3 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ humans steer, agents execute. Optimize changes for agent legibility.
- `config.py` — config loading/validation (LiteLLM passthrough, env).
- `converter.py` — document → markdown conversion (markitdown).
- `url_ingest.py` — fetch & ingest URLs (trafilatura).
- `source_cmd.py` — `openkb source` remote bindings (S3).
- `connectors/` — list, download, and sync a remote prefix through `add_single_file`.
- `api_connectors_router.py` — REST mirror of `openkb source`.
- `images.py` — figure/image extraction & handling.
- `indexer.py` — PageIndex tree indexing for long docs.
- `mutation.py` — crash-safe, serial KB mutations.
Expand Down
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,22 @@ Open `http://127.0.0.1:7566/` for the Workbench. Auth is off by default (local-f

> Working on the UI itself? Run the Vite dev server with `cd frontend && npm install && npm run dev` (it proxies `/api` to a running `openkb-web`), or `npm run build` to regenerate the bundled `openkb/web/`.

### Amazon S3

Bind a bucket prefix and compile those objects into the local wiki. The SDK is an optional extra. Credentials use the CLI order: process environment, then `<kb>/.env`, then `~/.config/openkb/.env`.

```bash
pip install "openkb[s3]"
# <kb>/.env
# OPENKB_S3_ACCESS_KEY_ID=...
# OPENKB_S3_SECRET_ACCESS_KEY=...

openkb source add s3 --name papers --region us-east-1 --bucket my-bucket --prefix papers/
openkb source sync papers
```

Google Drive and OneDrive are not available yet. An empty `--prefix` is the bucket root. `openkb source remove` deletes the binding and leaves compiled pages in place.

# 🧩 How OpenKB Works

### Architecture
Expand Down
259 changes: 230 additions & 29 deletions frontend/src/components/ConnectorCards.tsx
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import { useCallback, useEffect, useState } from "react"
import { Cloud, HardDrive, ThumbsUp, type LucideIcon } from "lucide-react"
import { useTranslation } from "react-i18next"
import { ApiError, apiFetch } from "@/api/client"

/**
* Remote document-source connectors are NOT implemented (no OAuth/S3/sync
* backend). Rather than dead "coming soon" tiles, each card links to a GitHub
* issue search where users 👍 to signal demand — a telemetry-free, OSS-native
* way to prioritize. The copy MUST never imply a connector is connected.
* Remote document-source connectors.
*
* `voteUrl` is an interim per-connector label search on the canonical repo; swap
* to a specific `.../issues/<N>` once a tracking issue exists (one-line change).
* A provider with no backend keeps the GitHub vote link (`label:connector:<id>`).
* Amazon S3 is shipped: with a knowledge base selected, zero bindings show a
* connect form and one or more bindings show Sync / Remove / Add another.
*/
export interface Connector {
id: string
Expand All @@ -21,38 +21,239 @@ const REPO = "https://github.com/VectifyAI/OpenKB"
const voteSearch = (label: string) =>
`${REPO}/issues?q=${encodeURIComponent(`is:issue label:connector:${label}`)}`

const SHIPPED = new Set(["s3"])

export const CONNECTORS: Connector[] = [
{ id: "gdrive", label: "Google Drive", icon: Cloud, voteUrl: voteSearch("gdrive") },
{ id: "s3", label: "Amazon S3", icon: HardDrive, voteUrl: voteSearch("s3") },
{ id: "onedrive", label: "OneDrive", icon: Cloud, voteUrl: voteSearch("onedrive") },
]

/** Grid of connector "want this?" cards linking to GitHub demand-vote searches. */
export default function ConnectorCards() {
interface BindingView {
name: string
provider: string
description: string
stale: boolean
error: string | null
}

const inputCls =
"mt-1 w-full h-8 rounded-md border border-input bg-transparent px-2 text-[12px] font-mono2 outline-none focus-visible:ring-2 focus-visible:ring-ring"

function VoteCard({ connector }: { connector: Connector }) {
const { t } = useTranslation("common")
const Icon = connector.icon
return (
<div className="grid grid-cols-1 sm:grid-cols-3 gap-2.5">
{CONNECTORS.map((c) => (
<a
key={c.id}
href={c.voteUrl}
target="_blank"
rel="noopener noreferrer"
title={t("connector.tooltip")}
className="group rounded-2xl border border-[hsl(var(--glass-border))] glass px-4 py-3.5 flex items-center gap-3 hover:shadow-glass hover:-translate-y-0.5 transition duration-fast ease-out-apple active:scale-[0.99]"
<a
href={connector.voteUrl}
target="_blank"
rel="noopener noreferrer"
title={t("connector.tooltip")}
className="group rounded-2xl border border-[hsl(var(--glass-border))] glass px-4 py-3.5 flex items-center gap-3 hover:shadow-glass hover:-translate-y-0.5 transition duration-fast ease-out-apple active:scale-[0.99]"
>
<span className="w-9 h-9 rounded-xl glass-2 border border-[hsl(var(--glass-border))] grid place-items-center shrink-0">
<Icon className="w-4 h-4 text-muted-foreground group-hover:text-accent-brand transition-colors" />
</span>
<div className="min-w-0 flex-1">
<div className="text-[13px] font-medium text-foreground truncate">{connector.label}</div>
<div className="text-[11.5px] text-muted-foreground mt-0.5 inline-flex items-center gap-1">
<ThumbsUp className="w-3 h-3" />
{t("connector.vote")}
</div>
</div>
</a>
)
}

function S3Card({ kb }: { kb?: string }) {
const { t } = useTranslation("common")
const [bindings, setBindings] = useState<BindingView[]>([])
const [adding, setAdding] = useState(false)
const [pending, setPending] = useState<"save" | "sync" | "remove" | null>(null)
const [error, setError] = useState<string | null>(null)
const [name, setName] = useState("")
const [region, setRegion] = useState("")
const [bucket, setBucket] = useState("")
const [prefix, setPrefix] = useState("")

const reload = useCallback(async () => {
if (!kb) return
const body = await apiFetch<{ connectors: BindingView[] }>(
`/api/v1/connectors?kb=${encodeURIComponent(kb)}`,
)
setBindings(body.connectors.filter((item) => item.provider === "s3"))
}, [kb])

useEffect(() => {
if (!kb) return
let cancelled = false
reload().catch((exc: unknown) => {
if (!cancelled) setError(exc instanceof Error ? exc.message : String(exc))
})
return () => {
cancelled = true
}
}, [kb, reload])

const save = async () => {
if (!kb) return
setPending("save")
setError(null)
try {
await apiFetch("/api/v1/connectors", {
method: "POST",
body: { kb, name, provider: "s3", region, bucket, prefix },
})
setName("")
setRegion("")
setBucket("")
setPrefix("")
setAdding(false)
await reload()
} catch (exc) {
const message = exc instanceof ApiError ? exc.message : String(exc)
setError(t("connector.formError", { error: message }))
} finally {
setPending(null)
}
}

const syncOne = async (bindingName: string) => {
if (!kb) return
setPending("sync")
setError(null)
try {
await apiFetch("/api/v1/connectors/sync", {
method: "POST",
body: { kb, name: bindingName, non_interactive: true },
})
await reload()
} catch (exc) {
setError(exc instanceof Error ? exc.message : String(exc))
} finally {
setPending(null)
}
}

const removeOne = async (bindingName: string) => {
if (!kb) return
setPending("remove")
setError(null)
try {
await apiFetch("/api/v1/connectors/remove", {
method: "POST",
body: { kb, name: bindingName },
})
await reload()
} catch (exc) {
setError(exc instanceof Error ? exc.message : String(exc))
} finally {
setPending(null)
}
}

const showForm = Boolean(kb) && (bindings.length === 0 || adding)

return (
<div className="rounded-2xl border border-[hsl(var(--glass-border))] glass px-4 py-3.5">
<div className="flex items-center gap-3">
<span className="w-9 h-9 rounded-xl glass-2 border border-[hsl(var(--glass-border))] grid place-items-center shrink-0">
<HardDrive className="w-4 h-4 text-muted-foreground" />
</span>
<div className="text-[13px] font-medium text-foreground">Amazon S3</div>
</div>
{!kb && <p className="mt-2 text-[12px] text-muted-foreground">{t("connector.needKb")}</p>}
{kb && bindings.length > 0 && (
<ul className="mt-3 space-y-2">
{bindings.map((binding) => (
<li key={binding.name} className="text-[12px]">
<div className="font-medium text-foreground">{binding.name}</div>
<div className="text-muted-foreground truncate">{binding.description}</div>
<div className="text-muted-foreground">
{binding.stale ? t("connector.notSynced") : t("connector.synced")}
</div>
{binding.error && <div className="text-red-600">{binding.error}</div>}
<div className="mt-1 flex gap-2">
<button
type="button"
disabled={pending !== null}
onClick={() => syncOne(binding.name)}
className="h-7 px-2 rounded-lg text-[12px] font-medium hover:bg-accent"
>
{t("connector.sync")}
</button>
<button
type="button"
disabled={pending !== null}
onClick={() => removeOne(binding.name)}
className="h-7 px-2 rounded-lg text-[12px] font-medium text-red-600 hover:bg-red-50 dark:hover:bg-red-500/10"
>
{t("connector.remove")}
</button>
</div>
</li>
))}
</ul>
)}
{kb && bindings.length > 0 && !adding && (
<button
type="button"
className="mt-3 text-[12px] font-medium text-accent-brand"
onClick={() => setAdding(true)}
>
{t("connector.addAnother")}
</button>
)}
{showForm && (
<form
className="mt-3 space-y-2"
onSubmit={(event) => {
event.preventDefault()
void save()
}}
>
<span className="w-9 h-9 rounded-xl glass-2 border border-[hsl(var(--glass-border))] grid place-items-center shrink-0">
<c.icon className="w-4 h-4 text-muted-foreground group-hover:text-accent-brand transition-colors" />
</span>
<div className="min-w-0 flex-1">
<div className="text-[13px] font-medium text-foreground truncate">{c.label}</div>
<div className="text-[11.5px] text-muted-foreground mt-0.5 inline-flex items-center gap-1">
<ThumbsUp className="w-3 h-3" />
{t("connector.vote")}
</div>
</div>
</a>
))}
<label className="block text-[11.5px] text-muted-foreground">
{t("connector.name")}
<input className={inputCls} value={name} onChange={(event) => setName(event.target.value)} />
</label>
<label className="block text-[11.5px] text-muted-foreground">
{t("connector.region")}
<input className={inputCls} value={region} onChange={(event) => setRegion(event.target.value)} />
</label>
<label className="block text-[11.5px] text-muted-foreground">
{t("connector.bucket")}
<input className={inputCls} value={bucket} onChange={(event) => setBucket(event.target.value)} />
</label>
<label className="block text-[11.5px] text-muted-foreground">
{t("connector.prefix")}
<input className={inputCls} value={prefix} onChange={(event) => setPrefix(event.target.value)} />
<span className="mt-0.5 block">{t("connector.prefixHint")}</span>
</label>
<button
type="submit"
disabled={pending !== null}
className="h-8 px-3 rounded-lg text-[12px] font-medium bg-accent-brand text-white disabled:opacity-60"
>
{pending === "save" ? t("connector.saving") : t("connector.save")}
</button>
</form>
)}
{error && <p className="mt-2 text-[12px] text-red-600">{error}</p>}
</div>
)
}

/** Grid of connector cards. Pass `kb` on a knowledge-base page so S3 can bind. */
export default function ConnectorCards({ kb }: { kb?: string }) {
return (
<div className="grid grid-cols-1 sm:grid-cols-3 gap-2.5">
{CONNECTORS.map((connector) =>
SHIPPED.has(connector.id) ? (
<S3Card key={connector.id} kb={kb} />
) : (
<VoteCard key={connector.id} connector={connector} />
),
)}
</div>
)
}
20 changes: 18 additions & 2 deletions frontend/src/locales/en/common.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,24 @@
"baseLabel": "API address (optional; leave blank to use the current origin)",
"connect": "Connect"
},
"connector": {
"connector": {
"tooltip": "Cloud sync is in development · vote for it on GitHub",
"vote": "Want it? Vote"
"vote": "Want it? Vote",
"connect": "Connect",
"addAnother": "Add another",
"sync": "Sync",
"remove": "Remove",
"name": "Name",
"region": "Region",
"bucket": "Bucket",
"prefix": "Prefix",
"prefixHint": "Empty prefix is the whole bucket",
"notSynced": "Not synced since this change",
"synced": "Synced",
"save": "Save binding",
"saving": "Saving…",
"needKb": "Open a knowledge base to connect Amazon S3.",
"missingExtra": "Install the s3 extra on the server: pip install \"openkb[s3]\"",
"formError": "Could not save the binding: {{error}}"
}
}
2 changes: 1 addition & 1 deletion frontend/src/locales/en/kb.json
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
},
"remote": {
"heading": "Remote data sources",
"note": "Cloud connectors are in development and not yet available · want one? Click a card to vote on GitHub"
"note": "Amazon S3 connects on this page. Google Drive and OneDrive are not available yet — click a card to vote on GitHub."
},
"overview": {
"index": { "label": "Index", "caption": "wiki root" },
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/locales/en/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"kbRootEnvPinned": "Pinned by the OPENKB_KB_ROOT environment variable; edits here won't take effect until it's unset."
},
"conn": {
"note": "Cloud data-source connectors (OAuth / S3) are in development and not yet available; for now, upload local files from the KB detail page. Want a connector? Click a card below to vote on GitHub and help us prioritize."
"note": "Amazon S3 is connected from a knowledge base page. Google Drive and OneDrive are not available yet — click a card to vote on GitHub."
},
"about": {
"links": {
Expand Down
Loading