From c6a14704478a1a30c55720c72a362dfad893bcf6 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 10:17:58 +0000 Subject: [PATCH] feat(node): every node is a disk provider, listed on d1sks by signed heartbeat; v0.2.95 Storage and hosting are on for every node unless turned off: an explicit roles list (config.toml or --roles) no longer drops them. Opt-outs: [storage] enabled = false / --no-storage / C0MPUTE_STORAGE=0, [hosting] enabled = false / --no-hosting / C0MPUTE_HOSTING=0. Private mode adds nothing. [storage] root now defaults, so a partial [storage] section parses. Every storage node lists its disk on d1sks, NAT'd or not: a payload (c0mpute.disk.announce/1: marketplace host, peer id, key, ts, expires_in, origin, OpenDisk descriptor with c0mpute.addrs and reachable) signed with the node's libp2p ed25519 identity, POSTed to /api/v1/disks/announce 20 s after start and every 5 min, expiring 20 min after the last one. Replaces the origin-only listing (which needed a public https gateway.announce). Opt out with [storage] list_on_d1sks = false or C0MPUTE_D1SKS=0; private mode never lists. Optional [storage] country / C0MPUTE_COUNTRY. hosting publish --disks auto joins the hosting gossip first and ranks d1sks disks live on gossip first, then fresh reachable, fresh NAT'd, stale; then price, standing, space. hosting disks shows reach, last heartbeat and, with --live, gossip liveness. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 32 +-- Cargo.toml | 2 +- apps/tui/package.json | 2 +- apps/web/package.json | 2 +- .../web/src/app/releases/latest.json/route.ts | 2 +- docs/storage-opendisk.md | 57 +++- node/crates/c0mpute-cli/src/main.rs | 85 +++++- node/crates/c0mpute-core/src/config.rs | 131 ++++++++- .../crates/c0mpute-core/src/hosting/market.rs | 78 +++++- node/crates/c0mpute-core/src/hosting/mod.rs | 58 +++- node/crates/c0mpute-core/src/opendisk.rs | 256 +++++++++++++++--- node/crates/c0mpute-core/src/supervisor.rs | 5 +- package.json | 2 +- packages/shared/package.json | 2 +- plugins/ad-generator-agent/module.toml | 2 +- plugins/ads-manager/module.toml | 2 +- plugins/coinpay/module.toml | 2 +- plugins/crawl/module.toml | 2 +- plugins/crowdgpt/module.toml | 2 +- plugins/ctv-ads/module.toml | 2 +- plugins/diffusion/module.toml | 2 +- plugins/embed/module.toml | 2 +- plugins/gameserver/module.toml | 2 +- plugins/hosting/module.toml | 2 +- plugins/infernet/module.toml | 2 +- plugins/live-stream/module.toml | 2 +- plugins/music/module.toml | 2 +- plugins/ocr/module.toml | 2 +- plugins/podcast-ads/module.toml | 2 +- plugins/podcasting/module.toml | 2 +- plugins/podcasts/module.toml | 2 +- plugins/render/module.toml | 2 +- plugins/secure-chat/module.toml | 2 +- plugins/storage/module.toml | 2 +- plugins/train/module.toml | 2 +- plugins/transcode/module.toml | 2 +- plugins/tts/module.toml | 2 +- plugins/video/module.toml | 2 +- plugins/webhook/module.toml | 2 +- plugins/whisper/module.toml | 2 +- plugins/workflow-step/module.toml | 2 +- scripts/install.sh | 9 + 42 files changed, 662 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bc7bc99..8ab22b55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,7 +478,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "c0mpute-proto", @@ -493,7 +493,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "blake3", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "axum", @@ -534,7 +534,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.2.94" +version = "0.2.95" dependencies = [ "aes-gcm", "anyhow", @@ -581,7 +581,7 @@ dependencies = [ [[package]] name = "c0mpute-doctor" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "serde", @@ -593,7 +593,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.2.94" +version = "0.2.95" dependencies = [ "blake3", "bs58", @@ -606,7 +606,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "axum", @@ -628,7 +628,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.2.94" +version = "0.2.95" dependencies = [ "c0mpute-envelope", "serde", @@ -638,7 +638,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "async-trait", @@ -655,7 +655,7 @@ dependencies = [ [[package]] name = "c0mpute-placement" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "async-trait", @@ -676,7 +676,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.2.94" +version = "0.2.95" dependencies = [ "blake3", "hex", @@ -688,7 +688,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.2.94" +version = "0.2.95" dependencies = [ "aes-gcm", "anyhow", @@ -711,7 +711,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "blake3", @@ -729,7 +729,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "c0mpute-proto", @@ -742,7 +742,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "hex", @@ -757,7 +757,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.2.94" +version = "0.2.95" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index 08de7f29..2cfcbbb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "0.2.94" +version = "0.2.95" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/apps/tui/package.json b/apps/tui/package.json index 24efd2a0..85d71be9 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.2.94", + "version": "0.2.95", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index 9d15e004..a8797132 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.2.94", + "version": "0.2.95", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index 65104940..50789ef5 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.2.94"; +const FALLBACK_VERSION = "0.2.95"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/docs/storage-opendisk.md b/docs/storage-opendisk.md index 2d468631..8671920a 100644 --- a/docs/storage-opendisk.md +++ b/docs/storage-opendisk.md @@ -1,7 +1,8 @@ # Storage on every node, listed on OpenDisk -Storage is a default role. Every node that runs it (that is every node unless -`--roles` leaves it out) serves two things on its gateway, `0.0.0.0:7777`: +Every node is a disk provider by default: `storage` and `hosting` are on even when +`roles` in config.toml (or `--roles`) lists other roles, until the operator turns them +off (below). A node with the gateway role serves two things on `0.0.0.0:7777`: | Path | What | |---|---| @@ -14,28 +15,62 @@ A node with the `hosting` role also says so: `accepts.hosting: true`, `c0mpute.r ## Listing on d1sks -A node lists itself; nothing central does it for the node. When the gateway announces a -public **https** origin, the node submits that origin to the OpenDisk marketplace -(`storage.d1sks_url`, default `https://d1sks.pwa.pit.world`) a few seconds after -start and again every 24 hours. d1sks then reads the descriptor itself, every hour, -and dedupes on the node's key. +A node lists itself; nothing central does it for the node. Every storage node, NAT'd or +not, pushes a **signed listing** to the OpenDisk marketplace (`storage.d1sks_url`, +default `https://d1sks.pwa.pit.world`) 20 seconds after start and then every 5 minutes: + +``` +POST /api/v1/disks/announce +{ "payload": "", "signature": "" } +``` + +The payload is `{ type: "c0mpute.disk.announce/1", marketplace, peer_id, key, ts, +expires_in, origin, descriptor }`. It is signed with the node's libp2p identity, so d1sks +checks that `key` is the key inside `peer_id` and that the signature holds; no origin +fetch is needed. `marketplace` (the d1sks host) and `ts` stop replays. The descriptor is the +same OpenDisk document the gateway serves, plus `c0mpute.addrs` (up to 8 multiaddrs, public +first) and `c0mpute.reachable`: + +| `reachable` | Meaning | +|---|---| +| `https` | the gateway announces a public https origin (`gateway.announce`); d1sks shows its URL | +| `direct` | a public IP multiaddr; dialable over libp2p | +| `c0mpute` | behind NAT; reached through c0mpute over connections the node opens. Still a full holder: storage and hosting move over libp2p | + +A listing expires 20 minutes after the last heartbeat, so a node that goes quiet drops off +by itself. `c0mpute hosting publish --disks auto` ranks disks seen live on the hosting +gossip first, then fresh publicly reachable ones, then fresh NAT'd ones, then stale ones +(price, standing and free space break ties); `c0mpute hosting disks --live` shows the same. + +Origin-based listings still work for any OpenDisk server: `POST /api/v1/disks {"url": ...}` +and d1sks fetches `/.well-known/opendisk.json` hourly. ```toml # ~/.config/c0mpute/config.toml [gateway] bind = "0.0.0.0:7777" -announce = "https://disk.example.net" # TLS in front of :7777 +announce = "https://disk.example.net" # optional: TLS in front of :7777 [storage] +enabled = true # false: not a disk provider at all +list_on_d1sks = true # false: store for the network, stay off d1sks +country = "DE" # optional, shown and filterable on d1sks cap_bytes = 500_000_000_000 # offer 500 GB, not the whole disk price_usd_per_gib_month = 0.004 # the default trusted_writers = [] # extra ed25519 keys (64 hex) allowed to write opendisk = true # false: no descriptor, no listing + +[hosting] +enabled = true # false: hold no sites for the network ``` -A node without a public https origin (most home nodes, behind NAT) still serves its -descriptor locally and still stores shards for the network; it is simply not on the -marketplace until it has an origin d1sks can reach. +| Opt-out | Config | CLI | Environment | +|---|---|---|---| +| no storage, not listed | `[storage] enabled = false` | `worker start --no-storage` | `C0MPUTE_STORAGE=0` | +| no hosting | `[hosting] enabled = false` | `worker start --no-hosting` | `C0MPUTE_HOSTING=0` | +| store, but stay off d1sks | `[storage] list_on_d1sks = false` | | `C0MPUTE_D1SKS=0` | + +Private mode never lists and never adds roles the node did not ask for. ## On the node board diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 954e8d02..641fc376 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -446,11 +446,21 @@ enum WorkerCmd { Start { /// Comma-separated roles: storage, transcode, tts, stt, gateway, /// verifier, workflow-step, video, diffusion, music. Default: the config's - /// roles plus tts, stt, workflow-step, diffusion and music. + /// roles plus tts, stt, workflow-step, diffusion and music. Every node + /// is also a disk provider (storage + hosting, listed on d1sks) on top + /// of whatever is listed here, unless turned off below or in config. #[arg(long, value_delimiter = ',')] roles: Option>, #[arg(long)] storage: Option, + /// Do not be a disk provider: no storage role, not listed on d1sks + /// (same as `[storage] enabled = false` or `C0MPUTE_STORAGE=0`). + #[arg(long)] + no_storage: bool, + /// Do not hold other people's sites (same as `[hosting] enabled = + /// false` or `C0MPUTE_HOSTING=0`). + #[arg(long)] + no_hosting: bool, #[arg(long)] gpu: bool, /// Private mode for this run: only your own buyers, not advertised. @@ -673,6 +683,10 @@ enum HostingCmd { country: Option, #[arg(long, default_value = c0mpute_core::opendisk::DEFAULT_D1SKS_URL)] d1sks: String, + /// Also join the hosting gossip (about a minute) and mark which + /// listed disks are announcing themselves as live hosts right now. + #[arg(long)] + live: bool, }, /// List the sites a gateway knows, with live copy counts. Ls { @@ -1244,6 +1258,8 @@ fn main() -> Result<()> { attach: true, roles, storage, + no_storage, + no_hosting, gpu, private, daemon: _, @@ -1254,6 +1270,12 @@ fn main() -> Result<()> { if *private { passthrough.push("--private".to_string()); } + if *no_storage { + passthrough.push("--no-storage".to_string()); + } + if *no_hosting { + passthrough.push("--no-hosting".to_string()); + } if let Some(rs) = roles { passthrough.push("--roles".to_string()); passthrough.push(rs.join(",")); @@ -1653,6 +1675,8 @@ async fn run_worker(cmd: WorkerCmd, config_path: &std::path::Path) -> Result<()> WorkerCmd::Start { roles, storage: _, + no_storage, + no_hosting, gpu, private, // Both handled in `main` before the runtime started; at this point @@ -1709,6 +1733,23 @@ async fn run_worker(cmd: WorkerCmd, config_path: &std::path::Path) -> Result<()> if gpu && !cfg.roles.contains(&Role::Transcode) { cfg.roles.push(Role::Transcode); } + if no_storage { + cfg.storage.enabled = false; + } + if no_hosting { + cfg.hosting.enabled = false; + } + // Every node is a disk provider unless it opted out: storage and + // hosting join even an explicit --roles list. + cfg.apply_disk_provider_defaults(); + if cfg.lists_on_d1sks() { + eprintln!( + "disk provider: storing shards and holding sites for the network, listed on {} \ + (opt out: --no-storage / --no-hosting, [storage] enabled = false, \ + [storage] list_on_d1sks = false, C0MPUTE_D1SKS=0)", + cfg.storage.d1sks_url + ); + } // Speech is a nice-to-have, not a precondition for joining: a box // that cannot build a TTS runtime should still serve storage, // gateway and verifier work. So this warns rather than aborting, @@ -3556,21 +3597,55 @@ async fn run_hosting(cmd: HostingCmd) -> Result<()> { max_price, country, d1sks, + live, } => { let q = disk_query(&query, max_price.as_deref(), country.as_deref())?; - let disks = c0mpute_core::hosting::market::query(&d1sks, &q).await?; + let m = c0mpute_core::hosting::MarketOptions { + query: q.clone(), + copies: 1, + d1sks_url: d1sks.clone(), + }; + let disks = if live { + eprintln!( + "listening to the hosting gossip for up to {}s ...", + c0mpute_core::hosting::GOSSIP_CHECK.as_secs() + ); + let sup = c0mpute_core::Supervisor::boot_buyer(c0mpute_core::Config::default()).await?; + let h = c0mpute_core::hosting::Hosting::start( + sup.libp2p.clone(), + sup.store.clone(), + c0mpute_core::hosting::Options::default(), + ) + .await?; + c0mpute_core::hosting::market_disks(&m, Some(&h)).await? + } else { + c0mpute_core::hosting::market_disks(&m, None).await? + }; let ranked = c0mpute_core::hosting::market::rank(&disks, &q, 0); if ranked.is_empty() { println!("no c0mpute hosting disks on {d1sks} match ({} listed)", disks.len()); + } else { + println!( + "{:<54} {:<10} {:>10} {:<8} {:<8} {:<8} {:<6} NAME", + "PEER", "$/GiB-MO", "FREE", "REACH", "SEEN", "GOSSIP", "WHERE" + ); } for d in ranked { + let seen = match d.heartbeat_age_secs { + Some(s) if s < 120 => format!("{s}s"), + Some(s) => format!("{}m", s / 60), + None => "-".into(), + }; + let gossip = if d.live { "live" } else if live { "silent" } else { "-" }; println!( - "{:<54} ${:<9} {:>8.1} GiB standing {:<5} {:<8} {}", + "{:<54} {:<10} {:>6.1} GiB {:<8} {:<8} {:<8} {:<6} {}", d.peer_id, c0mpute_core::payments::format_usd(d.price), d.free_bytes as f64 / (1024.0 * 1024.0 * 1024.0), - d.standing.map(|s| format!("{s:.0}")).unwrap_or_else(|| "-".into()), - d.countries.join(","), + d.reachable.as_deref().unwrap_or("-"), + seen, + gossip, + if d.countries.is_empty() { "-".to_string() } else { d.countries.join(",") }, d.name ); } diff --git a/node/crates/c0mpute-core/src/config.rs b/node/crates/c0mpute-core/src/config.rs index d2f56057..902f78cf 100644 --- a/node/crates/c0mpute-core/src/config.rs +++ b/node/crates/c0mpute-core/src/config.rs @@ -161,6 +161,11 @@ impl Default for GameserverConfig { /// sites so the network keeps three of each; every gateway serves them. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct HostingConfig { + /// Hold sites for the network. On by default on every node, whatever + /// `roles` lists; `false` (or `C0MPUTE_HOSTING=0`, or `worker start + /// --no-hosting`) is the opt-out. + #[serde(default = "default_true")] + pub enabled: bool, /// Most bytes of other people's sites this node will hold. #[serde(default = "default_hosting_max_bytes")] pub max_bytes: u64, @@ -193,6 +198,7 @@ pub struct HostingConfig { impl Default for HostingConfig { fn default() -> Self { Self { + enabled: true, max_bytes: default_hosting_max_bytes(), price_usd_per_gib_month: default_hosting_price(), payout_wallet: None, @@ -589,6 +595,12 @@ impl Default for ApiConfig { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct StorageConfig { + /// Be a disk provider: store shards for the network and offer this disk + /// on d1sks. On by default on every node, whatever `roles` lists; `false` + /// (or `C0MPUTE_STORAGE=0`, or `worker start --no-storage`) is the opt-out. + #[serde(default = "default_true")] + pub enabled: bool, + #[serde(default = "default_storage_root")] pub root: PathBuf, pub cap_bytes: Option, /// The price this node asks to hold data, per GiB-month in USD. Published @@ -607,6 +619,20 @@ pub struct StorageConfig { /// The OpenDisk marketplace this node registers with. #[serde(default = "default_d1sks_url")] pub d1sks_url: String, + /// List this disk on d1sks with a signed heartbeat (every few minutes, + /// expiring when the node goes quiet), NAT or not. On by default; + /// `false` or `C0MPUTE_D1SKS=0` keeps the disk off the marketplace while + /// it still stores for the network. Private mode never lists. + #[serde(default = "default_true")] + pub list_on_d1sks: bool, + /// ISO 3166-1 alpha-2 country this disk is in, published in the listing + /// so publishers can pick hosts by country (`C0MPUTE_COUNTRY` wins). + #[serde(default)] + pub country: Option, +} + +fn default_storage_root() -> PathBuf { + data_dir().unwrap_or_else(|| PathBuf::from("./c0mpute-data")) } fn default_storage_price() -> f64 { @@ -620,12 +646,15 @@ fn default_d1sks_url() -> String { impl Default for StorageConfig { fn default() -> Self { Self { - root: data_dir().unwrap_or_else(|| PathBuf::from("./c0mpute-data")), + enabled: true, + root: default_storage_root(), cap_bytes: None, price_usd_per_gib_month: default_storage_price(), trusted_writers: Vec::new(), opendisk: true, d1sks_url: default_d1sks_url(), + list_on_d1sks: true, + country: None, } } } @@ -679,8 +708,67 @@ pub fn default_config_path() -> PathBuf { .join("config.toml") } +/// An on/off switch from the environment: `0`/`false`/`no`/`off` is off, +/// anything else set is on, unset keeps `configured`. +fn env_switch(name: &str, configured: bool) -> bool { + auto_install_enabled_with(configured, std::env::var(name).ok().filter(|v| !v.trim().is_empty())) +} + impl Config { + /// Storage (disk provider) is on: the config and `C0MPUTE_STORAGE` agree. + pub fn storage_enabled(&self) -> bool { + env_switch("C0MPUTE_STORAGE", self.storage.enabled) + } + + /// Hosting (holding sites) is on: the config and `C0MPUTE_HOSTING` agree. + pub fn hosting_enabled(&self) -> bool { + env_switch("C0MPUTE_HOSTING", self.hosting.enabled) + } + + /// This disk goes on d1sks: storage on, OpenDisk on, listing on, + /// `C0MPUTE_D1SKS` not off, and not private mode. + pub fn lists_on_d1sks(&self) -> bool { + self.roles.contains(&Role::Storage) + && self.storage.opendisk + && env_switch("C0MPUTE_D1SKS", self.storage.list_on_d1sks) + && !self.private.enabled + } + + /// The country to publish: `C0MPUTE_COUNTRY`, else `[storage] country`, + /// when it is a two-letter code. + pub fn country(&self) -> Option { + let raw = std::env::var("C0MPUTE_COUNTRY").ok().or_else(|| self.storage.country.clone())?; + let c = raw.trim().to_ascii_uppercase(); + (c.len() == 2 && c.chars().all(|ch| ch.is_ascii_uppercase())).then_some(c) + } + + /// Every node is a disk provider: `storage` and `hosting` join the roles + /// even when `roles` (or `--roles`) lists others, unless the operator + /// turned them off (`enabled = false` or the env switch), which also + /// takes them out of an explicit list. Private mode adds nothing: a + /// private node keeps exactly the roles it asked for. + pub fn apply_disk_provider_defaults(&mut self) { + let wanted = [ + (Role::Storage, self.storage_enabled()), + (Role::Hosting, self.hosting_enabled()), + ]; + for (role, on) in wanted { + if !on { + self.roles.retain(|r| *r != role); + } else if !self.private.enabled && !self.roles.contains(&role) { + self.roles.push(role); + } + } + } + pub fn load_or_default(path: &Path) -> Result { + let mut cfg = Self::load_raw(path)?; + cfg.apply_disk_provider_defaults(); + Ok(cfg) + } + + /// The file as written, without the disk-provider defaults applied. + pub fn load_raw(path: &Path) -> Result { if !path.exists() { return Ok(Self::default()); } @@ -729,6 +817,47 @@ impl Config { mod tests { use super::*; + #[test] + fn an_explicit_role_list_still_provides_disk() { + let mut cfg: Config = toml::from_str("roles = [\"tts\", \"stt\"]\n").unwrap(); + cfg.apply_disk_provider_defaults(); + assert!(cfg.roles.contains(&Role::Storage)); + assert!(cfg.roles.contains(&Role::Hosting)); + assert!(cfg.roles.contains(&Role::Tts)); + assert!(cfg.storage.list_on_d1sks); + } + + #[test] + fn enabled_false_is_the_opt_out_even_from_an_explicit_list() { + let mut cfg: Config = toml::from_str( + "roles = [\"storage\", \"hosting\", \"tts\"]\n[storage]\nenabled = false\n[hosting]\nenabled = false\n", + ) + .unwrap(); + cfg.apply_disk_provider_defaults(); + assert!(!cfg.roles.contains(&Role::Storage)); + assert!(!cfg.roles.contains(&Role::Hosting)); + assert!(!cfg.lists_on_d1sks()); + } + + #[test] + fn private_mode_adds_nothing_and_never_lists() { + let mut cfg: Config = toml::from_str("roles = [\"tts\"]\n[private]\nenabled = true\n").unwrap(); + cfg.apply_disk_provider_defaults(); + assert_eq!(cfg.roles, vec![Role::Tts]); + let mut cfg = Config::default(); + cfg.private.enabled = true; + assert!(!cfg.lists_on_d1sks()); + } + + #[test] + fn listing_can_be_turned_off_alone() { + let mut cfg: Config = toml::from_str("[storage]\nlist_on_d1sks = false\n").unwrap(); + cfg.apply_disk_provider_defaults(); + assert!(cfg.roles.contains(&Role::Storage), "still stores for the network"); + assert!(!cfg.lists_on_d1sks()); + assert!(Config::default().lists_on_d1sks()); + } + #[test] fn game_servers_are_opt_in() { let config = Config::default(); diff --git a/node/crates/c0mpute-core/src/hosting/market.rs b/node/crates/c0mpute-core/src/hosting/market.rs index fa2f2800..d6c16db1 100644 --- a/node/crates/c0mpute-core/src/hosting/market.rs +++ b/node/crates/c0mpute-core/src/hosting/market.rs @@ -17,6 +17,9 @@ use crate::payments::{Micros, parse_usd}; const GIB: f64 = 1024.0 * 1024.0 * 1024.0; +/// A listing whose last signed heartbeat is older than this ranks last. +pub const FRESH_SECS: u64 = 15 * 60; + /// What to ask d1sks for. `--disks auto` is the empty query; otherwise a /// d1sks-style query string such as `country=DE,FI&max_price=0.01&min_gib=5`. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -107,6 +110,39 @@ pub struct HostDisk { /// d1sks standing 0-100, when it has one. pub standing: Option, pub countries: Vec, + /// How d1sks says the disk is reached: `https`, `direct` or `c0mpute` + /// (NAT; reached over connections it opens). `None` from older listings. + pub reachable: Option, + /// Seconds since the disk's last signed heartbeat, when it pushes them. + pub heartbeat_age_secs: Option, + /// Seen announcing itself as a hosting node on the live hosting gossip + /// (set by [`mark_live`], never by d1sks). + pub live: bool, +} + +impl HostDisk { + /// Ranking tier, lower first: live on the hosting gossip; then fresh and + /// publicly reachable (or an origin-polled listing); then fresh behind + /// NAT; then a listing whose heartbeat has gone stale. + pub fn tier(&self) -> u8 { + if self.live { + 0 + } else if self.heartbeat_age_secs.is_some_and(|a| a > FRESH_SECS) { + 3 + } else if self.reachable.as_deref() == Some("c0mpute") { + 2 + } else { + 1 + } + } +} + +/// Mark the disks whose peer ids announced themselves as hosting nodes on +/// the live hosting gossip (`Hosting::live_hosts`). +pub fn mark_live(disks: &mut [HostDisk], live: &[String]) { + for d in disks.iter_mut() { + d.live = live.iter().any(|p| p == &d.peer_id); + } } fn usd_micros(v: &Value) -> Option { @@ -173,14 +209,22 @@ pub fn parse_disks(body: &Value) -> Vec { free_bytes: (free_gib.max(0.0) * GIB) as u64, standing: row["standing"].as_f64().or_else(|| data["record"]["standing"].as_f64()), countries, + reachable: row["reachable"] + .as_str() + .or_else(|| c["reachable"].as_str()) + .filter(|r| matches!(*r, "https" | "direct" | "c0mpute")) + .map(str::to_string), + heartbeat_age_secs: row["heartbeat_age_s"].as_f64().filter(|a| *a >= 0.0).map(|a| a as u64), + live: false, }); } out } -/// Disks that fit the query and the site, best first: cheapest, then -/// highest standing (unrated last), then most free space, then peer id, so -/// the same listing always gives the same order. +/// Disks that fit the query and the site, best first: by [`HostDisk::tier`] +/// (live on gossip, fresh and reachable, fresh behind NAT, stale), then +/// cheapest, then highest standing (unrated last), then most free space, +/// then peer id, so the same listing always gives the same order. pub fn rank(disks: &[HostDisk], q: &DiskQuery, site_bytes: u64) -> Vec { let mut v: Vec = disks .iter() @@ -192,8 +236,9 @@ pub fn rank(disks: &[HostDisk], q: &DiskQuery, site_bytes: u64) -> Vec .cloned() .collect(); v.sort_by(|a, b| { - a.price - .cmp(&b.price) + a.tier() + .cmp(&b.tier()) + .then(a.price.cmp(&b.price)) .then_with(|| { let (x, y) = (a.standing.unwrap_or(-1.0), b.standing.unwrap_or(-1.0)); y.total_cmp(&x) @@ -356,6 +401,29 @@ mod tests { assert_eq!(order, vec!["a", "d"]); } + #[test] + fn live_and_fresh_reachable_disks_rank_first() { + let mut body = fixture(); + let rows = body["disks"].as_array_mut().unwrap(); + // a: cheapest but its heartbeat went stale; b: behind NAT, fresh; + // c: fresh and directly reachable; d: priciest but live on gossip. + rows[0]["heartbeat_age_s"] = serde_json::json!(3600); + rows[0]["reachable"] = serde_json::json!("direct"); + rows[1]["heartbeat_age_s"] = serde_json::json!(60); + rows[1]["reachable"] = serde_json::json!("c0mpute"); + rows[2]["heartbeat_age_s"] = serde_json::json!(30); + rows[2]["reachable"] = serde_json::json!("direct"); + let mut disks = parse_disks(&body); + assert_eq!(disks[1].reachable.as_deref(), Some("c0mpute")); + assert_eq!(disks[0].heartbeat_age_secs, Some(3600)); + mark_live(&mut disks, &[pid(4)]); + let order: Vec = rank(&disks, &DiskQuery::default(), 1024) + .into_iter() + .map(|d| d.name) + .collect(); + assert_eq!(order, vec!["d", "i", "c", "b", "a"]); + } + #[test] fn choosing_agrees_on_the_highest_ask() { let disks = parse_disks(&fixture()); diff --git a/node/crates/c0mpute-core/src/hosting/mod.rs b/node/crates/c0mpute-core/src/hosting/mod.rs index 1767b00a..c186ff4f 100644 --- a/node/crates/c0mpute-core/src/hosting/mod.rs +++ b/node/crates/c0mpute-core/src/hosting/mod.rs @@ -765,6 +765,27 @@ impl Hosting { self.0.map.read().await.holders(site_id) } + /// Hosting peers heard announcing themselves on the gossip within the + /// holder TTL, sorted (this node included when it hosts). + pub async fn live_hosts(&self) -> Vec { + let mut map = self.0.map.write().await; + map.prune(Instant::now(), self.0.timing.holder_ttl); + map.live_peers() + } + + /// Listen to the hosting gossip until every peer in `want` has announced + /// itself as a hosting node, or `max` passes. Returns the live hosts. + pub async fn wait_for_hosts(&self, want: &[String], max: Duration) -> Vec { + let started = Instant::now(); + loop { + let live = self.live_hosts().await; + if want.iter().all(|w| live.contains(w)) || started.elapsed() >= max { + return live; + } + tokio::time::sleep(Duration::from_secs(3)).await; + } + } + /// Every known site: record plus live copy count. pub async fn sites(&self) -> Vec<(SiteRecord, usize)> { let map = self.0.map.read().await; @@ -1307,11 +1328,29 @@ pub struct MarketOptions { pub d1sks_url: String, } -/// Pick preferred hosts from d1sks for a site of `size` bytes. `None` (with -/// a note on stderr) when d1sks is unreachable or nothing matches: the site -/// then publishes as a plain free site, which the network still keeps. -pub async fn pick_terms(m: &MarketOptions, size: u64) -> Option { - let disks = match market::query(&m.d1sks_url, &m.query).await { +/// How long a publisher listens to the hosting gossip to see which d1sks +/// disks are live hosts (a hosting node announces once a minute). +pub const GOSSIP_CHECK: Duration = Duration::from_secs(70); + +/// The d1sks hosting disks for `m`, checked against the live hosting gossip +/// when `hosting` is given: disks that announced themselves are marked live +/// and rank first. `Err` when d1sks cannot be read. +pub async fn market_disks(m: &MarketOptions, hosting: Option<&Hosting>) -> Result> { + let mut disks = market::query(&m.d1sks_url, &m.query).await?; + if let Some(h) = hosting { + let want: Vec = disks.iter().map(|d| d.peer_id.clone()).collect(); + let live = h.wait_for_hosts(&want, GOSSIP_CHECK).await; + market::mark_live(&mut disks, &live); + } + Ok(disks) +} + +/// Pick preferred hosts from d1sks for a site of `size` bytes, preferring +/// disks seen live on the hosting gossip. `None` (with a note on stderr) +/// when d1sks is unreachable or nothing matches: the site then publishes as +/// a plain free site, which the network still keeps. +pub async fn pick_terms(m: &MarketOptions, size: u64, hosting: Option<&Hosting>) -> Option { + let disks = match market_disks(m, hosting).await { Ok(d) => d, Err(e) => { eprintln!("d1sks: {e}; publishing without preferred hosts (the network still keeps 3 free copies)"); @@ -1328,10 +1367,11 @@ pub async fn pick_terms(m: &MarketOptions, size: u64) -> Option pick_terms(m, built.manifest.total_bytes()).await, + Some(m) => pick_terms(m, built.manifest.total_bytes(), Some(&hosting)).await, None => None, }; let record = SiteRecord::sign_with_terms( @@ -1382,7 +1425,6 @@ pub async fn publish_dir_with( domains, terms, )?; - let hosting = Hosting::start(supervisor.libp2p.clone(), supervisor.store.clone(), Options::default()).await?; // Let the swarm reach peers before the first gossip. tokio::time::sleep(Duration::from_secs(5)).await; hosting.publish_site(&built, record.clone()).await?; diff --git a/node/crates/c0mpute-core/src/opendisk.rs b/node/crates/c0mpute-core/src/opendisk.rs index 487b24db..70ca593f 100644 --- a/node/crates/c0mpute-core/src/opendisk.rs +++ b/node/crates/c0mpute-core/src/opendisk.rs @@ -1,11 +1,14 @@ //! OpenDisk: every storage node describes its own disk at //! `/.well-known/opendisk.json` on its gateway, and lists itself on an OpenDisk -//! marketplace (d1sks) when it has a public https origin. +//! marketplace (d1sks) with a signed heartbeat, NAT or not. //! //! There is no central registry in this path. A node states its own name, -//! identity key, free space and price; the marketplace only reads what the -//! node serves, and re-reads it every hour. A node behind NAT still serves the -//! descriptor locally, it just has no origin a marketplace can reach yet. +//! identity key, free space and price, signs that with its libp2p (ed25519) +//! identity, and pushes it to d1sks every [`HEARTBEAT_EVERY`]. d1sks checks +//! that the key is the peer id and that the signature holds, and lets the +//! listing expire [`LISTING_TTL`] after the last heartbeat. A node with a +//! public https origin says so in the same message; one behind NAT is listed +//! as reachable through c0mpute (holders and gateways talk over libp2p). use std::path::Path; use std::time::Duration; @@ -25,9 +28,21 @@ pub const DEFAULT_D1SKS_URL: &str = "https://d1sks.pwa.pit.world"; /// The descriptor path the OpenDisk spec fixes. pub const WELL_KNOWN_PATH: &str = "/.well-known/opendisk.json"; -/// How often a node re-submits itself, so a marketplace that lost it (or a -/// node whose origin changed) catches up without anyone stepping in. -pub const REREGISTER_EVERY: Duration = Duration::from_secs(24 * 60 * 60); +/// How often a node re-sends its signed listing. +pub const HEARTBEAT_EVERY: Duration = Duration::from_secs(5 * 60); + +/// How long a listing stays up after its last heartbeat (three missed beats, +/// plus slack). +pub const LISTING_TTL: Duration = Duration::from_secs(20 * 60); + +/// Where d1sks takes signed listings. +pub const ANNOUNCE_PATH: &str = "/api/v1/disks/announce"; + +/// The `type` of a signed listing payload. +pub const ANNOUNCE_TYPE: &str = "c0mpute.disk.announce/1"; + +/// Multiaddrs sent per listing. +pub const MAX_LISTED_ADDRS: usize = 8; const GIB: f64 = 1024.0 * 1024.0 * 1024.0; @@ -209,52 +224,181 @@ pub fn public_origin(config: &Config) -> Option { Some(url.origin().ascii_serialization()) } -/// Submit `origin` to the marketplace. Returns the marketplace's answer. -pub async fn register(d1sks_url: &str, origin: &str) -> anyhow::Result { +/// How a marketplace (or a publisher) can reach this disk: `https` (a public +/// https origin), `direct` (a public IP multiaddr), or `c0mpute` (only over +/// connections the node opens itself; still a full holder, since hosting and +/// storage move over libp2p). +pub fn reachability(origin: Option<&str>, addrs: &[String]) -> &'static str { + if origin.is_some() { + "https" + } else if addrs.iter().any(|a| is_public_multiaddr(a)) { + "direct" + } else { + "c0mpute" + } +} + +/// A multiaddr on a public IP (or a DNS name). +pub fn is_public_multiaddr(addr: &str) -> bool { + use c0mpute_net::Protocol; + let Ok(ma) = addr.parse::() else { + return false; + }; + ma.iter().any(|p| match p { + Protocol::Ip4(ip) => { + let o = ip.octets(); + !(ip.is_private() + || ip.is_loopback() + || ip.is_link_local() + || ip.is_unspecified() + || ip.is_broadcast() + || ip.is_documentation() + || (o[0] == 100 && (o[1] & 0xc0) == 64)) + } + Protocol::Ip6(ip) => { + let seg = ip.segments(); + !(ip.is_loopback() + || ip.is_unspecified() + || (seg[0] & 0xfe00) == 0xfc00 + || (seg[0] & 0xffc0) == 0xfe80) + } + Protocol::Dns(_) | Protocol::Dns4(_) | Protocol::Dns6(_) => true, + _ => false, + }) +} + +/// The addresses worth listing: no wildcard binds, at most [`MAX_LISTED_ADDRS`], +/// public ones first. +pub fn listable_addrs(addrs: Vec) -> Vec { + let mut v: Vec = addrs + .into_iter() + .filter(|a| !a.contains("/ip4/0.0.0.0") && !a.contains("/ip6/::/") && a.len() <= 200) + .collect(); + v.sort_by_key(|a| !is_public_multiaddr(a)); + v.dedup(); + v.truncate(MAX_LISTED_ADDRS); + v +} + +/// The signed listing d1sks takes at [`ANNOUNCE_PATH`]: +/// `{ "payload": "", "signature": "" }`. +/// +/// The payload binds the marketplace host (so a listing cannot be replayed to +/// another marketplace), the time (d1sks refuses old or reused ones), the +/// peer id and its key, and the OpenDisk descriptor with the c0mpute block +/// (roles, version, addrs, reachability). +pub fn signed_listing( + config: &Config, + keypair: &c0mpute_net::libp2p::identity::Keypair, + market_url: &str, + origin: Option<&str>, + addrs: Vec, + cap: Capacity, + unix_secs: u64, +) -> anyhow::Result { + let peer = keypair.public().to_peer_id(); + let key_hex = c0mpute_net::identity::ed25519_public_hex(&peer) + .ok_or_else(|| anyhow::anyhow!("this identity is not an ed25519 key"))?; + let peer_id = peer.to_base58(); + let addrs = listable_addrs(addrs); + let mut doc = descriptor(config, &peer_id, Some(&key_hex), origin, cap); + doc["c0mpute"]["addrs"] = json!(addrs); + doc["c0mpute"]["reachable"] = json!(reachability(origin, &addrs)); + if let Some(c) = config.country() { + doc["location"] = json!({ "countries": [c] }); + } + let market_host = reqwest::Url::parse(market_url) + .ok() + .and_then(|u| u.host_str().map(str::to_string)) + .unwrap_or_default(); + let payload = json!({ + "type": ANNOUNCE_TYPE, + "marketplace": market_host, + "peer_id": peer_id, + "key": format!("ed25519:{key_hex}"), + "ts": unix_secs, + "expires_in": LISTING_TTL.as_secs(), + "origin": origin, + "descriptor": doc, + }); + let text = serde_json::to_string(&payload)?; + let sig = keypair + .sign(text.as_bytes()) + .map_err(|e| anyhow::anyhow!("sign listing: {e}"))?; + Ok(json!({ "payload": text, "signature": hex::encode(sig) })) +} + +/// Push one signed listing to the marketplace. Returns its answer. +pub async fn announce(market_url: &str, body: &Value) -> anyhow::Result { let client = reqwest::Client::builder() .timeout(Duration::from_secs(20)) .user_agent(concat!("c0mpute/", env!("CARGO_PKG_VERSION"))) .build()?; let res = client - .post(format!("{}/api/v1/disks", d1sks_url.trim_end_matches('/'))) - .json(&json!({ "url": origin })) + .post(format!("{}{ANNOUNCE_PATH}", market_url.trim_end_matches('/'))) + .json(body) .send() .await?; let status = res.status(); - let body: Value = res.json().await.unwrap_or(Value::Null); + let answer: Value = res.json().await.unwrap_or(Value::Null); if !status.is_success() { - anyhow::bail!("d1sks answered {status}: {body}"); + anyhow::bail!("d1sks answered {status}: {answer}"); } - Ok(body) + Ok(answer) } -/// Keep this node listed: register now, then once a day. A no-op when the -/// operator turned OpenDisk off or the gateway has no public https origin. -pub fn spawn_self_listing(config: &Config) { - if !config.storage.opendisk { - return; - } - if config.private.enabled { - info!("opendisk: private mode, not listing this disk on d1sks"); +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Keep this disk listed on d1sks: a signed heartbeat now and every +/// [`HEARTBEAT_EVERY`], NAT or not. A no-op when the node does not list +/// (`Config::lists_on_d1sks`: storage off, OpenDisk off, `list_on_d1sks = +/// false`, `C0MPUTE_D1SKS=0`, or private mode). +pub fn spawn_listing(config: &Config, net: std::sync::Arc) { + if !config.lists_on_d1sks() { + info!("opendisk: not listing this disk on d1sks (storage, listing or private mode says no)"); return; } - let Some(origin) = public_origin(config) else { - info!( - "opendisk: descriptor served at {WELL_KNOWN_PATH}; not listing on d1sks \ - (set gateway.announce to this node's public https origin to list it)" - ); - return; + let identity_dir = crate::config::config_dir().unwrap_or_else(|| std::path::PathBuf::from(".")); + let keypair = match c0mpute_net::identity::load_or_create(&identity_dir) { + Ok(k) if k.public().to_peer_id() == net.peer_id() => k, + Ok(_) => { + warn!("opendisk: the identity on disk is not this node's peer id; not listing"); + return; + } + Err(e) => { + warn!(err = %e, "opendisk: no identity to sign the d1sks listing with"); + return; + } }; - let market = config.storage.d1sks_url.clone(); + let config = config.clone(); tokio::spawn(async move { - // Let the gateway bind before the marketplace fetches from it. - tokio::time::sleep(Duration::from_secs(15)).await; + // Let the swarm learn its addresses before the first listing. + tokio::time::sleep(Duration::from_secs(20)).await; + let market = config.storage.d1sks_url.clone(); + let origin = public_origin(&config); + let mut logged_ok = false; loop { - match register(&market, &origin).await { - Ok(_) => info!(%origin, %market, "opendisk: listed on d1sks"), - Err(e) => warn!(%origin, %market, err = %e, "opendisk: d1sks listing failed"), + let addrs: Vec = net.advertised_addrs().await.into_iter().map(|a| a.to_string()).collect(); + let cap = capacity(&config.storage, 0); + match signed_listing(&config, &keypair, &market, origin.as_deref(), addrs, cap, unix_now()) { + Ok(body) => match announce(&market, &body).await { + Ok(_) if !logged_ok => { + info!(%market, "opendisk: listed on d1sks (signed heartbeat every {}s)", HEARTBEAT_EVERY.as_secs()); + logged_ok = true; + } + Ok(_) => tracing::debug!(%market, "opendisk: d1sks heartbeat"), + Err(e) => warn!(%market, err = %e, "opendisk: d1sks listing failed"), + }, + Err(e) => warn!(err = %e, "opendisk: could not build the d1sks listing"), } - tokio::time::sleep(REREGISTER_EVERY).await; + // A little jitter so a fleet restarted together does not beat in step. + let jitter = Duration::from_secs(unix_now() % 30); + tokio::time::sleep(HEARTBEAT_EVERY + jitter).await; } }); } @@ -355,6 +499,48 @@ mod tests { assert_eq!(public_origin(&cfg).as_deref(), Some("https://disk.example")); } + #[test] + fn signed_listing_verifies_with_the_peer_key() { + let kp = c0mpute_net::libp2p::identity::Keypair::generate_ed25519(); + let cfg = Config::default(); + let body = signed_listing( + &cfg, + &kp, + "https://d1sks.example/", + None, + vec![ + "/ip4/0.0.0.0/tcp/1".into(), + "/ip4/192.168.1.5/tcp/4001".into(), + ], + Capacity { free_bytes: 10 * G, total_bytes: 20 * G }, + 1_800_000_000, + ) + .unwrap(); + let text = body["payload"].as_str().unwrap(); + let sig = hex::decode(body["signature"].as_str().unwrap()).unwrap(); + assert!(kp.public().verify(text.as_bytes(), &sig)); + let p: Value = serde_json::from_str(text).unwrap(); + assert_eq!(p["type"], ANNOUNCE_TYPE); + assert_eq!(p["marketplace"], "d1sks.example"); + assert_eq!(p["peer_id"], kp.public().to_peer_id().to_base58()); + assert_eq!(p["ts"], 1_800_000_000u64); + assert_eq!(p["descriptor"]["key"], p["key"]); + assert_eq!(p["descriptor"]["c0mpute"]["reachable"], "c0mpute"); + assert_eq!(p["descriptor"]["c0mpute"]["addrs"], json!(["/ip4/192.168.1.5/tcp/4001"])); + assert_eq!(p["descriptor"]["capacity"]["free_gib"], 10.0); + } + + #[test] + fn reachability_reads_origin_then_public_addrs() { + let lan = vec!["/ip4/10.0.0.2/tcp/1".to_string(), "/ip4/100.64.1.1/tcp/1".to_string()]; + assert_eq!(reachability(None, &lan), "c0mpute"); + assert_eq!(reachability(Some("https://x.example"), &lan), "https"); + assert_eq!(reachability(None, &["/ip4/203.0.113.9/tcp/1".to_string()]), "c0mpute", "documentation range"); + assert_eq!(reachability(None, &["/ip4/5.9.10.11/tcp/46341".to_string()]), "direct"); + assert_eq!(reachability(None, &["/ip6/2a01:4f8::1/tcp/1".to_string()]), "direct"); + assert_eq!(reachability(None, &["/ip6/fd00::1/tcp/1".to_string()]), "c0mpute"); + } + #[test] fn civil_dates() { assert_eq!(civil_from_days(0), (1970, 1, 1)); diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index 974eac6f..b7587648 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -618,6 +618,10 @@ impl Supervisor { None }; + // Every storage node lists its disk on d1sks with a signed heartbeat, + // gateway or not, NAT or not (opt-outs in `Config::lists_on_d1sks`). + crate::opendisk::spawn_listing(&self.config, self.libp2p.clone()); + if self.config.roles.contains(&c0mpute_proto::Role::Gateway) { let bind: std::net::SocketAddr = self.config.gateway.bind.parse()?; let state = c0mpute_gateway::GatewayState { @@ -871,7 +875,6 @@ impl Supervisor { } }), ); - crate::opendisk::spawn_self_listing(&self.config); } app } diff --git a/package.json b/package.json index 75c0ea23..ba443500 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.2.94", + "version": "0.2.95", "private": true, "workspaces": [ "apps/*", diff --git a/packages/shared/package.json b/packages/shared/package.json index 1896d19a..435c866e 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.2.94", + "version": "0.2.95", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 0d04cf10..82210029 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index 22658ab0..cc3c708f 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -38,7 +38,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "partial" diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index aabe3cca..737c7ab2 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index 95bebbff..5850e9cd 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "data" status = "available" diff --git a/plugins/crowdgpt/module.toml b/plugins/crowdgpt/module.toml index c698cdcd..840295c9 100644 --- a/plugins/crowdgpt/module.toml +++ b/plugins/crowdgpt/module.toml @@ -43,7 +43,7 @@ [module] id = "crowdgpt" name = "CrowdGPT: donate idle compute" -version = "0.2.94" +version = "0.2.95" kind = "donation" category = "ai" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index 7f6ffb6f..7d72c7c2 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -28,7 +28,7 @@ [module] id = "ctv-ads" name = "CTV Ads — VAST and VMAP from every gateway" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "partial" diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index ec4f8d85..8cfae230 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index 4798103e..b2cbae3f 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -19,7 +19,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index fb1d3d45..1128a3ac 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -55,7 +55,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index 83842229..928e7e80 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -30,7 +30,7 @@ [module] id = "hosting" name = "Hosting — p2p static sites" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "data" status = "available" diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index b173434a..6192c254 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -22,7 +22,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index 899c731f..945d112d 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -34,7 +34,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "media" status = "available" diff --git a/plugins/music/module.toml b/plugins/music/module.toml index 367b4f6b..ef698e30 100644 --- a/plugins/music/module.toml +++ b/plugins/music/module.toml @@ -15,7 +15,7 @@ [module] id = "music" name = "Music — song generation (ACE-Step 1.5)" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index 714e5b85..27d35cb9 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -19,7 +19,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index 7f1d4887..557d4678 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -31,7 +31,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — dynamic ad insertion" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "partial" diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index 8f7eb36d..03d130cc 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "media" status = "available" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index a90c94b5..6c15936d 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "media" status = "available" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index 1ff82704..72f5bb6d 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -33,7 +33,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "media" status = "available" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index 932d1be2..c29954be 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -21,7 +21,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "applications" status = "available" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index f7ff4b85..78796f2c 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.2.94" +version = "0.2.95" kind = "service" category = "data" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 6d86293b..37ee85b2 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -55,7 +55,7 @@ [module] id = "train" name = "Fine-tune — LoRA training" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index 25d81856..43f67c82 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index 76ae0cce..61baaf6e 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index c84453ee..3fc9075f 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -65,7 +65,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, Wan 2.1 1.3B on 8 GiB or CPU, LTX-2.3 with audio)" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 04e7a849..f576bbd9 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index af00d9ac..9cfd6e54 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index 7c28fa90..e43fa393 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.2.94" +version = "0.2.95" kind = "workload" category = "infrastructure" status = "available" diff --git a/scripts/install.sh b/scripts/install.sh index 3d695abc..edd2ec41 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -990,6 +990,15 @@ have the model). Check it with: c0mpute worker status # running? + which models are eligible to distribute c0mpute -v # versions +Disk provider (on by default): every node stores shards and holds sites for the +network, and lists its disk on d1sks (https://d1sks.pwa.pit.world) with a signed +heartbeat, behind NAT or not. Cap it with [storage] cap_bytes, or opt out in +~/.config/c0mpute/config.toml: + [storage] enabled = false # no storage role, not listed (or --no-storage, C0MPUTE_STORAGE=0) + [storage] list_on_d1sks = false # store for the network, stay off d1sks (or C0MPUTE_D1SKS=0) + [hosting] enabled = false # hold no sites (or --no-hosting, C0MPUTE_HOSTING=0) +Private mode never lists. + Optional — claim this node under your account (for the dashboard + earnings; NOT required to serve): c0mpute login && infernet pubkey link