diff --git a/CHANGELOG.md b/CHANGELOG.md index b35f216..6f01f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,11 @@ All notable changes to RustFS Operator are documented in this file. The format i - Declarative Tenant bucket lifecycle rules with drift-aware ownership and explicit removal. - Bucket lifecycle `transition` and `noncurrentVersionTransition` actions that move objects to a remote tier registered in RustFS. +- Tenant `spec.tiers` for declarative S3, RustFS, and MinIO remote tiers with Secret-sourced + credentials, rotation on Secret updates, and conflict reporting for immutable tier settings. - Declarative Tenant bucket versioning and Object Lock default retention with drift-aware ownership. +- Declarative default bucket encryption (`SSE-S3` or `SSE-KMS`) with drift-aware ownership, + explicit removal, and validation of the Tenant SSE-S3 master key or KMS default key. - Helm chart `cosiDriver.enabled` to deploy the RustFS COSI (`rustfs.objectstorage.k8s.io`) driver alongside the upstream provisioner sidecar, with dedicated ServiceAccount/RBAC. - Seven built-in RustFS Grafana dashboards, published by default as sidecar-discoverable diff --git a/console-web/app/(dashboard)/tenants/[namespace]/[name]/tenant-detail-client.tsx b/console-web/app/(dashboard)/tenants/[namespace]/[name]/tenant-detail-client.tsx index 0850ab5..3eca432 100644 --- a/console-web/app/(dashboard)/tenants/[namespace]/[name]/tenant-detail-client.tsx +++ b/console-web/app/(dashboard)/tenants/[namespace]/[name]/tenant-detail-client.tsx @@ -104,6 +104,7 @@ function provisioningGroups(tenant: TenantDetailsResponse) { return [ { type: "Policy", items: provisioning.policies ?? [] }, { type: "User", items: provisioning.users ?? [] }, + { type: "Tier", items: provisioning.tiers ?? [] }, { type: "Bucket", items: provisioning.buckets ?? [] }, ].filter((group) => group.items.length > 0) } diff --git a/console-web/types/api.ts b/console-web/types/api.ts index 78a860f..4866b9a 100644 --- a/console-web/types/api.ts +++ b/console-web/types/api.ts @@ -130,16 +130,37 @@ export interface ProvisioningBucket { objectLockConfiguration?: BucketObjectLockConfiguration anonymous?: "Private" | "Download" | "Upload" | "Public" policy?: PolicyDocumentSource + encryption?: BucketEncryptionSpec lifecycle?: BucketLifecycleSpec deletionPolicy?: ProvisioningDeletionPolicy } +export interface ProvisioningTier { + name: string + type: "S3" | "RustFS" | "MinIO" + endpoint: string + bucket: string + prefix?: string + region?: string + credsSecret: { + name: string + } + force?: boolean + deletionPolicy?: ProvisioningDeletionPolicy +} + export interface BucketObjectLockConfiguration { state?: "Present" | "Absent" mode?: "Governance" | "Compliance" days?: number } +export interface BucketEncryptionSpec { + state?: "Present" | "Absent" + mode?: "SSE-S3" | "SSE-KMS" + kmsKeyId?: string +} + export interface BucketLifecycleSpec { state: "Present" | "Absent" rules?: BucketLifecycleRule[] @@ -190,6 +211,8 @@ export interface ProvisioningBucketStatus extends ProvisioningItemStatus { versioning?: "Unversioned" | "Enabled" | "Suspended" | null objectLockConfigurationDesiredHash?: string | null objectLockConfigurationLastAppliedHash?: string | null + encryptionDesiredHash?: string | null + encryptionLastAppliedHash?: string | null } export interface ProvisioningStatus { @@ -198,6 +221,7 @@ export interface ProvisioningStatus { policies?: ProvisioningItemStatus[] users?: ProvisioningItemStatus[] buckets?: ProvisioningBucketStatus[] + tiers?: ProvisioningItemStatus[] } export interface CreatePoolRequest { @@ -225,6 +249,7 @@ export interface CreateTenantRequest { policies?: ProvisioningPolicy[] users?: ProvisioningUser[] buckets?: ProvisioningBucket[] + tiers?: ProvisioningTier[] security_context?: CreateSecurityContextRequest } @@ -238,6 +263,7 @@ export interface UpdateTenantRequest { policies?: ProvisioningPolicy[] users?: ProvisioningUser[] buckets?: ProvisioningBucket[] + tiers?: ProvisioningTier[] logging?: { logType: string volumeSize?: string diff --git a/crates/rustfs-admin/src/admin_ops.rs b/crates/rustfs-admin/src/admin_ops.rs index 1ac7079..0982d39 100644 --- a/crates/rustfs-admin/src/admin_ops.rs +++ b/crates/rustfs-admin/src/admin_ops.rs @@ -20,13 +20,13 @@ use std::collections::{BTreeMap, BTreeSet}; use super::helpers::{ body_mentions_not_found, build_canonical_query, extract_canned_policy_document, - is_absent_resource, parse_tier_names, + is_absent_resource, parse_tiers, uri_encode, }; use super::{ ADD_CANNED_POLICY_PATH, ADD_USER_PATH, ADMIN_SIGNING_SERVICE, INFO_CANNED_POLICY_PATH, - JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, LIST_TIERS_PATH, REMOVE_USER_PATH, - RustfsAdminClient, RustfsClientError, RustfsServerInfo, RustfsServerInfoResponse, - RustfsUserInfo, SERVER_INFO_PATH, SET_POLICY_PATH, USER_INFO_PATH, + JSON_CONTENT_TYPE, LIST_CANNED_POLICIES_PATH, REMOVE_USER_PATH, RustfsAdminClient, + RustfsClientError, RustfsServerInfo, RustfsServerInfoResponse, RustfsTier, RustfsTierConfig, + RustfsUserInfo, SERVER_INFO_PATH, SET_POLICY_PATH, TIER_PATH, USER_INFO_PATH, }; use reqwest::StatusCode; use serde_json::Value; @@ -175,13 +175,94 @@ impl RustfsAdminClient { .collect() } - /// Names of the remote tiers registered in RustFS. Only names are kept; the response also - /// carries tier endpoints and credentials, which are dropped while parsing. + /// Names of the remote tiers registered in RustFS. pub async fn list_tier_names(&self) -> Result, RustfsClientError> { + Ok(self + .list_tiers() + .await? + .into_iter() + .map(|tier| tier.config.name) + .collect()) + } + + /// Remote tiers registered in RustFS. Secret keys in the response are dropped while parsing. + pub async fn list_tiers(&self) -> Result, RustfsClientError> { let body = self - .send_admin_request("GET", LIST_TIERS_PATH, "", "", None) + .send_admin_request("GET", TIER_PATH, "", "", None) .await?; - parse_tier_names(&body) + parse_tiers(&body) + } + + /// Register an S3-style remote tier. Without `force`, RustFS rejects a remote prefix that + /// already holds objects with HTTP 409. + pub async fn add_tier( + &self, + config: &RustfsTierConfig, + access_key: &str, + secret_key: &str, + force: bool, + ) -> Result<(), RustfsClientError> { + if config.name.trim().is_empty() || config.tier_type.trim().is_empty() { + return Err(RustfsClientError::InvalidTierName); + } + if access_key.trim().is_empty() { + return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); + } + if secret_key.is_empty() { + return Err(RustfsClientError::EmptyCredentialValue { key: "secretkey" }); + } + + let mut body = serde_json::Map::new(); + body.insert("type".to_string(), Value::from(config.tier_type.as_str())); + body.insert("name".to_string(), Value::from(config.name.as_str())); + body.insert( + config.tier_type.clone(), + serde_json::json!({ + "name": config.name, + "endpoint": config.endpoint, + "accessKey": access_key, + "secretKey": secret_key, + "bucket": config.bucket, + "prefix": config.prefix, + "region": config.region, + }), + ); + let body = Value::Object(body).to_string(); + let query = build_canonical_query(&[("force", if force { "true" } else { "false" })]); + + self.send_admin_request("PUT", TIER_PATH, &query, &body, Some(JSON_CONTENT_TYPE)) + .await + .map(|_| ()) + } + + /// Replace the credentials of a registered tier. RustFS only allows credentials to change + /// on an existing tier; endpoint, bucket and prefix are fixed at registration. + pub async fn edit_tier_credentials( + &self, + tier_name: &str, + access_key: &str, + secret_key: &str, + ) -> Result<(), RustfsClientError> { + if tier_name.trim().is_empty() { + return Err(RustfsClientError::InvalidTierName); + } + if access_key.trim().is_empty() { + return Err(RustfsClientError::InvalidCredentialValue { key: "accesskey" }); + } + if secret_key.is_empty() { + return Err(RustfsClientError::EmptyCredentialValue { key: "secretkey" }); + } + + let body = serde_json::json!({ + "access": access_key, + "secret": secret_key, + }) + .to_string(); + let path = format!("{TIER_PATH}/{}", uri_encode(tier_name)); + + self.send_admin_request("POST", &path, "", &body, Some(JSON_CONTENT_TYPE)) + .await + .map(|_| ()) } pub async fn server_info(&self) -> Result { diff --git a/crates/rustfs-admin/src/helpers.rs b/crates/rustfs-admin/src/helpers.rs index fb5b008..2e740fa 100644 --- a/crates/rustfs-admin/src/helpers.rs +++ b/crates/rustfs-admin/src/helpers.rs @@ -13,7 +13,6 @@ // limitations under the License. //! Internal helper duties: shared credential parsing, signature/hash utilities, and parsers. -use std::collections::BTreeSet; use hmac::{Hmac, Mac}; use reqwest::StatusCode; @@ -21,7 +20,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use url::form_urlencoded; -use super::RustfsClientError; +use super::{RustfsClientError, RustfsTier, RustfsTierConfig}; /// Encode an `application/x-www-form-urlencoded` request body. pub(super) fn build_form_body(params: &[(&str, &str)]) -> String { @@ -54,7 +53,7 @@ pub(super) fn build_canonical_query(params: &[(&str, &str)]) -> String { .join("&") } -fn uri_encode(value: &str) -> String { +pub(super) fn uri_encode(value: &str) -> String { const HEX: &[u8; 16] = b"0123456789ABCDEF"; let mut encoded = String::with_capacity(value.len()); @@ -102,6 +101,7 @@ pub(super) fn body_mentions_not_found(body: &str) -> bool { || body.contains("nosuchpolicy") || body.contains("no such policy") || body.contains("objectlockconfigurationnotfound") + || body.contains("serversideencryptionconfigurationnotfound") || body.contains("nosuchbucketpolicy") || body.contains("no such bucket policy") || body.contains("not found") @@ -153,12 +153,13 @@ pub(super) fn extract_canned_policy_document(body: &str) -> Result", "": {"name": ..}}` -/// entries where the backend object is keyed by the tier type. -pub(super) fn parse_tier_names(body: &str) -> Result, RustfsClientError> { +/// entries where the backend object is keyed by the tier type. `secretKey` is dropped and +/// never leaves this function. +pub(super) fn parse_tiers(body: &str) -> Result, RustfsClientError> { let tiers = match serde_json::from_str::(body) .map_err(|_| RustfsClientError::ParseResponseFailed)? { - Value::Null => return Ok(BTreeSet::new()), + Value::Null => return Ok(Vec::new()), Value::Array(tiers) => tiers, _ => return Err(RustfsClientError::ParseResponseFailed), }; @@ -169,12 +170,31 @@ pub(super) fn parse_tier_names(body: &str) -> Result, RustfsCli .get("type") .and_then(Value::as_str) .ok_or(RustfsClientError::ParseResponseFailed)?; - tier.get(tier_type) - .and_then(|backend| backend.get("name")) - .and_then(Value::as_str) - .filter(|name| !name.is_empty()) - .map(ToOwned::to_owned) - .ok_or(RustfsClientError::ParseResponseFailed) + let backend = tier + .get(tier_type) + .ok_or(RustfsClientError::ParseResponseFailed)?; + let field = |key: &str| { + backend + .get(key) + .and_then(Value::as_str) + .unwrap_or_default() + .to_string() + }; + let name = field("name"); + if name.is_empty() { + return Err(RustfsClientError::ParseResponseFailed); + } + Ok(RustfsTier { + config: RustfsTierConfig { + tier_type: tier_type.to_string(), + name, + endpoint: field("endpoint"), + bucket: field("bucket"), + prefix: field("prefix"), + region: field("region"), + }, + access_key: field("accessKey"), + }) }) .collect() } diff --git a/crates/rustfs-admin/src/lib.rs b/crates/rustfs-admin/src/lib.rs index 09a673b..f067484 100644 --- a/crates/rustfs-admin/src/lib.rs +++ b/crates/rustfs-admin/src/lib.rs @@ -33,6 +33,7 @@ mod pool_ops; #[path = "s3_ops.rs"] mod s3_ops; pub use s3_ops::{ + BucketEncryptionAlgorithm, BucketEncryptionConfiguration, BucketLifecycleAbortIncompleteMultipartUpload, BucketLifecycleConfiguration, BucketLifecycleExpiration, BucketLifecycleNoncurrentVersionTransition, BucketLifecycleRule, BucketLifecycleRuleStatus, BucketLifecycleTransition, BucketObjectLockConfiguration, @@ -55,7 +56,7 @@ const LIST_CANNED_POLICIES_PATH: &str = "/rustfs/admin/v3/list-canned-policies"; const ADD_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/add-canned-policy"; const INFO_CANNED_POLICY_PATH: &str = "/rustfs/admin/v3/info-canned-policy"; const SERVER_INFO_PATH: &str = "/rustfs/admin/v3/info"; -const LIST_TIERS_PATH: &str = "/rustfs/admin/v3/tier"; +const TIER_PATH: &str = "/rustfs/admin/v3/tier"; const POOLS_LIST_PATH: &str = "/rustfs/admin/v3/pools/list"; const POOLS_STATUS_PATH: &str = "/rustfs/admin/v3/pools/status"; const POOLS_DECOMMISSION_PATH: &str = "/rustfs/admin/v3/pools/decommission"; @@ -120,6 +121,24 @@ pub struct RustfsUserInfo { pub policy_names: Vec, } +/// Non-secret settings of an S3-style remote tier (`s3`, `rustfs` and `minio` backends). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct RustfsTierConfig { + pub tier_type: String, + pub name: String, + pub endpoint: String, + pub bucket: String, + pub prefix: String, + pub region: String, +} + +/// Remote tier listed by `GET /rustfs/admin/v3/tier`. The secret key is never kept. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RustfsTier { + pub config: RustfsTierConfig, + pub access_key: String, +} + #[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)] pub struct RustfsPoolDecommissionInfo { #[serde(rename = "startTime")] @@ -217,6 +236,7 @@ pub enum RustfsClientError { }, TenantSecretLookupFailed, InvalidPolicyName, + InvalidTierName, InvalidPolicyDocument, TenantTlsRequired, TenantTlsNotReady, @@ -238,6 +258,7 @@ pub enum RustfsClientError { }, InvalidBucketVersioningResponse, InvalidObjectLockConfigurationResponse, + InvalidBucketEncryptionResponse, InvalidLifecycleConfigurationResponse, ParseResponseFailed, SigningFailed, @@ -257,6 +278,7 @@ impl std::fmt::Display for RustfsClientError { write!(f, "failed to load tenant credential secret") } Self::InvalidPolicyName => write!(f, "invalid policy name"), + Self::InvalidTierName => write!(f, "invalid tier name"), Self::InvalidPolicyDocument => write!(f, "failed to parse canned policy response"), Self::TenantTlsRequired => write!(f, "STS requires a TLS-enabled tenant"), Self::TenantTlsNotReady => write!(f, "tenant TLS status is not ready"), @@ -289,6 +311,9 @@ impl std::fmt::Display for RustfsClientError { Self::InvalidObjectLockConfigurationResponse => { write!(f, "invalid bucket Object Lock configuration response") } + Self::InvalidBucketEncryptionResponse => { + write!(f, "invalid bucket encryption configuration response") + } Self::ParseResponseFailed => write!(f, "failed to parse AssumeRole response"), Self::SigningFailed => write!(f, "failed to compute request signature"), } @@ -305,6 +330,11 @@ impl RustfsClientError { matches!(self, Self::UnexpectedStatus { status, .. } if *status == StatusCode::NOT_FOUND) } + /// Whether RustFS answered HTTP 409, e.g. a tier whose remote prefix already holds objects. + pub fn is_conflict(&self) -> bool { + matches!(self, Self::UnexpectedStatus { status, .. } if *status == StatusCode::CONFLICT) + } + /// Network, timeout, and retryable upstream failures. Permanent configuration and /// 4xx semantic errors are not transient. pub fn is_transient(&self) -> bool { @@ -326,6 +356,7 @@ impl RustfsClientError { | Self::EmptyCredentialValue { .. } | Self::InvalidCredentialValue { .. } | Self::InvalidPolicyName + | Self::InvalidTierName | Self::InvalidPolicyDocument | Self::TenantTlsRequired | Self::TenantTlsClientCertificateRequired @@ -335,6 +366,7 @@ impl RustfsClientError { | Self::RequestBuildFailed | Self::InvalidBucketVersioningResponse | Self::InvalidObjectLockConfigurationResponse + | Self::InvalidBucketEncryptionResponse | Self::InvalidLifecycleConfigurationResponse | Self::SigningFailed => false, } @@ -559,7 +591,16 @@ impl RustfsAdminClient { #[cfg(test)] mod tests { - use super::helpers::{build_canonical_query, parse_tier_names}; + use super::helpers::{build_canonical_query, parse_tiers}; + use super::{RustfsTier, RustfsTierConfig}; + use std::collections::BTreeSet; + + fn parse_tier_names(body: &str) -> Result, RustfsClientError> { + Ok(parse_tiers(body)? + .into_iter() + .map(|tier| tier.config.name) + .collect()) + } use super::{RustfsClientError, StatusCode}; #[test] @@ -579,6 +620,44 @@ mod tests { assert!(parse_tier_names("null").expect("null list").is_empty()); } + #[test] + fn tier_settings_are_read_without_the_secret_key() { + let body = r#"[ + {"type":"s3","s3":{"name":"COLD","endpoint":"https://gateway.example.com","accessKey":"AK","secretKey":"REDACTED","bucket":"cold","prefix":"rustfs/","region":"us-1","storageClass":""}}, + {"type":"minio","minio":{"name":"ARCHIVE","endpoint":"https://minio.example.com","accessKey":"AK2","secretKey":"REDACTED","bucket":"archive"}} + ]"#; + + let tiers = parse_tiers(body).expect("tier list parses"); + assert_eq!( + tiers, + vec![ + RustfsTier { + config: RustfsTierConfig { + tier_type: "s3".to_string(), + name: "COLD".to_string(), + endpoint: "https://gateway.example.com".to_string(), + bucket: "cold".to_string(), + prefix: "rustfs/".to_string(), + region: "us-1".to_string(), + }, + access_key: "AK".to_string(), + }, + RustfsTier { + config: RustfsTierConfig { + tier_type: "minio".to_string(), + name: "ARCHIVE".to_string(), + endpoint: "https://minio.example.com".to_string(), + bucket: "archive".to_string(), + prefix: String::new(), + region: String::new(), + }, + access_key: "AK2".to_string(), + }, + ] + ); + assert!(!format!("{tiers:?}").contains("REDACTED")); + } + #[test] fn tier_list_without_a_backend_name_is_rejected() { for body in [ diff --git a/crates/rustfs-admin/src/s3_ops.rs b/crates/rustfs-admin/src/s3_ops.rs index ff43c3e..0ad6ec8 100644 --- a/crates/rustfs-admin/src/s3_ops.rs +++ b/crates/rustfs-admin/src/s3_ops.rs @@ -14,6 +14,7 @@ //! S3 boundary: //! - bucket lifecycle methods (create/lookup features) +//! - default bucket encryption (`?encryption`) //! - request semantics for S3-style object storage operations. use super::helpers::{ @@ -140,6 +141,52 @@ impl BucketObjectLockConfiguration { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BucketEncryptionAlgorithm { + /// SSE-S3 (`AES256`). + Aes256, + /// SSE-KMS (`aws:kms`). + AwsKms, +} + +impl BucketEncryptionAlgorithm { + const fn as_s3_str(self) -> &'static str { + match self { + Self::Aes256 => "AES256", + Self::AwsKms => "aws:kms", + } + } +} + +/// Default bucket encryption (`ServerSideEncryptionConfiguration` with a single rule). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BucketEncryptionConfiguration { + pub algorithm: BucketEncryptionAlgorithm, + pub kms_master_key_id: Option, + pub bucket_key_enabled: bool, +} + +impl BucketEncryptionConfiguration { + pub fn to_xml(&self) -> String { + let mut xml = format!( + "" + ); + xml.push_str(self.algorithm.as_s3_str()); + xml.push_str(""); + if let Some(key_id) = &self.kms_master_key_id { + xml.push_str(""); + xml.push_str(&escape_xml(key_id)); + xml.push_str(""); + } + xml.push_str(""); + if self.bucket_key_enabled { + xml.push_str("true"); + } + xml.push_str(""); + xml + } +} + #[derive(Debug)] struct StrictXmlElement { name: String, @@ -257,6 +304,58 @@ fn parse_bucket_object_lock_xml( }) } +fn parse_bucket_encryption_xml( + xml: &str, +) -> Result { + let invalid = || RustfsClientError::InvalidBucketEncryptionResponse; + let root = parse_strict_configuration_xml(xml, "ServerSideEncryptionConfiguration") + .map_err(|()| invalid())?; + root.validate_children(&["Rule"]).map_err(|()| invalid())?; + let rule = root + .child("Rule") + .map_err(|()| invalid())? + .ok_or_else(invalid)?; + rule.validate_children(&["ApplyServerSideEncryptionByDefault", "BucketKeyEnabled"]) + .map_err(|()| invalid())?; + let by_default = rule + .child("ApplyServerSideEncryptionByDefault") + .map_err(|()| invalid())? + .ok_or_else(invalid)?; + by_default + .validate_children(&["SSEAlgorithm", "KMSMasterKeyID"]) + .map_err(|()| invalid())?; + let algorithm = match by_default + .child("SSEAlgorithm") + .map_err(|()| invalid())? + .ok_or_else(invalid)? + .leaf_text() + .map_err(|()| invalid())? + { + "AES256" => BucketEncryptionAlgorithm::Aes256, + "aws:kms" => BucketEncryptionAlgorithm::AwsKms, + _ => return Err(invalid()), + }; + let kms_master_key_id = match by_default.child("KMSMasterKeyID").map_err(|()| invalid())? { + Some(key_id) if key_id.text.trim().is_empty() && key_id.children.is_empty() => None, + Some(key_id) => Some(key_id.leaf_text().map_err(|()| invalid())?.to_string()), + None => None, + }; + let bucket_key_enabled = match rule.child("BucketKeyEnabled").map_err(|()| invalid())? { + Some(enabled) => match enabled.leaf_text().map_err(|()| invalid())? { + "true" => true, + "false" => false, + _ => return Err(invalid()), + }, + None => false, + }; + + Ok(BucketEncryptionConfiguration { + algorithm, + kms_master_key_id, + bucket_key_enabled, + }) +} + fn parse_positive_i32(value: &str) -> Option { value.parse::().ok().filter(|value| *value > 0) } @@ -1034,6 +1133,139 @@ impl RustfsAdminClient { .map(|configuration| configuration.is_some()) } + pub async fn get_bucket_encryption( + &self, + bucket: &str, + ) -> Result, RustfsClientError> { + if bucket.trim().is_empty() { + return Err(RustfsClientError::RequestBuildFailed); + } + + let path = format!("/{bucket}"); + let query = build_canonical_query(&[("encryption", "")]); + let signed = self.sign_request("GET", &path, &query, "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + let response = self + .http_client + .get(format!( + "{}{}?{query}", + self.base_url.trim_end_matches('/'), + path + )) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + let body = read_bucket_configuration_response(response, || { + RustfsClientError::InvalidBucketEncryptionResponse + }) + .await?; + return parse_bucket_encryption_xml(&body).map(Some); + } + + // RustFS answers a missing configuration with 400 ServerSideEncryptionConfigurationNotFoundError. + let status = response.status(); + let (body, truncated) = RustfsClientError::limited_response_body(response).await; + if is_absent_resource(status, &body) { + return Ok(None); + } + Err(RustfsClientError::unexpected_status_with_limited_body( + status, &body, truncated, + )) + } + + pub async fn put_bucket_encryption( + &self, + bucket: &str, + configuration: &BucketEncryptionConfiguration, + ) -> Result<(), RustfsClientError> { + if bucket.trim().is_empty() { + return Err(RustfsClientError::RequestBuildFailed); + } + + let path = format!("/{bucket}"); + let query = build_canonical_query(&[("encryption", "")]); + let body = configuration.to_xml(); + let content_md5 = content_md5(&body); + let signed = self.sign_request_with_extra_headers( + "PUT", + &path, + &query, + &body, + ADMIN_SIGNING_SERVICE, + &[ + ("content-md5", content_md5.as_str()), + ("content-type", "application/xml"), + ], + )?; + let host = self.host()?; + let response = self + .http_client + .put(format!( + "{}{}?{query}", + self.base_url.trim_end_matches('/'), + path + )) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .header("content-md5", content_md5) + .header("content-type", "application/xml") + .body(body) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + Ok(()) + } else { + Err(RustfsClientError::unexpected_response(response).await) + } + } + + pub async fn delete_bucket_encryption(&self, bucket: &str) -> Result<(), RustfsClientError> { + if bucket.trim().is_empty() { + return Err(RustfsClientError::RequestBuildFailed); + } + + let path = format!("/{bucket}"); + let query = build_canonical_query(&[("encryption", "")]); + let signed = self.sign_request("DELETE", &path, &query, "", None, ADMIN_SIGNING_SERVICE)?; + let host = self.host()?; + let response = self + .http_client + .delete(format!( + "{}{}?{query}", + self.base_url.trim_end_matches('/'), + path + )) + .header("x-amz-date", &signed.amz_date) + .header("x-amz-content-sha256", &signed.payload_hash) + .header("authorization", &signed.authorization) + .header("host", host) + .send() + .await + .map_err(|_| RustfsClientError::RequestFailed)?; + + if response.status().is_success() { + return Ok(()); + } + let status = response.status(); + let (body, truncated) = RustfsClientError::limited_response_body(response).await; + if is_absent_resource(status, &body) { + return Ok(()); + } + Err(RustfsClientError::unexpected_status_with_limited_body( + status, &body, truncated, + )) + } + pub async fn put_bucket_policy( &self, bucket: &str, @@ -1415,6 +1647,68 @@ mod tests { )); } + #[test] + fn bucket_encryption_xml_matches_rustfs_get_response_shape() { + // GetBucketEncryption body from RustFS 1.0.2-preview.2 after `PUT ?encryption` with AES256. + let rustfs_get = "AES256"; + let sse_s3 = BucketEncryptionConfiguration { + algorithm: BucketEncryptionAlgorithm::Aes256, + kms_master_key_id: None, + bucket_key_enabled: false, + }; + assert_eq!(parse_bucket_encryption_xml(rustfs_get).unwrap(), sse_s3); + assert_eq!( + sse_s3.to_xml(), + "AES256" + ); + assert_eq!( + parse_bucket_encryption_xml(&sse_s3.to_xml()).unwrap(), + sse_s3 + ); + + let sse_kms = BucketEncryptionConfiguration { + algorithm: BucketEncryptionAlgorithm::AwsKms, + kms_master_key_id: Some("app-key".to_string()), + bucket_key_enabled: true, + }; + assert!( + sse_kms + .to_xml() + .contains("app-key") + ); + assert_eq!( + parse_bucket_encryption_xml(&sse_kms.to_xml()).unwrap(), + sse_kms + ); + } + + #[test] + fn bucket_encryption_xml_rejects_unknown_ambiguous_or_missing_fields() { + for xml in [ + "", + "", + "AES128", + "AES256AES256", + "AES256", + "AES256yes", + "", + ] { + assert!( + matches!( + parse_bucket_encryption_xml(xml), + Err(RustfsClientError::InvalidBucketEncryptionResponse) + ), + "{xml} must be rejected" + ); + } + let empty_key = parse_bucket_encryption_xml( + "aws:kmsfalse", + ) + .unwrap(); + assert_eq!(empty_key.kms_master_key_id, None); + assert!(!empty_key.bucket_key_enabled); + } + #[test] fn lifecycle_xml_is_stable_and_escapes_values() { let configuration = BucketLifecycleConfiguration { diff --git a/deploy/rustfs-operator/README.md b/deploy/rustfs-operator/README.md index 908a986..68c90aa 100755 --- a/deploy/rustfs-operator/README.md +++ b/deploy/rustfs-operator/README.md @@ -298,7 +298,7 @@ field updates the StatefulSet Pod template and starts a rolling update. ### Tenant Provisioning -Tenants can declare RustFS canned policies, regular users, and buckets directly in `spec.policies`, `spec.users`, and `spec.buckets`. Provisioning starts only after the Tenant workload is ready, uses `spec.credsSecret` as the RustFS admin credential source, and reports progress under `status.provisioning`. +Tenants can declare RustFS canned policies, regular users, remote tiers, and buckets directly in `spec.policies`, `spec.users`, `spec.tiers`, and `spec.buckets`. Provisioning starts only after the Tenant workload is ready, uses `spec.credsSecret` as the RustFS admin credential source, and reports progress under `status.provisioning`. User provisioning requires a non-empty direct policy mapping: @@ -336,7 +336,7 @@ spec: days: 30 ``` -Policy ConfigMaps and user Secrets must live in the Tenant namespace. `users[].credsSecret.name` selects the credentials Secret; when omitted, the operator falls back to a Secret named after `users[].name` for compatibility with existing manifests. Bucket versioning can be enabled or suspended. Object Lock automatically requires enabled versioning and cannot be disabled after activation; `objectLockConfiguration` manages an optional default `Governance` or `Compliance` retention period in days. Omitting the configuration leaves an existing rule unmanaged, while `state: Absent` removes only an operator-owned default rule. Bucket lifecycle supports expiration, incomplete multipart upload cleanup, and transition of current or noncurrent versions to a remote tier registered in RustFS. Omission leaves lifecycle unmanaged; `state: Absent` deletes only a configuration whose live hash still matches the Operator's ownership record. The operator indexes references from Tenant specs, so creating or updating a referenced object enqueues every referencing Tenant without requiring or mutating labels or requiring write access to that object. Provisioned resources are retained when removed from the Tenant spec. +Policy ConfigMaps, user Secrets, and tier Secrets must live in the Tenant namespace. `users[].credsSecret.name` selects the credentials Secret; when omitted, the operator falls back to a Secret named after `users[].name` for compatibility with existing manifests. Bucket versioning can be enabled or suspended. Object Lock automatically requires enabled versioning and cannot be disabled after activation; `objectLockConfiguration` manages an optional default `Governance` or `Compliance` retention period in days. Omitting the configuration leaves an existing rule unmanaged, while `state: Absent` removes only an operator-owned default rule. `encryption` sets the bucket default encryption (`SSE-S3`, or `SSE-KMS` with an optional `kmsKeyId` when the Tenant enables KMS) with the same ownership rules. Bucket lifecycle supports expiration, incomplete multipart upload cleanup, and transition of current or noncurrent versions to a remote tier registered in RustFS. `spec.tiers` registers such tiers before buckets are reconciled; tier credentials come from `tiers[].credsSecret` and are rotated when the Secret changes, while a live tier with different immutable settings is reported as a conflict. Omission leaves lifecycle unmanaged; `state: Absent` deletes only a configuration whose live hash still matches the Operator's ownership record. The operator indexes references from Tenant specs, so creating or updating a referenced object enqueues every referencing Tenant without requiring or mutating labels or requiring write access to that object. Provisioned resources are retained when removed from the Tenant spec. ### RBAC Configuration diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index f7cbbf9..8ce920f 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -1235,6 +1235,43 @@ spec: enum: - Retain type: string + encryption: + description: |- + Default bucket encryption. Omission leaves the live configuration unmanaged; `Absent` + removes only a configuration previously owned by the operator. + nullable: true + properties: + kmsKeyId: + description: KMS key for `SSE-KMS`. Omission uses the Tenant `spec.encryption.defaultKeyId`. + maxLength: 1024 + minLength: 1 + nullable: true + pattern: ^[A-Za-z0-9._:/+=@-]+$ + type: string + mode: + description: |- + `SSE-S3` stores S3 `AES256` and requires Tenant KMS or `RUSTFS_SSE_S3_MASTER_KEY` in + `spec.env`; `SSE-KMS` stores `aws:kms` and requires `spec.encryption.enabled` on the Tenant. + enum: + - SSE-S3 + - SSE-KMS + - null + nullable: true + type: string + state: + description: Omission is equivalent to `Present`. + enum: + - Present + - Absent + - null + nullable: true + type: string + type: object + x-kubernetes-validations: + - message: Present bucket encryption requires mode; Absent forbids mode and kmsKeyId + rule: '(!has(self.state) || self.state == ''Present'') ? has(self.mode) : !has(self.mode) && !has(self.kmsKeyId)' + - message: kmsKeyId requires mode SSE-KMS + rule: '!has(self.kmsKeyId) || (has(self.mode) && self.mode == ''SSE-KMS'')' lifecycle: description: |- Declarative S3 bucket lifecycle configuration. Omission leaves the live lifecycle @@ -3188,6 +3225,77 @@ spec: serviceAccountName: nullable: true type: string + tiers: + description: |- + Remote storage tiers that bucket lifecycle transitions can reference by name. + Tiers are reconciled before buckets. + items: + description: Remote storage tier registered in RustFS for bucket lifecycle transitions. + properties: + bucket: + description: Remote bucket that receives transitioned objects. + minLength: 1 + type: string + credsSecret: + description: |- + Secret with `accesskey` and `secretkey` for the remote bucket. Updating the Secret + rotates the tier credentials in RustFS. + properties: + name: + maxLength: 253 + minLength: 1 + type: string + required: + - name + type: object + deletionPolicy: + enum: + - Retain + type: string + endpoint: + description: Remote S3 endpoint URL, for example `https://gateway.storjshare.io`. + minLength: 1 + type: string + force: + description: |- + Register the tier even when the remote prefix already holds objects, for example a + prefix written by this Tenant before a restore. RustFS refuses such a prefix otherwise. + type: boolean + name: + description: |- + Tier name, referenced as `storageClass` by bucket lifecycle transitions. Uppercase ASCII + letters, digits, `_` and `-` only: RustFS looks up a tier by the raw URL path segment. + maxLength: 253 + minLength: 1 + pattern: ^[A-Z0-9_-]+$ + type: string + x-kubernetes-validations: + - message: tier name must not be a reserved storage class + rule: self != 'STANDARD' && self != 'REDUCED_REDUNDANCY' + prefix: + description: Object prefix inside the remote bucket. + type: string + region: + type: string + type: + description: Remote backend of a tier. All three use the S3 API with static credentials. + enum: + - S3 + - RustFS + - MinIO + type: string + required: + - bucket + - credsSecret + - endpoint + - name + - type + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map tls: nullable: true properties: @@ -3926,6 +4034,12 @@ spec: desiredHash: nullable: true type: string + encryptionDesiredHash: + nullable: true + type: string + encryptionLastAppliedHash: + nullable: true + type: string lastAppliedAccessKeyHash: nullable: true type: string @@ -4052,6 +4166,56 @@ spec: - state type: object type: array + tiers: + items: + properties: + desiredHash: + nullable: true + type: string + lastAppliedAccessKeyHash: + nullable: true + type: string + lastAppliedGeneration: + format: int64 + nullable: true + type: integer + lastAppliedHash: + nullable: true + type: string + lastTransitionTime: + nullable: true + type: string + message: + nullable: true + type: string + name: + type: string + objectLock: + nullable: true + type: boolean + observedSecretName: + nullable: true + type: string + observedSecretResourceVersion: + nullable: true + type: string + policies: + items: + type: string + type: array + reason: + type: string + region: + nullable: true + type: string + state: + type: string + required: + - name + - reason + - state + type: object + type: array users: items: description: |- diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index f5350cf..62ed2c0 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -952,35 +952,67 @@ spec: ### 7.8 Bootstrap Provisioning -The operator can create RustFS policies, users, and buckets after the Tenant workload is ready. Configure: +The operator can create RustFS policies, users, remote tiers, and buckets after the Tenant workload is ready. Configure: - `spec.credsSecret` for RustFS admin credentials. - `spec.policies` for canned policies sourced from ConfigMaps. - `spec.users` for regular users. Each user must have at least one direct policy mapping. +- `spec.tiers` for remote storage tiers that bucket lifecycle transitions move data to. - `spec.buckets` for buckets, versioning, Object Lock and default retention, canned anonymous access (`Private`, `Download`, `Upload`, `Public`), or a custom bucket policy ConfigMap. Non-private `anonymous` values and `policy` are mutually exclusive. Explicit `anonymous: Private` without `policy` removes a bucket policy previously applied by the Operator; it reports a conflict instead of deleting an unowned live policy. For compatibility, legacy `Private` plus `policy` configurations keep the custom policy. When both fields are omitted, the Operator does not change a live bucket policy and retains its ownership record for a later explicit `Private` transition. Bucket versioning and Object Lock are declarative but conservative. `versioning: true` enables versioning. `versioning: false` keeps a never-versioned bucket unchanged or requests S3 `Suspended` after versioning has been enabled; omitting the field leaves versioning unmanaged unless Object Lock requires it. `objectLock: true` enables Object Lock after versioning is enabled. Object Lock cannot be disabled, and the Operator refuses `objectLock: false` or versioning suspension when the live bucket has Object Lock enabled. `objectLockConfiguration` manages the default retention rule: omit `state` for `Present`, specify `mode: Governance` or `Compliance` with `days`, or use `state: Absent` without `mode` and `days` to remove only the default rule. Omitting the entire field leaves a live default-retention rule unmanaged. The Operator adopts an identical live rule, but does not replace or remove a different rule unless its hash matches the last configuration recorded in status. Default retention applies to new object versions; it does not retroactively lock existing versions. +Bucket default encryption is declared with `encryption`. `mode: SSE-S3` stores S3 `AES256` and requires an SSE-S3 key for RustFS: either `spec.encryption.enabled` on the Tenant, or `RUSTFS_SSE_S3_MASTER_KEY` in `spec.env`, a base64-encoded 32-byte key set by value or from a Secret key, which the Operator validates before applying the bucket default. `mode: SSE-KMS` stores `aws:kms`, requires `spec.encryption.enabled`, and uses `kmsKeyId` or, when it is omitted, the Tenant `spec.encryption.defaultKeyId`; changing either key re-applies an owned configuration. Default encryption applies to objects written after it is set; it does not re-encrypt existing objects. `state: Absent` without `mode` and `kmsKeyId` removes the configuration. The Operator adopts a matching live configuration, but never replaces or removes a different one unless its hash matches the configuration last recorded in `status.provisioning.buckets`. Omitting `encryption` performs no encryption API calls and leaves the live configuration untouched. + Each bucket may also declare `lifecycle.state: Present` with one or more S3 lifecycle rules. The current schema supports expiration after a positive number of days, aborting incomplete multipart uploads after a positive number of days, `transition` of current versions after zero or more days, and `noncurrentVersionTransition` of noncurrent versions after zero or more days; every rule has a unique ID, an `Enabled` or `Disabled` status, and an explicit prefix filter (use `prefix: ""` for all objects). `lifecycle.state: Absent` requests deletion and cannot include rules. The Operator adopts an identical live configuration, but never replaces or deletes a different configuration without a matching ownership hash in `status.provisioning.buckets`. Omitting `lifecycle` performs no lifecycle API calls and leaves the live configuration untouched. -Transition actions move data to a remote tier. `storageClass` is the name of a tier already registered in RustFS (for example with `rc ilm tier add`); tier names are uppercase, and `STANDARD` and `REDUCED_REDUNDANCY` are reserved. The Operator does not register tiers. Before applying a lifecycle with transition actions it checks that every referenced tier exists; while one is missing, the bucket stays `Pending` with a message naming the tier and the Operator retries, so the tier can be registered after the rule is added: +Transition actions move data to a remote tier. `storageClass` is a tier name: a tier declared in `spec.tiers`, or one registered in RustFS by other means (for example with `rc ilm tier add`). Tier names use only uppercase ASCII letters, digits, `_`, and `-`, and `STANDARD` and `REDUCED_REDUNDANCY` are reserved. Before applying a lifecycle with transition actions the Operator checks that every referenced tier exists; while one is missing, the bucket stays `Pending` with a message naming the tier and the Operator retries, so the tier can be registered after the rule is added. A referenced tier declared in `spec.tiers` must also be `Ready`: while it is pending the bucket stays `Pending`, and when it has failed, for example with `TierConflict`, the bucket fails with a message naming the tier. + +`spec.tiers` registers S3-compatible remote tiers (`type: S3`, `RustFS`, or `MinIO`). Tiers are reconciled before buckets, so a lifecycle rule can reference a tier declared in the same Tenant. Each tier sets the remote `endpoint` and `bucket`, an optional `prefix` and `region`, and `credsSecret`, a Secret in the Tenant namespace with `accesskey` and `secretkey`, each at least 8 characters. RustFS does not return a tier's secret key, so the Operator records the Secret's `resourceVersion` in `status.provisioning.tiers` and updates the tier credentials when the Secret changes, when `credsSecret` points to another Secret, or when the access key listed by RustFS differs from the Secret. RustFS cannot change the type, endpoint, bucket, prefix, or region of a registered tier: a live tier with the same name and different settings is reported as `TierConflict` and left untouched, while one with matching settings is adopted. RustFS refuses to register a tier whose remote prefix already holds objects; set `force: true` only when that prefix belongs to this Tenant, for example after a restore. Tiers removed from the spec are retained in RustFS. ```yaml -lifecycle: - state: Present - rules: - - id: to-cold - status: Enabled - filter: - prefix: "" - transition: - days: 7 - storageClass: COLD +apiVersion: v1 +kind: Secret +metadata: + name: cold-tier + namespace: storage +type: Opaque +stringData: + accesskey: + secretkey: +--- +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: rustfs-a + namespace: storage +spec: + # credsSecret and pools as in the full example below + tiers: + - name: COLD + type: S3 + endpoint: https://s3.example.com + bucket: rustfs-cold + prefix: rustfs-a/ + credsSecret: + name: cold-tier + buckets: + - name: app-data + lifecycle: + state: Present + rules: + - id: to-cold + status: Enabled + filter: + prefix: "" + transition: + days: 7 + storageClass: COLD ``` Transient Kubernetes and RustFS admin/S3 failures (timeouts, 429, 5xx, connection errors, TLS not ready) leave provisioning items `Pending` and requeue instead of marking the Tenant `Failed`. Permanent 4xx configuration errors still fail and wait for a spec or object change. -ConfigMaps and user Secrets must live in the Tenant namespace. The Operator indexes references from Tenant specs, so creating or updating a referenced object enqueues every referencing Tenant without requiring or mutating labels or requiring write access to that object. +ConfigMaps, user Secrets, and tier Secrets must live in the Tenant namespace. The Operator indexes references from Tenant specs, so creating or updating a referenced object enqueues every referencing Tenant without requiring or mutating labels or requiring write access to that object. Policy documents are parsed by RustFS. Use S3 ARN resource patterns such as `arn:aws:s3:::bucket` and `arn:aws:s3:::bucket/*`; for all buckets, use `arn:aws:s3:::*`. A bare `Resource: "*"` is not accepted by RustFS policy parsing. @@ -1019,6 +1051,15 @@ stringData: accesskey: appuser01 secretkey: appuser01secret --- +apiVersion: v1 +kind: Secret +metadata: + name: rustfs-sse-s3 + namespace: storage +stringData: + # openssl rand -base64 32 + master-key: REPLACE_WITH_BASE64_32_BYTE_KEY +--- apiVersion: rustfs.com/v1alpha1 kind: Tenant metadata: @@ -1032,6 +1073,12 @@ spec: servers: 1 persistence: volumesPerServer: 4 + env: + - name: RUSTFS_SSE_S3_MASTER_KEY + valueFrom: + secretKeyRef: + name: rustfs-sse-s3 + key: master-key policies: - name: app-readwrite document: @@ -1051,6 +1098,8 @@ spec: objectLockConfiguration: mode: Compliance days: 30 + encryption: + mode: SSE-S3 anonymous: Download lifecycle: state: Present diff --git a/docs/operator-user-guide.zh-CN.md b/docs/operator-user-guide.zh-CN.md index f24e35d..5add73f 100644 --- a/docs/operator-user-guide.zh-CN.md +++ b/docs/operator-user-guide.zh-CN.md @@ -896,35 +896,67 @@ spec: ### 7.8 初始化 Provisioning -Operator 可以在 Tenant workload Ready 后自动创建 RustFS policy、user 和 bucket。需要配置： +Operator 可以在 Tenant workload Ready 后自动创建 RustFS policy、user、远程 tier 和 bucket。需要配置： - `spec.credsSecret`：RustFS 管理员凭据。 - `spec.policies`：从 ConfigMap 读取 policy document。 - `spec.users`：普通用户。每个 user 必须至少直接绑定一个 policy。 +- `spec.tiers`：bucket 生命周期迁移操作使用的远程存储层（tier）。 - `spec.buckets`：bucket、版本控制、Object Lock 及默认保留策略、匿名访问（`Private` / `Download` / `Upload` / `Public`），或来自 ConfigMap 的自定义 bucket policy。非 `Private` 的 `anonymous` 值与 `policy` 互斥。未设置 `policy` 时，显式设置 `anonymous: Private` 会删除此前由 Operator 应用的 bucket policy；若现有 policy 不归 Operator 管理，则报告冲突而不删除。为兼容旧对象，`Private` 与 `policy` 同时存在时继续以自定义 policy 为准。两者都省略时，Operator 不会改写已有 bucket policy，并会保留所有权记录供后续显式切换为 `Private` 时校验。 Bucket 版本控制和 Object Lock 采用声明式但保守的调和策略。`versioning: true` 启用版本控制。`versioning: false` 会让从未启用版本控制的 bucket 保持原状，或在版本控制已启用后请求 S3 `Suspended`；省略该字段时保持现状，除非 Object Lock 要求先启用版本控制。`objectLock: true` 会在版本控制启用后开启 Object Lock。Object Lock 不可关闭；当线上 bucket 已启用 Object Lock 时，Operator 会拒绝 `objectLock: false` 或暂停版本控制。`objectLockConfiguration` 管理默认保留规则：省略 `state` 等同于 `Present`，此时必须设置 `mode: Governance` 或 `Compliance` 以及 `days`；`state: Absent` 必须省略 `mode` 和 `days`，仅移除默认规则。完全省略 `objectLockConfiguration` 时，不管理已有默认保留规则。Operator 可以接管与声明完全相同的线上规则，但只有线上配置哈希仍与 status 中最后一次应用的配置一致时，才会替换或移除不同规则。默认保留策略只作用于之后创建的对象版本，不会追溯锁定已有版本。 +Bucket 默认加密通过 `encryption` 声明。`mode: SSE-S3` 写入 S3 `AES256`，并要求 RustFS 具备 SSE-S3 密钥：Tenant 设置 `spec.encryption.enabled`，或在 `spec.env` 中提供 `RUSTFS_SSE_S3_MASTER_KEY`（base64 编码的 32 字节密钥，直接设置或来自 Secret key），Operator 会在应用 bucket 默认加密前校验该密钥。`mode: SSE-KMS` 写入 `aws:kms`，要求设置 `spec.encryption.enabled`，并使用 `kmsKeyId`；省略时使用 Tenant 的 `spec.encryption.defaultKeyId`。任一 key 变更时，Operator 会重新应用其拥有的配置。默认加密只作用于设置之后写入的对象，不会重新加密已有对象。`state: Absent`（不能包含 `mode` 和 `kmsKeyId`）表示移除该配置。Operator 可以接管与声明一致的线上配置，但只有线上配置哈希与 `status.provisioning.buckets` 中最后记录的配置一致时，才会替换或移除不同的配置。省略 `encryption` 时不会调用加密 API，也不会改动现有配置。 + 每个 bucket 还可以通过 `lifecycle.state: Present` 声明一个或多个 S3 生命周期规则。当前 schema 支持按正整数天数过期对象、按正整数天数清理未完成的分片上传、通过 `transition` 在零天或更多天后迁移当前版本，以及通过 `noncurrentVersionTransition` 在零天或更多天后迁移非当前版本；每条规则必须有唯一 ID、`Enabled` 或 `Disabled` 状态，并显式指定 prefix filter（`prefix: ""` 表示全部对象）。`lifecycle.state: Absent` 表示请求删除，且不能包含规则。Operator 可以接管与声明完全相同的现有配置，但没有 `status.provisioning.buckets` 中匹配的所有权哈希时，不会覆盖或删除不同的现有配置。省略 `lifecycle` 时不会调用 Lifecycle API，也不会改动现有配置。 -迁移类操作会把数据移动到远程层（tier）。`storageClass` 是 RustFS 中已注册的 tier 名称（例如通过 `rc ilm tier add` 注册）；tier 名称必须为大写，`STANDARD` 和 `REDUCED_REDUNDANCY` 为保留名称。Operator 不负责注册 tier。在应用包含迁移操作的生命周期配置之前，Operator 会检查所有被引用的 tier 是否存在；如果有 tier 尚未注册，bucket 会保持 `Pending` 状态并在消息中列出该 tier，Operator 会持续重试，因此可以在添加规则之后再注册 tier： +迁移类操作会把数据移动到远程层（tier）。`storageClass` 是 tier 名称：可以是 `spec.tiers` 中声明的 tier，也可以是通过其他方式（例如 `rc ilm tier add`）在 RustFS 中注册的 tier。tier 名称只能包含大写 ASCII 字母、数字、`_` 和 `-`，`STANDARD` 和 `REDUCED_REDUNDANCY` 为保留名称。在应用包含迁移操作的生命周期配置之前，Operator 会检查所有被引用的 tier 是否存在；如果有 tier 尚未注册，bucket 会保持 `Pending` 状态并在消息中列出该 tier，Operator 会持续重试，因此可以在添加规则之后再注册 tier。被引用且在 `spec.tiers` 中声明的 tier 还必须处于 `Ready` 状态：该 tier 处于等待状态时 bucket 保持 `Pending`；该 tier 失败（例如 `TierConflict`）时 bucket 也会失败，并在消息中列出该 tier。 + +`spec.tiers` 用于注册兼容 S3 的远程 tier（`type: S3`、`RustFS` 或 `MinIO`）。tier 会先于 bucket 调和，因此生命周期规则可以引用同一 Tenant 中声明的 tier。每个 tier 需要设置远程 `endpoint` 和 `bucket`，可选的 `prefix` 和 `region`，以及 `credsSecret`：位于 Tenant namespace、包含 `accesskey` 和 `secretkey`（均至少 8 个字符）的 Secret。RustFS 不会返回 tier 的 secret key，因此 Operator 会在 `status.provisioning.tiers` 中记录 Secret 的 `resourceVersion`，并在 Secret 变更、`credsSecret` 指向另一个 Secret，或 RustFS 列出的 access key 与 Secret 不一致时更新 tier 凭据。RustFS 不能修改已注册 tier 的类型、endpoint、bucket、prefix 或 region：同名但配置不同的现有 tier 会报告为 `TierConflict` 且不会被改动；配置一致的现有 tier 会被接管。如果远程 prefix 中已有对象，RustFS 会拒绝注册该 tier；仅当该 prefix 属于此 Tenant 时（例如恢复之后）才设置 `force: true`。从 spec 中移除的 tier 会保留在 RustFS 中。 ```yaml -lifecycle: - state: Present - rules: - - id: to-cold - status: Enabled - filter: - prefix: "" - transition: - days: 7 - storageClass: COLD +apiVersion: v1 +kind: Secret +metadata: + name: cold-tier + namespace: storage +type: Opaque +stringData: + accesskey: + secretkey: +--- +apiVersion: rustfs.com/v1alpha1 +kind: Tenant +metadata: + name: rustfs-a + namespace: storage +spec: + # credsSecret 和 pools 参见下方完整示例 + tiers: + - name: COLD + type: S3 + endpoint: https://s3.example.com + bucket: rustfs-cold + prefix: rustfs-a/ + credsSecret: + name: cold-tier + buckets: + - name: app-data + lifecycle: + state: Present + rules: + - id: to-cold + status: Enabled + filter: + prefix: "" + transition: + days: 7 + storageClass: COLD ``` Kubernetes 或 RustFS 管理/S3 API 的瞬时失败（超时、429、5xx、连接错误、TLS 未就绪）会把 provisioning 条目保持为 `Pending` 并重新入队，而不会把 Tenant 标为 `Failed`。永久性 4xx 配置错误仍会失败并等待 spec 或对象变更。 -ConfigMap 和 user Secret 必须位于 Tenant namespace。Operator 会从 Tenant spec 建立反向引用索引，因此被引用资源的创建或更新会触发所有引用它的 Tenant reconcile；无需要求或修改资源标签，也不需要对这些资源拥有写权限。 +ConfigMap、user Secret 和 tier Secret 必须位于 Tenant namespace。Operator 会从 Tenant spec 建立反向引用索引，因此被引用资源的创建或更新会触发所有引用它的 Tenant reconcile；无需要求或修改资源标签，也不需要对这些资源拥有写权限。 Policy document 由 RustFS 解析。请使用 `arn:aws:s3:::bucket` 和 `arn:aws:s3:::bucket/*` 这类 S3 ARN resource 写法；如需匹配所有 bucket，请使用 `arn:aws:s3:::*`。RustFS policy parser 不接受裸 `Resource: "*"`。 @@ -963,6 +995,15 @@ stringData: accesskey: appuser01 secretkey: appuser01secret --- +apiVersion: v1 +kind: Secret +metadata: + name: rustfs-sse-s3 + namespace: storage +stringData: + # openssl rand -base64 32 + master-key: REPLACE_WITH_BASE64_32_BYTE_KEY +--- apiVersion: rustfs.com/v1alpha1 kind: Tenant metadata: @@ -976,6 +1017,12 @@ spec: servers: 1 persistence: volumesPerServer: 4 + env: + - name: RUSTFS_SSE_S3_MASTER_KEY + valueFrom: + secretKeyRef: + name: rustfs-sse-s3 + key: master-key policies: - name: app-readwrite document: @@ -995,6 +1042,8 @@ spec: objectLockConfiguration: mode: Compliance days: 30 + encryption: + mode: SSE-S3 anonymous: Download lifecycle: state: Present diff --git a/e2e/src/cases/mod.rs b/e2e/src/cases/mod.rs index 19e55c8..99d012a 100644 --- a/e2e/src/cases/mod.rs +++ b/e2e/src/cases/mod.rs @@ -145,7 +145,7 @@ mod tests { }); assert_eq!(counts.get(&Suite::Smoke).copied().unwrap_or_default(), 3); - assert_eq!(counts.get(&Suite::Operator).copied().unwrap_or_default(), 2); + assert_eq!(counts.get(&Suite::Operator).copied().unwrap_or_default(), 3); assert_eq!(counts.get(&Suite::Sts).copied().unwrap_or_default(), 2); assert_eq!(counts.get(&Suite::Console).copied().unwrap_or_default(), 1); assert_eq!( diff --git a/e2e/src/cases/operator.rs b/e2e/src/cases/operator.rs index 1fc097d..fbd4e2a 100644 --- a/e2e/src/cases/operator.rs +++ b/e2e/src/cases/operator.rs @@ -30,6 +30,13 @@ pub fn cases() -> Vec { "operator/status", "operator", ), + CaseSpec::new( + Suite::Operator, + "operator_live_sse_s3_bucket_encrypts_new_objects", + "Provision an SSE-S3 bucket with the master key from a Secret and verify new objects are stored AES256-encrypted and read back intact.", + "operator/provisioning", + "operator", + ), ] } @@ -49,6 +56,7 @@ mod tests { vec![ "operator_live_tenant_is_ready_and_observed", "operator_live_status_subresource_enforces_cas_and_pruning", + "operator_live_sse_s3_bucket_encrypts_new_objects", ] ); } diff --git a/e2e/tests/operator.rs b/e2e/tests/operator.rs index 266ffb9..8796751 100644 --- a/e2e/tests/operator.rs +++ b/e2e/tests/operator.rs @@ -12,18 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Result, ensure}; +use anyhow::{Context, Result, ensure}; +use aws_config::BehaviorVersion; +use aws_credential_types::Credentials; +use aws_sdk_s3::{Client, config::Region, primitives::ByteStream, types::ServerSideEncryption}; use kube::Api; use rustfs_operator_e2e::framework::{ assertions, config::E2eConfig, kube_client, kubectl::Kubectl, live, + port_forward::PortForwardSpec, resources, wait, }; use serde_json::{Value, json}; +use std::time::Duration; +use uuid::Uuid; use operator::types::v1alpha1::tenant::Tenant; const CHECKPOINT_TEST_CRD: &str = "ownershipcheckpointtests.e2e.rustfs.com"; const CHECKPOINT_TEST_RESOURCE: &str = "ownershipcheckpointtests.e2e.rustfs.com"; const CHECKPOINT_TEST_NAME: &str = "ownership-checkpoint-contract"; +const SSE_S3_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; +const SSE_S3_MASTER_KEY_SECRET_KEY: &str = "master-key"; +// base64 of the 32 ASCII bytes "0123456789abcdef0123456789abcdef". +const SSE_S3_TEST_MASTER_KEY: &str = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="; +const SSE_S3_PROBE_OBJECT: &str = "sse-s3-probe.bin"; const CHECKPOINT_TEST_CRD_YAML: &str = r#" apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -123,6 +134,217 @@ async fn operator_live_status_subresource_enforces_cas_and_pruning() -> Result<( Ok(()) } +#[tokio::test] +#[ignore = "requires a dedicated live cluster; run through `make e2e-live-run`"] +async fn operator_live_sse_s3_bucket_encrypts_new_objects() -> Result<()> { + let config = E2eConfig::from_env(); + live::require_live_enabled(&config)?; + live::ensure_dedicated_context(&config)?; + + let kubectl = Kubectl::new(&config); + let secret_name = format!("{}-sse-s3", config.tenant_name); + let bucket = format!("e2e-sse-s3-{}", &Uuid::new_v4().simple().to_string()[..8]); + + let result = verify_sse_s3_bucket_encryption(&config, &kubectl, &secret_name, &bucket).await; + let restore_result = restore_smoke_tenant(&config, &kubectl, &secret_name).await; + + result?; + restore_result?; + Ok(()) +} + +async fn verify_sse_s3_bucket_encryption( + config: &E2eConfig, + kubectl: &Kubectl, + secret_name: &str, + bucket: &str, +) -> Result<()> { + kubectl + .apply_yaml_command(sse_s3_secret_manifest(config, secret_name)) + .run_checked()?; + kubectl + .apply_yaml_command(sse_s3_tenant_manifest(config, secret_name, bucket)?) + .run_checked()?; + + let client = kube_client::default_client().await?; + let tenants: Api = kube_client::tenant_api(client, &config.test_namespace); + wait::wait_for_tenant_ready(tenants.clone(), &config.tenant_name, config.timeout).await?; + wait_for_bucket_encryption_applied(&tenants, &config.tenant_name, bucket, config.timeout) + .await?; + + let spec = + PortForwardSpec::tenant_io_on_available_port(&config.test_namespace, &config.tenant_name)?; + let mut port_forward = spec + .start_with_temp_log(kubectl) + .context("start RustFS tenant IO port-forward")?; + let s3 = s3_client(&spec.local_base_url()).await; + + let payload = format!("sse-s3 probe {bucket}\n").repeat(4096).into_bytes(); + wait::wait_until( + "S3 PutObject through the tenant IO port-forward", + Duration::from_secs(60), + Duration::from_secs(2), + || { + let s3 = s3.clone(); + let payload = payload.clone(); + async move { + Ok(s3 + .put_object() + .bucket(bucket) + .key(SSE_S3_PROBE_OBJECT) + .body(ByteStream::from(payload)) + .send() + .await + .ok()) + } + }, + ) + .await?; + port_forward.ensure_running()?; + + let head = s3 + .head_object() + .bucket(bucket) + .key(SSE_S3_PROBE_OBJECT) + .send() + .await + .context("HeadObject on the SSE-S3 probe object")?; + ensure!( + head.server_side_encryption() == Some(&ServerSideEncryption::Aes256), + "object written without SSE headers is not AES256-encrypted: {:?}", + head.server_side_encryption() + ); + + let body = s3 + .get_object() + .bucket(bucket) + .key(SSE_S3_PROBE_OBJECT) + .send() + .await + .context("GetObject on the SSE-S3 probe object")? + .body + .collect() + .await? + .into_bytes(); + ensure!( + body.as_ref() == payload.as_slice(), + "decrypted object differs from the uploaded payload ({} vs {} bytes)", + body.len(), + payload.len() + ); + + s3.delete_object() + .bucket(bucket) + .key(SSE_S3_PROBE_OBJECT) + .send() + .await + .context("delete the SSE-S3 probe object")?; + s3.delete_bucket() + .bucket(bucket) + .send() + .await + .context("delete the SSE-S3 probe bucket")?; + Ok(()) +} + +async fn restore_smoke_tenant( + config: &E2eConfig, + kubectl: &Kubectl, + secret_name: &str, +) -> Result<()> { + resources::apply_smoke_tenant_resources(config)?; + let client = kube_client::default_client().await?; + let tenants: Api = kube_client::tenant_api(client, &config.test_namespace); + wait::wait_for_tenant_ready(tenants, &config.tenant_name, config.timeout).await?; + kubectl + .clone() + .namespaced(&config.test_namespace) + .command(["delete", "secret", secret_name, "--ignore-not-found=true"]) + .run_checked()?; + Ok(()) +} + +fn sse_s3_secret_manifest(config: &E2eConfig, secret_name: &str) -> String { + format!( + r#"apiVersion: v1 +kind: Secret +metadata: + name: {secret_name} + namespace: {namespace} +type: Opaque +stringData: + {key}: {value} +"#, + namespace = config.test_namespace, + key = SSE_S3_MASTER_KEY_SECRET_KEY, + value = SSE_S3_TEST_MASTER_KEY, + ) +} + +fn sse_s3_tenant_manifest(config: &E2eConfig, secret_name: &str, bucket: &str) -> Result { + let mut tenant = serde_json::to_value(resources::smoke_tenant_template(config).build())?; + let spec = tenant["spec"] + .as_object_mut() + .context("smoke Tenant has no spec")?; + spec.entry("env") + .or_insert_with(|| json!([])) + .as_array_mut() + .context("smoke Tenant spec.env is not a list")? + .push(json!({ + "name": SSE_S3_MASTER_KEY_ENV, + "valueFrom": { + "secretKeyRef": {"name": secret_name, "key": SSE_S3_MASTER_KEY_SECRET_KEY} + } + })); + spec.insert( + "buckets".to_string(), + json!([{"name": bucket, "encryption": {"mode": "SSE-S3"}}]), + ); + Ok(serde_yaml_ng::to_string(&tenant)?) +} + +async fn wait_for_bucket_encryption_applied( + tenants: &Api, + tenant_name: &str, + bucket: &str, + timeout: Duration, +) -> Result<()> { + wait::wait_until( + &format!("bucket {bucket} SSE-S3 encryption to be applied"), + timeout, + Duration::from_secs(5), + || async move { + let tenant = tenants.get(tenant_name).await?; + let applied = tenant.status.as_ref().is_some_and(|status| { + status.provisioning.buckets.iter().any(|item| { + item.item.name == bucket + && item.item.state == "Ready" + && item.encryption_desired_hash.is_some() + && item.encryption_desired_hash == item.encryption_last_applied_hash + }) + }); + Ok(applied.then_some(())) + }, + ) + .await +} + +async fn s3_client(endpoint: &str) -> Client { + let (access_key, secret_key) = resources::test_credentials(); + let credentials = Credentials::new(access_key, secret_key, None, None, "rustfs-e2e-sse-s3"); + let shared_config = aws_config::defaults(BehaviorVersion::latest()) + .region(Region::new("us-east-1")) + .credentials_provider(credentials) + .endpoint_url(endpoint) + .load() + .await; + Client::from_conf( + aws_sdk_s3::config::Builder::from(&shared_config) + .force_path_style(true) + .build(), + ) +} + fn verify_status_subresource_contract(kubectl: &Kubectl, namespace: &str) -> Result<()> { kubectl .apply_yaml_command(CHECKPOINT_TEST_CRD_YAML) diff --git a/src/console/handlers/tenants.rs b/src/console/handlers/tenants.rs index 1f23588..8f7dcbc 100755 --- a/src/console/handlers/tenants.rs +++ b/src/console/handlers/tenants.rs @@ -327,6 +327,7 @@ fn tenant_from_create_request(req: CreateTenantRequest) -> Result { policies: req.policies.unwrap_or_default(), users: req.users.unwrap_or_default(), buckets: req.buckets.unwrap_or_default(), + tiers: req.tiers.unwrap_or_default(), security_context, ..Default::default() }, @@ -503,6 +504,11 @@ pub async fn update_tenant( updated_fields.push("buckets".to_string()); } + if let Some(tiers) = req.tiers { + tenant.spec.tiers = tiers; + updated_fields.push("tiers".to_string()); + } + if updated_fields.is_empty() { return Err(Error::BadRequest { message: "No fields to update".to_string(), @@ -997,6 +1003,7 @@ spec: policies: None, users: None, buckets: None, + tiers: None, security_context: None, } } diff --git a/src/console/models/tenant.rs b/src/console/models/tenant.rs index 81d6e68..f9c2389 100755 --- a/src/console/models/tenant.rs +++ b/src/console/models/tenant.rs @@ -13,7 +13,7 @@ // limitations under the License. use crate::types::v1alpha1::{ - provisioning::{ProvisioningBucket, ProvisioningPolicy, ProvisioningUser}, + provisioning::{ProvisioningBucket, ProvisioningPolicy, ProvisioningTier, ProvisioningUser}, status::{ ConditionStatus, ConditionType, CurrentState, Reason, Status, canonical_filter_state, canonical_state, certificate, next_actions_for_reason, primary_condition, provisioning, @@ -158,6 +158,7 @@ pub struct CreateTenantRequest { pub policies: Option>, pub users: Option>, pub buckets: Option>, + pub tiers: Option>, /// Optional legacy Pod security context form fields. pub security_context: Option, } @@ -212,6 +213,9 @@ pub struct UpdateTenantRequest { /// Replace bucket provisioning declarations. pub buckets: Option>, + + /// Replace remote tier provisioning declarations. + pub tiers: Option>, } /// Key/value environment variable diff --git a/src/console/openapi.rs b/src/console/openapi.rs index 23581f6..6574f68 100644 --- a/src/console/openapi.rs +++ b/src/console/openapi.rs @@ -55,13 +55,14 @@ use crate::console::models::topology::{ TopologyOverviewResponse, TopologyPod, TopologyPool, TopologyTenant, TopologyTenantSummary, }; use crate::types::v1alpha1::provisioning::{ - BucketAnonymousAccess, BucketLifecycleAbortIncompleteMultipartUpload, - BucketLifecycleExpiration, BucketLifecycleNoncurrentVersionTransition, BucketLifecycleRule, - BucketLifecycleRuleFilter, BucketLifecycleRuleStatus, BucketLifecycleSpec, - BucketLifecycleState, BucketLifecycleTransition, BucketObjectLockConfiguration, - BucketObjectLockConfigurationState, BucketObjectLockRetentionMode, ConfigMapKeyReference, - PolicyDocumentSource, ProvisioningBucket, ProvisioningDeletionPolicy, ProvisioningPolicy, - ProvisioningUser, UserCredentialsSecretRef, + BucketAnonymousAccess, BucketEncryptionMode, BucketEncryptionSpec, BucketEncryptionState, + BucketLifecycleAbortIncompleteMultipartUpload, BucketLifecycleExpiration, + BucketLifecycleNoncurrentVersionTransition, BucketLifecycleRule, BucketLifecycleRuleFilter, + BucketLifecycleRuleStatus, BucketLifecycleSpec, BucketLifecycleState, + BucketLifecycleTransition, BucketObjectLockConfiguration, BucketObjectLockConfigurationState, + BucketObjectLockRetentionMode, ConfigMapKeyReference, PolicyDocumentSource, ProvisioningBucket, + ProvisioningDeletionPolicy, ProvisioningPolicy, ProvisioningTier, ProvisioningTierType, + ProvisioningUser, TierCredentialsSecretRef, UserCredentialsSecretRef, }; use crate::types::v1alpha1::status::provisioning::{ BucketVersioningStatus, ProvisioningBucketStatus, ProvisioningItemState, @@ -130,6 +131,9 @@ use crate::types::v1alpha1::status::provisioning::{ ProvisioningUser, UserCredentialsSecretRef, ProvisioningBucket, + ProvisioningTier, + ProvisioningTierType, + TierCredentialsSecretRef, ProvisioningDeletionPolicy, PolicyDocumentSource, ConfigMapKeyReference, @@ -137,6 +141,9 @@ use crate::types::v1alpha1::status::provisioning::{ BucketObjectLockConfiguration, BucketObjectLockConfigurationState, BucketObjectLockRetentionMode, + BucketEncryptionSpec, + BucketEncryptionState, + BucketEncryptionMode, BucketLifecycleSpec, BucketLifecycleState, BucketLifecycleRule, @@ -798,7 +805,21 @@ mod tests { assert!(schemas.contains_key("BucketLifecycleTransition")); assert!(schemas.contains_key("BucketLifecycleNoncurrentVersionTransition")); assert!(schemas.contains_key("BucketObjectLockConfiguration")); + assert!(schemas.contains_key("BucketEncryptionSpec")); assert!(schemas.contains_key("UserCredentialsSecretRef")); + assert!(schemas.contains_key("ProvisioningTier")); + assert!(schemas.contains_key("ProvisioningTierType")); + assert!(schemas.contains_key("TierCredentialsSecretRef")); + assert_eq!( + spec.pointer("/components/schemas/UpdateTenantRequest/properties/tiers/items/$ref") + .and_then(Value::as_str), + Some("#/components/schemas/ProvisioningTier") + ); + assert_eq!( + spec.pointer("/components/schemas/ProvisioningStatus/properties/tiers/items/$ref") + .and_then(Value::as_str), + Some("#/components/schemas/ProvisioningItemStatus") + ); assert_eq!( spec.pointer("/components/schemas/TenantDetailsResponse/properties/provisioning/$ref") .and_then(Value::as_str), @@ -835,6 +856,13 @@ mod tests { .and_then(Value::as_str), Some("#/components/schemas/BucketObjectLockConfiguration") ); + assert_eq!( + spec.pointer( + "/components/schemas/ProvisioningBucket/properties/encryption/oneOf/1/$ref", + ) + .and_then(Value::as_str), + Some("#/components/schemas/BucketEncryptionSpec") + ); assert_eq!( spec.pointer("/components/schemas/ProvisioningStatus/properties/buckets/items/$ref") .and_then(Value::as_str), diff --git a/src/reconcile/provisioning.rs b/src/reconcile/provisioning.rs index 6380c48..c033a5d 100644 --- a/src/reconcile/provisioning.rs +++ b/src/reconcile/provisioning.rs @@ -14,6 +14,8 @@ use crate::context::{self, Context}; use crate::sts::rustfs_client::{ + BucketEncryptionAlgorithm as AdminEncryptionAlgorithm, + BucketEncryptionConfiguration as AdminEncryptionConfiguration, BucketLifecycleAbortIncompleteMultipartUpload as AdminLifecycleAbort, BucketLifecycleConfiguration as AdminLifecycleConfiguration, BucketLifecycleExpiration as AdminLifecycleExpiration, @@ -25,15 +27,17 @@ use crate::sts::rustfs_client::{ BucketObjectLockDefaultRetention as AdminObjectLockDefaultRetention, BucketObjectLockRetentionMode as AdminObjectLockRetentionMode, BucketObjectLockRetentionPeriod as AdminObjectLockRetentionPeriod, BucketVersioningState, - BucketVersioningUpdate, CreateBucketResult, RustfsAdminClient, RustfsClientError, - canonicalize_bucket_lifecycle_xml, + BucketVersioningUpdate, CreateBucketResult, RustfsAdminClient, RustfsClientError, RustfsTier, + RustfsTierConfig, canonicalize_bucket_lifecycle_xml, }; use crate::types::v1alpha1::provisioning::{ - BucketAnonymousAccess, BucketLifecycleRuleStatus, BucketLifecycleSpec, BucketLifecycleState, + BucketAnonymousAccess, BucketEncryptionMode, BucketEncryptionSpec, BucketEncryptionState, + BucketLifecycleRuleStatus, BucketLifecycleSpec, BucketLifecycleState, BucketObjectLockConfigurationState, BucketObjectLockRetentionMode, - MAX_BUCKET_LIFECYCLE_RULE_ID_LENGTH, MAX_BUCKET_LIFECYCLE_RULES, - MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS, PolicyDocumentSource, ProvisioningBucket, - ProvisioningPolicy, ProvisioningUser, duplicate_user_credentials_secret_names, + MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH, MAX_BUCKET_LIFECYCLE_RULE_ID_LENGTH, + MAX_BUCKET_LIFECYCLE_RULES, MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS, PolicyDocumentSource, + ProvisioningBucket, ProvisioningPolicy, ProvisioningTier, ProvisioningUser, + duplicate_user_credentials_secret_names, }; use crate::types::v1alpha1::status::Reason; use crate::types::v1alpha1::status::provisioning::{ @@ -42,8 +46,9 @@ use crate::types::v1alpha1::status::provisioning::{ ProvisioningUserOwnershipStatus, ProvisioningUserStatus, }; use crate::types::v1alpha1::tenant::Tenant; +use base64::Engine as _; use k8s_openapi::ByteString; -use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use k8s_openapi::api::core::v1::{ConfigMap, EnvVarSource, Secret}; use kube::Api; use kube::api::{Patch, PatchParams}; use serde_json::Value; @@ -54,6 +59,7 @@ use tracing::{info, warn}; const CHECKPOINT_CONFLICT_RETRY: Duration = Duration::from_secs(2); const CHECKPOINT_TRANSIENT_RETRY: Duration = Duration::from_secs(10); +const SSE_S3_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; pub(super) struct ProvisioningReconcileResult { pub status: ProvisioningStatus, @@ -111,6 +117,14 @@ struct UserCredentials { resource_version: Option, } +#[derive(Clone)] +struct TierCredentials { + access_key: String, + secret_key: String, + secret_name: String, + resource_version: Option, +} + enum UserCredentialsCheck { DuplicateSecret, Checked { @@ -194,6 +208,10 @@ impl ProvisioningRun<'_> { self.previous_bucket_status(name).map(AsRef::as_ref) } + fn previous_tier(&self, name: &str) -> Option<&ProvisioningItemStatus> { + self.previous.tiers.iter().find(|item| item.name == name) + } + fn push_policy(&mut self, item: ProvisioningItemStatus) { self.log_item_transition("policy", self.previous_policy(&item.name), &item); if item.state == ProvisioningItemState::Failed.as_str() { @@ -225,6 +243,15 @@ impl ProvisioningRun<'_> { self.status.buckets.push(item); } + fn push_tier(&mut self, item: ProvisioningItemStatus) { + self.log_item_transition("tier", self.previous_tier(&item.name), &item); + if item.state == ProvisioningItemState::Failed.as_str() { + self.failures + .push((reason_from_str(&item.reason), item_message(&item))); + } + self.status.tiers.push(item); + } + fn log_item_transition( &self, item_type: &'static str, @@ -370,6 +397,7 @@ impl ProvisioningRun<'_> { .buckets .iter() .any(|item| is_pending(&item.state)) + || self.status.tiers.iter().any(|item| is_pending(&item.state)) } fn first_pending_message(&self) -> String { @@ -378,6 +406,13 @@ impl ProvisioningRun<'_> { .iter() .find(|item| item.state == ProvisioningItemState::Pending.as_str()) .map(item_message) + .or_else(|| { + self.status + .tiers + .iter() + .find(|item| item.state == ProvisioningItemState::Pending.as_str()) + .map(item_message) + }) .or_else(|| { self.status.buckets.iter().find_map(|item| { (item.state == ProvisioningItemState::Pending.as_str()) @@ -482,6 +517,19 @@ impl ProvisioningRun<'_> { } self.push_bucket(item); } + for tier in &self.tenant.spec.tiers { + let mut item = self.item( + self.previous_tier(&tier.name), + &tier.name, + state.clone(), + reason, + message, + ); + if let Some(previous) = self.previous_tier(&tier.name) { + copy_tier_ownership(&mut item, previous); + } + self.push_tier(item); + } } fn fail_all_active(&mut self, reason: Reason, message: &str) { @@ -509,6 +557,13 @@ impl ProvisioningRun<'_> { self.status.buckets.push(self.retained_bucket(previous)); } } + + let tiers = desired_names(self.tenant.spec.tiers.iter().map(|tier| &tier.name)); + for previous in &self.previous.tiers { + if !tiers.contains(&previous.name) { + self.status.tiers.push(self.retained_item(previous)); + } + } } fn prepare_status(&mut self, phase: ProvisioningPhase) { @@ -516,6 +571,7 @@ impl ProvisioningRun<'_> { self.status.policies.sort_by(|a, b| a.name.cmp(&b.name)); self.status.users.sort_by(|a, b| a.name.cmp(&b.name)); self.status.buckets.sort_by(|a, b| a.name.cmp(&b.name)); + self.status.tiers.sort_by(|a, b| a.name.cmp(&b.name)); if !self.status.is_empty() { self.status.observed_generation = self.tenant.metadata.generation; self.status.phase = Some(phase); @@ -652,6 +708,7 @@ pub(super) async fn reconcile_provisioning( }, }; } + reconcile_tiers(&mut run, &client).await; reconcile_buckets(&mut run, &client).await; run.finish() } @@ -2030,6 +2087,366 @@ fn validate_user_secret_key(secret_key: &str) -> Result<(), String> { Ok(()) } +async fn reconcile_tiers(run: &mut ProvisioningRun<'_>, client: &RustfsAdminClient) { + let tiers = run.tenant.spec.tiers.clone(); + if tiers.is_empty() { + return; + } + + let live = match client.list_tiers().await { + Ok(live) => live, + Err(error) => { + let error = SpecLoadError { + reason: Reason::TierApplyFailed, + message: format!("failed to list RustFS tiers: {error}"), + transient: error.is_transient(), + }; + for tier in &tiers { + let previous = run.previous_tier(&tier.name).cloned(); + let mut item = + run.item_from_spec_error(previous.as_ref(), &tier.name, error.clone()); + if let Some(previous) = previous.as_ref() { + copy_tier_ownership(&mut item, previous); + } + run.push_tier(item); + } + return; + } + }; + + for tier in &tiers { + let credentials = load_tier_secret(run, tier).await; + let live_tier = live.iter().find(|live| live.config.name == tier.name); + let item = reconcile_tier(run, client, tier, live_tier, credentials).await; + run.push_tier(item); + } +} + +async fn reconcile_tier( + run: &mut ProvisioningRun<'_>, + client: &RustfsAdminClient, + tier: &ProvisioningTier, + live: Option<&RustfsTier>, + credentials: Result, +) -> ProvisioningItemStatus { + let previous = run.previous_tier(&tier.name).cloned(); + let generation = run.tenant.metadata.generation; + if let Err(message) = validate_tier_name(&tier.name) { + let item = run.item( + previous.as_ref(), + &tier.name, + ProvisioningItemState::Failed, + Reason::TierApplyFailed, + message, + ); + return finalize_tier_item(item, previous.as_ref(), None, None, generation); + } + + let desired = tier_config(tier); + let desired_hash = tier_config_hash(&desired); + let credentials = match credentials { + Ok(credentials) => credentials, + Err(error) => { + let item = run.item_from_spec_error(previous.as_ref(), &tier.name, error); + return finalize_tier_item( + item, + previous.as_ref(), + Some(desired_hash), + None, + generation, + ); + } + }; + + let ready = |run: &ProvisioningRun<'_>, message: &str| { + run.item( + previous.as_ref(), + &tier.name, + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured, + message, + ) + }; + let (item, applied) = match live { + None => match client + .add_tier( + &desired, + &credentials.access_key, + &credentials.secret_key, + tier.force, + ) + .await + { + Ok(()) => (ready(run, "RustFS tier was registered"), true), + Err(error) if error.is_conflict() => ( + run.item( + previous.as_ref(), + &tier.name, + ProvisioningItemState::Failed, + Reason::TierConflict, + format!( + "RustFS refused to register the tier because remote bucket '{}' prefix '{}' is in use; set force: true only if this prefix belongs to this Tenant: {error}", + desired.bucket, desired.prefix + ), + ), + false, + ), + Err(error) => ( + run.item_from_admin_error( + previous.as_ref(), + &tier.name, + Reason::TierApplyFailed, + error, + format!("failed to register RustFS tier '{}'", tier.name), + ), + false, + ), + }, + Some(live) => { + let differences = tier_config_differences(&live.config, &desired); + if !differences.is_empty() { + ( + run.item( + previous.as_ref(), + &tier.name, + ProvisioningItemState::Failed, + Reason::TierConflict, + format!( + "live RustFS tier differs from spec in {}; RustFS cannot change these fields of a registered tier", + differences.join(", ") + ), + ), + false, + ) + } else if tier_credentials_need_apply(previous.as_ref(), &credentials, &live.access_key) + { + match client + .edit_tier_credentials( + &tier.name, + &credentials.access_key, + &credentials.secret_key, + ) + .await + { + Ok(()) => { + let message = if previous + .as_ref() + .and_then(|item| item.last_applied_hash.as_ref()) + .is_none() + { + "Existing RustFS tier matches spec and was adopted with the Secret credentials" + } else { + "RustFS tier credentials were updated from the Secret" + }; + (ready(run, message), true) + } + Err(error) => ( + run.item_from_admin_error( + previous.as_ref(), + &tier.name, + Reason::TierApplyFailed, + error, + format!("failed to update credentials of RustFS tier '{}'", tier.name), + ), + false, + ), + } + } else { + (ready(run, "RustFS tier matches spec"), false) + } + } + }; + + finalize_tier_item( + item, + previous.as_ref(), + Some(desired_hash), + applied.then_some(&credentials), + generation, + ) +} + +fn tier_config(tier: &ProvisioningTier) -> RustfsTierConfig { + RustfsTierConfig { + tier_type: tier.tier_type.rustfs_type().to_string(), + name: tier.name.clone(), + endpoint: tier.endpoint.clone(), + bucket: tier.bucket.clone(), + prefix: tier.prefix.clone(), + region: tier.region.clone(), + } +} + +fn tier_config_hash(config: &RustfsTierConfig) -> String { + hash_document( + &serde_json::json!([ + config.tier_type, + config.endpoint, + config.bucket, + config.prefix, + config.region, + ]) + .to_string(), + ) +} + +fn tier_config_differences( + live: &RustfsTierConfig, + desired: &RustfsTierConfig, +) -> Vec<&'static str> { + [ + ("type", &live.tier_type, &desired.tier_type), + ("endpoint", &live.endpoint, &desired.endpoint), + ("bucket", &live.bucket, &desired.bucket), + ("prefix", &live.prefix, &desired.prefix), + ("region", &live.region, &desired.region), + ] + .into_iter() + .filter_map(|(field, live, desired)| (live != desired).then_some(field)) + .collect() +} + +/// Credentials cannot be read back from RustFS, so the Secret version applied last is the +/// ownership record; the listed access key still reveals out-of-band changes. +fn tier_credentials_need_apply( + previous: Option<&ProvisioningItemStatus>, + credentials: &TierCredentials, + live_access_key: &str, +) -> bool { + let live_access_key_listed = !live_access_key.is_empty() && live_access_key != "REDACTED"; + if live_access_key_listed && live_access_key != credentials.access_key { + return true; + } + let (Some(previous), Some(resource_version)) = + (previous, credentials.resource_version.as_deref()) + else { + return true; + }; + previous.observed_secret_resource_version.as_deref() != Some(resource_version) + || previous.observed_secret_name.as_deref() != Some(credentials.secret_name.as_str()) + || previous.last_applied_access_key_hash.as_deref() + != Some(access_key_hash(&credentials.access_key).as_str()) +} + +fn finalize_tier_item( + mut item: ProvisioningItemStatus, + previous: Option<&ProvisioningItemStatus>, + desired_hash: Option, + applied_credentials: Option<&TierCredentials>, + generation: Option, +) -> ProvisioningItemStatus { + if let Some(previous) = previous { + copy_tier_ownership(&mut item, previous); + } + if let Some(credentials) = applied_credentials { + item.observed_secret_resource_version = credentials.resource_version.clone(); + item.observed_secret_name = Some(credentials.secret_name.clone()); + item.last_applied_access_key_hash = Some(access_key_hash(&credentials.access_key)); + } + if item.state == ProvisioningItemState::Ready.as_str() { + let changed = applied_credentials.is_some() + || item.last_applied_hash != desired_hash + || item.last_applied_generation.is_none(); + item.last_applied_hash = desired_hash.clone(); + if changed { + item.last_applied_generation = generation; + } + } + item.desired_hash = desired_hash; + item +} + +fn copy_tier_ownership(item: &mut ProvisioningItemStatus, previous: &ProvisioningItemStatus) { + item.desired_hash = previous.desired_hash.clone(); + item.last_applied_hash = previous.last_applied_hash.clone(); + item.last_applied_generation = previous.last_applied_generation; + item.observed_secret_resource_version = previous.observed_secret_resource_version.clone(); + item.observed_secret_name = previous.observed_secret_name.clone(); + item.last_applied_access_key_hash = previous.last_applied_access_key_hash.clone(); +} + +/// Mirrors the RustFS tier name checks; the name is also sent as an S3 XML storage class. +fn validate_tier_name(name: &str) -> Result<(), String> { + if name.is_empty() { + return Err("tier name must not be empty".to_string()); + } + if !name.bytes().all(|byte| { + byte.is_ascii_uppercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-') + }) { + return Err(format!( + "tier name '{name}' must contain only uppercase ASCII letters, digits, '_' and '-'" + )); + } + if name.eq_ignore_ascii_case("STANDARD") || name.eq_ignore_ascii_case("REDUCED_REDUNDANCY") { + return Err(format!( + "tier name '{name}' is a reserved storage class and cannot name a tier" + )); + } + Ok(()) +} + +async fn load_tier_secret( + run: &ProvisioningRun<'_>, + tier: &ProvisioningTier, +) -> Result { + let secret_name = tier.creds_secret.name.as_str(); + let invalid = |message: String| SpecLoadError { + reason: Reason::TierSecretInvalid, + message, + transient: false, + }; + let secret: Secret = run + .ctx + .get(secret_name, run.namespace) + .await + .map_err(|error| { + if context::is_kube_not_found(&error) { + invalid(format!("tier Secret '{secret_name}' was not found")) + } else { + SpecLoadError { + reason: Reason::TierSecretInvalid, + message: format!("failed to read tier Secret '{secret_name}': {error}"), + transient: context::is_transient_kube_error(&error), + } + } + })?; + let data = secret + .data + .as_ref() + .ok_or_else(|| invalid(format!("tier Secret '{secret_name}' has no data")))?; + + Ok(TierCredentials { + access_key: read_tier_secret_value(data, "accesskey", secret_name).map_err(invalid)?, + secret_key: read_tier_secret_value(data, "secretkey", secret_name).map_err(invalid)?, + secret_name: secret_name.to_string(), + resource_version: secret.metadata.resource_version, + }) +} + +const MIN_TIER_CREDENTIAL_LENGTH: usize = 8; + +fn read_tier_secret_value( + data: &BTreeMap, + key: &'static str, + secret_name: &str, +) -> Result { + let raw = data + .get(key) + .ok_or_else(|| format!("tier Secret '{secret_name}' is missing '{key}'"))?; + let value = String::from_utf8(raw.0.clone()) + .map_err(|_| format!("tier Secret '{secret_name}' key '{key}' must be valid UTF-8"))?; + let value = value.trim(); + if value.is_empty() { + return Err(format!("tier Secret '{secret_name}' key '{key}' is empty")); + } + if value.len() < MIN_TIER_CREDENTIAL_LENGTH { + return Err(format!( + "tier Secret '{secret_name}' key '{key}' must be at least {MIN_TIER_CREDENTIAL_LENGTH} characters" + )); + } + Ok(value.to_string()) +} + async fn reconcile_buckets(run: &mut ProvisioningRun<'_>, client: &RustfsAdminClient) { let buckets = run.tenant.spec.buckets.clone(); for bucket in &buckets { @@ -2050,6 +2467,10 @@ async fn reconcile_bucket( .object_lock_configuration .as_ref() .and_then(object_lock_configuration_desired_hash); + status.encryption_desired_hash = bucket + .encryption + .as_ref() + .and_then(|spec| bucket_encryption_desired_hash(run.tenant, spec)); if status.state != ProvisioningItemState::Ready.as_str() { status.lifecycle_desired_hash = bucket.lifecycle.as_ref().and_then(lifecycle_desired_hash); return status; @@ -2075,6 +2496,12 @@ async fn reconcile_bucket( return status; } + status = reconcile_bucket_encryption(run, client, bucket, previous.as_ref(), status).await; + if status.state != ProvisioningItemState::Ready.as_str() { + status.lifecycle_desired_hash = bucket.lifecycle.as_ref().and_then(lifecycle_desired_hash); + return status; + } + let mut completed = reconcile_bucket_lifecycle(run, client, bucket, previous.as_ref(), status.item.clone()) .await; @@ -2586,6 +3013,11 @@ fn validate_bucket_configuration(bucket: &ProvisioningBucket) -> Result<(), (Rea { return Err((Reason::BucketLifecycleApplyFailed, message)); } + if let Some(spec) = bucket.encryption.as_ref() + && let Err(message) = bucket_encryption_configuration(spec) + { + return Err((Reason::BucketEncryptionApplyFailed, message)); + } Ok(()) } @@ -2682,71 +3114,440 @@ fn copy_bucket_control_status( source.object_lock_configuration_desired_hash.clone(); target.object_lock_configuration_last_applied_hash = source.object_lock_configuration_last_applied_hash.clone(); + target.encryption_desired_hash = source.encryption_desired_hash.clone(); + target.encryption_last_applied_hash = source.encryption_last_applied_hash.clone(); } -async fn reconcile_bucket_lifecycle( +async fn reconcile_bucket_encryption( run: &mut ProvisioningRun<'_>, client: &RustfsAdminClient, bucket: &ProvisioningBucket, previous: Option<&ProvisioningBucketStatus>, - base_item: ProvisioningItemStatus, + mut status: ProvisioningBucketStatus, ) -> ProvisioningBucketStatus { - let Some(spec) = bucket.lifecycle.as_ref() else { - let mut status = bucket_status_with_preserved_lifecycle(base_item, previous); - status.lifecycle_desired_hash = None; + let Some(spec) = bucket.encryption.as_ref() else { return status; }; - - let desired = match lifecycle_configuration(spec) { - Ok(configuration) => configuration, + let desired = match bucket_encryption_configuration(spec) + .and_then(|desired| resolve_bucket_encryption(run.tenant, desired)) + { + Ok(desired) => desired, Err(message) => { - let item = replace_bucket_item_state( + status.item = replace_bucket_item_state( run, previous, - &base_item, + &status.item, ProvisioningItemState::Failed, - Reason::BucketLifecycleApplyFailed, + Reason::BucketEncryptionApplyFailed, message, ); - let mut status = bucket_status_with_preserved_lifecycle(item, previous); - status.lifecycle_desired_hash = None; return status; } }; - let desired_document = desired.as_ref().map(AdminLifecycleConfiguration::to_xml); - let desired_hash = match desired_document.as_deref() { - Some(document) => match canonicalize_bucket_lifecycle_xml(document) { - Ok(document) => Some(hash_document(&document)), - Err(error) => { - let item = replace_bucket_item_state( - run, - previous, - &base_item, - ProvisioningItemState::Failed, - Reason::BucketLifecycleApplyFailed, - format!("failed to encode RustFS bucket lifecycle configuration: {error}"), - ); - return bucket_status_with_preserved_lifecycle(item, previous); - } - }, - None => None, - }; - - let live_document = match client - .get_bucket_lifecycle_configuration(&bucket.name) - .await + if desired + .as_ref() + .is_some_and(|desired| desired.algorithm == AdminEncryptionAlgorithm::Aes256) + && let Err(error) = check_sse_s3_provider(run).await { - Ok(document) => document, + let mut item = run.item_from_spec_error(previous, &status.item.name, error); + copy_bucket_configuration_status(&mut item, &status.item); + status.item = item; + return status; + } + + let live = match client.get_bucket_encryption(&bucket.name).await { + Ok(live) => live, Err(error) => { - let item = bucket_item_from_admin_error( + status.item = bucket_item_from_admin_error( run, previous, - &base_item, - Reason::BucketLifecycleApplyFailed, + &status.item, + Reason::BucketEncryptionApplyFailed, error, - "failed to read RustFS bucket lifecycle configuration", + "failed to read RustFS bucket encryption", ); - return finalize_bucket_lifecycle_status(item, previous, desired_hash, false); + return status; + } + }; + let live_hash = live + .as_ref() + .map(|configuration| hash_document(&configuration.to_xml())); + let last_applied_hash = previous.and_then(|item| item.encryption_last_applied_hash.as_deref()); + + let Some(desired) = desired else { + status.encryption_desired_hash = None; + let Some(live_hash) = live_hash else { + status.encryption_last_applied_hash = None; + status.item.message = Some(append_bucket_message( + &status.item, + "RustFS bucket encryption is absent", + )); + return status; + }; + if last_applied_hash != Some(live_hash.as_str()) { + status.item = replace_bucket_item_state( + run, + previous, + &status.item, + ProvisioningItemState::Failed, + Reason::BucketEncryptionConflict, + bucket_encryption_conflict_message(last_applied_hash, "delete"), + ); + return status; + } + if let Err(error) = client.delete_bucket_encryption(&bucket.name).await { + status.item = bucket_item_from_admin_error( + run, + previous, + &status.item, + Reason::BucketEncryptionApplyFailed, + error, + "failed to remove RustFS bucket encryption", + ); + return status; + } + status.encryption_last_applied_hash = None; + status.item.message = Some(append_bucket_message( + &status.item, + "RustFS bucket encryption was removed", + )); + return status; + }; + + if live.as_ref() == Some(&desired) { + status.encryption_last_applied_hash = live_hash; + status.item.message = Some(append_bucket_message( + &status.item, + "RustFS bucket encryption matches spec", + )); + return status; + } + if let Some(live_hash) = live_hash.as_deref() + && last_applied_hash != Some(live_hash) + { + status.item = replace_bucket_item_state( + run, + previous, + &status.item, + ProvisioningItemState::Failed, + Reason::BucketEncryptionConflict, + bucket_encryption_conflict_message(last_applied_hash, "replace"), + ); + return status; + } + + if let Err(error) = client.put_bucket_encryption(&bucket.name, &desired).await { + status.item = bucket_item_from_admin_error( + run, + previous, + &status.item, + Reason::BucketEncryptionApplyFailed, + error, + "failed to apply RustFS bucket encryption", + ); + return status; + } + let verified = match client.get_bucket_encryption(&bucket.name).await { + Ok(Some(verified)) if verified == desired => verified, + Ok(_) => { + status.item = replace_bucket_item_state( + run, + previous, + &status.item, + ProvisioningItemState::Failed, + Reason::BucketEncryptionApplyFailed, + "RustFS bucket encryption did not match spec after update", + ); + return status; + } + Err(error) => { + status.item = bucket_item_from_admin_error( + run, + previous, + &status.item, + Reason::BucketEncryptionApplyFailed, + error, + "failed to verify RustFS bucket encryption after update", + ); + return status; + } + }; + status.encryption_last_applied_hash = Some(hash_document(&verified.to_xml())); + status.item.message = Some(append_bucket_message( + &status.item, + "RustFS bucket encryption was applied", + )); + status +} + +fn bucket_encryption_conflict_message(last_applied_hash: Option<&str>, action: &str) -> String { + if last_applied_hash.is_some() { + format!( + "Live RustFS bucket encryption changed since the operator last applied it; refusing to {action} it" + ) + } else { + format!( + "Live RustFS bucket encryption is not owned by this status; refusing to {action} it" + ) + } +} + +/// `Ok(None)` is a valid `Absent` spec. +fn bucket_encryption_configuration( + spec: &BucketEncryptionSpec, +) -> Result, String> { + match spec.state() { + BucketEncryptionState::Absent => { + if spec.mode.is_some() || spec.kms_key_id.is_some() { + return Err( + "Absent bucket encryption must not contain mode or kmsKeyId".to_string() + ); + } + Ok(None) + } + BucketEncryptionState::Present => { + let algorithm = match spec.mode { + Some(BucketEncryptionMode::SseS3) => AdminEncryptionAlgorithm::Aes256, + Some(BucketEncryptionMode::SseKms) => AdminEncryptionAlgorithm::AwsKms, + None => return Err("Present bucket encryption must contain a mode".to_string()), + }; + if let Some(key_id) = spec.kms_key_id.as_deref() { + if algorithm != AdminEncryptionAlgorithm::AwsKms { + return Err("bucket encryption kmsKeyId requires mode SSE-KMS".to_string()); + } + if !is_valid_kms_key_id(key_id) { + return Err(format!( + "bucket encryption kmsKeyId must contain between 1 and {MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH} characters from [A-Za-z0-9._:/+=@-]" + )); + } + } + Ok(Some(AdminEncryptionConfiguration { + algorithm, + kms_master_key_id: spec.kms_key_id.clone(), + bucket_key_enabled: false, + })) + } + } +} + +fn is_valid_kms_key_id(key_id: &str) -> bool { + !key_id.is_empty() + && key_id.len() <= MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH as usize + && key_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._:/+=@-".contains(&byte)) +} + +/// SSE-KMS requires Tenant KMS. Without `kmsKeyId` it names the Tenant's `defaultKeyId`, the key +/// RustFS records for such a request, so a changed default key is a spec change. +fn resolve_bucket_encryption( + tenant: &Tenant, + desired: Option, +) -> Result, String> { + let Some(mut desired) = desired else { + return Ok(None); + }; + if desired.algorithm != AdminEncryptionAlgorithm::AwsKms { + return Ok(Some(desired)); + } + let Some(encryption) = tenant + .spec + .encryption + .as_ref() + .filter(|encryption| encryption.enabled) + else { + return Err( + "SSE-KMS bucket encryption requires spec.encryption.enabled on the Tenant".to_string(), + ); + }; + if desired.kms_master_key_id.is_none() { + let Some(default_key_id) = encryption + .default_key_id + .as_deref() + .filter(|key_id| !key_id.is_empty()) + else { + return Err( + "SSE-KMS bucket encryption without kmsKeyId requires spec.encryption.defaultKeyId on the Tenant" + .to_string(), + ); + }; + if !is_valid_kms_key_id(default_key_id) { + return Err(format!( + "Tenant spec.encryption.defaultKeyId must contain between 1 and {MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH} characters from [A-Za-z0-9._:/+=@-] to be used as a bucket default" + )); + } + desired.kms_master_key_id = Some(default_key_id.to_string()); + } + Ok(Some(desired)) +} + +/// Without Tenant KMS, RustFS encrypts SSE-S3 objects with `RUSTFS_SSE_S3_MASTER_KEY` and +/// rejects every upload under an SSE-S3 bucket default when that key is missing or invalid. +async fn check_sse_s3_provider(run: &ProvisioningRun<'_>) -> Result<(), SpecLoadError> { + if run + .tenant + .spec + .encryption + .as_ref() + .is_some_and(|encryption| encryption.enabled) + { + return Ok(()); + } + let invalid = |message: String| SpecLoadError { + reason: Reason::BucketEncryptionApplyFailed, + message, + transient: false, + }; + let Some(env) = run + .tenant + .spec + .env + .iter() + .find(|env| env.name == SSE_S3_MASTER_KEY_ENV) + else { + return Err(invalid(format!( + "SSE-S3 bucket encryption requires spec.encryption.enabled or {SSE_S3_MASTER_KEY_ENV} in spec.env on the Tenant" + ))); + }; + let value = match (env.value.as_deref(), env.value_from.as_ref()) { + (Some(value), None) => value.to_string(), + ( + None, + Some(EnvVarSource { + secret_key_ref: Some(selector), + .. + }), + ) => read_sse_s3_master_key_secret(run, &selector.name, &selector.key).await?, + _ => { + return Err(invalid(format!( + "{SSE_S3_MASTER_KEY_ENV} in spec.env must be set by value or from a Secret key" + ))); + } + }; + match base64::engine::general_purpose::STANDARD.decode(value.trim()) { + Ok(key) if key.len() == 32 => Ok(()), + _ => Err(invalid(format!( + "{SSE_S3_MASTER_KEY_ENV} must be a base64-encoded 32-byte key for SSE-S3 bucket encryption" + ))), + } +} + +async fn read_sse_s3_master_key_secret( + run: &ProvisioningRun<'_>, + secret_name: &str, + key: &str, +) -> Result { + let invalid = |message: String| SpecLoadError { + reason: Reason::BucketEncryptionApplyFailed, + message, + transient: false, + }; + let secret: Secret = run + .ctx + .get(secret_name, run.namespace) + .await + .map_err(|error| { + if context::is_kube_not_found(&error) { + invalid(format!( + "{SSE_S3_MASTER_KEY_ENV} Secret '{secret_name}' was not found" + )) + } else { + SpecLoadError { + reason: Reason::BucketEncryptionApplyFailed, + message: format!( + "failed to read {SSE_S3_MASTER_KEY_ENV} Secret '{secret_name}': {error}" + ), + transient: context::is_transient_kube_error(&error), + } + } + })?; + let raw = secret + .data + .as_ref() + .and_then(|data| data.get(key)) + .ok_or_else(|| { + invalid(format!( + "{SSE_S3_MASTER_KEY_ENV} Secret '{secret_name}' has no key '{key}'" + )) + })?; + String::from_utf8(raw.0.clone()).map_err(|_| { + invalid(format!( + "{SSE_S3_MASTER_KEY_ENV} Secret '{secret_name}' key '{key}' must be valid UTF-8" + )) + }) +} + +fn bucket_encryption_desired_hash(tenant: &Tenant, spec: &BucketEncryptionSpec) -> Option { + bucket_encryption_configuration(spec) + .and_then(|desired| resolve_bucket_encryption(tenant, desired)) + .ok() + .flatten() + .map(|configuration| hash_document(&configuration.to_xml())) +} + +async fn reconcile_bucket_lifecycle( + run: &mut ProvisioningRun<'_>, + client: &RustfsAdminClient, + bucket: &ProvisioningBucket, + previous: Option<&ProvisioningBucketStatus>, + base_item: ProvisioningItemStatus, +) -> ProvisioningBucketStatus { + let Some(spec) = bucket.lifecycle.as_ref() else { + let mut status = bucket_status_with_preserved_lifecycle(base_item, previous); + status.lifecycle_desired_hash = None; + return status; + }; + + let desired = match lifecycle_configuration(spec) { + Ok(configuration) => configuration, + Err(message) => { + let item = replace_bucket_item_state( + run, + previous, + &base_item, + ProvisioningItemState::Failed, + Reason::BucketLifecycleApplyFailed, + message, + ); + let mut status = bucket_status_with_preserved_lifecycle(item, previous); + status.lifecycle_desired_hash = None; + return status; + } + }; + let desired_document = desired.as_ref().map(AdminLifecycleConfiguration::to_xml); + let desired_hash = match desired_document.as_deref() { + Some(document) => match canonicalize_bucket_lifecycle_xml(document) { + Ok(document) => Some(hash_document(&document)), + Err(error) => { + let item = replace_bucket_item_state( + run, + previous, + &base_item, + ProvisioningItemState::Failed, + Reason::BucketLifecycleApplyFailed, + format!("failed to encode RustFS bucket lifecycle configuration: {error}"), + ); + return bucket_status_with_preserved_lifecycle(item, previous); + } + }, + None => None, + }; + + let live_document = match client + .get_bucket_lifecycle_configuration(&bucket.name) + .await + { + Ok(document) => document, + Err(error) => { + let item = bucket_item_from_admin_error( + run, + previous, + &base_item, + Reason::BucketLifecycleApplyFailed, + error, + "failed to read RustFS bucket lifecycle configuration", + ); + return finalize_bucket_lifecycle_status(item, previous, desired_hash, false); } }; let live_hash = match live_document.as_deref() { @@ -2892,6 +3693,8 @@ async fn reconcile_bucket_lifecycle( /// Tenant is provisioned. RustFS rejects a lifecycle that references an unknown tier with a /// generic 400, which would leave the bucket `Failed` until the spec changes, so a missing tier /// is reported as `Pending` and retried instead. +/// A referenced tier declared in `spec.tiers` must also be `Ready` in the current run; its +/// failure or pending state is propagated to the bucket before any lifecycle write. async fn lifecycle_tiers_pending( run: &mut ProvisioningRun<'_>, client: &RustfsAdminClient, @@ -2917,26 +3720,72 @@ async fn lifecycle_tiers_pending( return None; } - let registered = match client.list_tier_names().await { - Ok(names) => names, - Err(error) => { - return Some(bucket_item_from_admin_error( + // A tier declared in spec.tiers must be Ready in this run: a live tier with the same name + // may point at a different remote destination than the one the Tenant declares. + let blocked = referenced + .iter() + .filter_map(|name| { + run.status + .tiers + .iter() + .find(|tier| tier.name == *name) + .filter(|tier| tier.state != ProvisioningItemState::Ready.as_str()) + }) + .map(|tier| (tier.name.clone(), tier.state.clone(), tier.reason.clone())) + .collect::>(); + if !blocked.is_empty() { + let failed = blocked + .iter() + .any(|(_, state, _)| state == ProvisioningItemState::Failed.as_str()); + let message = format!( + "bucket lifecycle references declared tier(s) that are not Ready: {}", + blocked + .iter() + .map(|(name, state, reason)| format!("{name} ({state}: {reason})")) + .collect::>() + .join(", ") + ); + if failed { + return Some(replace_bucket_item_state( run, previous, base_item, + ProvisioningItemState::Failed, Reason::BucketLifecycleApplyFailed, - error, - "failed to list RustFS tiers referenced by bucket lifecycle", + message, )); } - }; - let missing = referenced - .into_iter() - .filter(|name| !registered.contains(*name)) - .collect::>(); - if missing.is_empty() { - return None; - } + run.request_retry(message.clone()); + return Some(replace_bucket_item_state( + run, + previous, + base_item, + ProvisioningItemState::Pending, + Reason::ProvisioningPending, + message, + )); + } + + let registered = match client.list_tier_names().await { + Ok(names) => names, + Err(error) => { + return Some(bucket_item_from_admin_error( + run, + previous, + base_item, + Reason::BucketLifecycleApplyFailed, + error, + "failed to list RustFS tiers referenced by bucket lifecycle", + )); + } + }; + let missing = referenced + .into_iter() + .filter(|name| !registered.contains(*name)) + .collect::>(); + if missing.is_empty() { + return None; + } let message = format!( "waiting for RustFS tier(s) {} referenced by bucket lifecycle", @@ -3553,6 +4402,7 @@ fn has_active_spec(tenant: &Tenant) -> bool { !tenant.spec.policies.is_empty() || !tenant.spec.users.is_empty() || !tenant.spec.buckets.is_empty() + || !tenant.spec.tiers.is_empty() } fn desired_names<'a>(names: impl Iterator) -> BTreeSet { @@ -3743,8 +4593,13 @@ fn reason_from_str(reason: &str) -> Reason { "BucketObjectLockConfigurationConflict" => Reason::BucketObjectLockConfigurationConflict, "BucketPolicyApplyFailed" => Reason::BucketPolicyApplyFailed, "BucketPolicyConflict" => Reason::BucketPolicyConflict, + "BucketEncryptionApplyFailed" => Reason::BucketEncryptionApplyFailed, + "BucketEncryptionConflict" => Reason::BucketEncryptionConflict, "BucketLifecycleApplyFailed" => Reason::BucketLifecycleApplyFailed, "BucketLifecycleConflict" => Reason::BucketLifecycleConflict, + "TierSecretInvalid" => Reason::TierSecretInvalid, + "TierApplyFailed" => Reason::TierApplyFailed, + "TierConflict" => Reason::TierConflict, _ => Reason::ProvisioningFailed, } } @@ -3765,6 +4620,7 @@ mod tests { }; use http_body_util::BodyExt; use k8s_openapi::ByteString; + use k8s_openapi::api::core::v1::{EnvVar, SecretKeySelector}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::{Client, client::Body as KubeBody}; use std::convert::Infallible; @@ -3802,8 +4658,10 @@ mod tests { existing: bool, ignore_versioning_put: bool, ignore_object_lock_put: bool, + ignore_encryption_put: bool, versioning: Arc>, object_lock: Arc>>, + encryption: Arc>>, requests: Arc>>, put_requests: Arc>>, create_object_lock_header: Arc>>, @@ -3823,8 +4681,10 @@ mod tests { existing: false, ignore_versioning_put: false, ignore_object_lock_put: false, + ignore_encryption_put: false, versioning: Arc::new(Mutex::new(BucketVersioningState::Unversioned)), object_lock: Arc::new(Mutex::new(None)), + encryption: Arc::new(Mutex::new(None)), requests: Arc::new(Mutex::new(Vec::new())), put_requests: Arc::new(Mutex::new(Vec::new())), create_object_lock_header: Arc::new(Mutex::new(None)), @@ -3981,6 +4841,49 @@ mod tests { } StatusCode::OK.into_response() } + (axum::http::Method::GET, "encryption=") => { + match capture.encryption.lock().await.clone() { + // RustFS omits the namespace on GetBucketEncryption. + Some(configuration) => ( + StatusCode::OK, + configuration + .to_xml() + .replace(" xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"", ""), + ) + .into_response(), + // RustFS answers a missing configuration with 400, not 404. + None => ( + StatusCode::BAD_REQUEST, + "ServerSideEncryptionConfigurationNotFoundErrorThe server-side encryption configuration was not found.", + ) + .into_response(), + } + } + (axum::http::Method::PUT, "encryption=") => { + let body = capture_bucket_control_put(&capture, &query, request).await; + let kms = body.contains("aws:kms"); + let kms_master_key_id = body + .split_once("") + .and_then(|(_, tail)| tail.split_once("")) + .map(|(key, _)| key.to_string()) + .or_else(|| kms.then(|| "rustfs-default-key".to_string())); + if !capture.ignore_encryption_put { + *capture.encryption.lock().await = Some(AdminEncryptionConfiguration { + algorithm: if kms { + AdminEncryptionAlgorithm::AwsKms + } else { + AdminEncryptionAlgorithm::Aes256 + }, + kms_master_key_id, + bucket_key_enabled: false, + }); + } + StatusCode::OK.into_response() + } + (axum::http::Method::DELETE, "encryption=") => { + *capture.encryption.lock().await = None; + StatusCode::NO_CONTENT.into_response() + } _ => StatusCode::METHOD_NOT_ALLOWED.into_response(), } } @@ -6235,56 +7138,203 @@ mod tests { server.abort(); } - #[tokio::test] - async fn matching_live_object_lock_configuration_is_adopted_without_a_write() { - let live = AdminObjectLockConfiguration { - default_retention: Some(AdminObjectLockDefaultRetention { - mode: AdminObjectLockRetentionMode::Compliance, - period: AdminObjectLockRetentionPeriod::Days(30), + fn sse_s3() -> AdminEncryptionConfiguration { + AdminEncryptionConfiguration { + algorithm: AdminEncryptionAlgorithm::Aes256, + kms_master_key_id: None, + bucket_key_enabled: false, + } + } + + fn encryption_bucket(encryption: serde_json::Value) -> ProvisioningBucket { + serde_json::from_value(serde_json::json!({ + "name": "secure-bucket", + "encryption": encryption + })) + .expect("encryption bucket should deserialize") + } + + // base64 of the 32 bytes "0123456789abcdef0123456789abcdef". + const SSE_S3_TEST_MASTER_KEY: &str = "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="; + + fn sse_s3_master_key_env(value: Option<&str>, secret_key: Option<&str>) -> EnvVar { + EnvVar { + name: "RUSTFS_SSE_S3_MASTER_KEY".to_string(), + value: value.map(str::to_string), + value_from: secret_key.map(|key| EnvVarSource { + secret_key_ref: Some(SecretKeySelector { + name: "sse-s3".to_string(), + key: key.to_string(), + optional: Some(false), + }), + ..Default::default() }), - }; + } + } + + fn sse_s3_tenant() -> Tenant { + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let mut tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + tenant + .spec + .env + .push(sse_s3_master_key_env(Some(SSE_S3_TEST_MASTER_KEY), None)); + tenant + } + + fn kms_tenant(default_key_id: Option<&str>) -> Tenant { + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let mut tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + tenant.spec.encryption = Some(crate::types::v1alpha1::encryption::EncryptionConfig { + enabled: true, + default_key_id: default_key_id.map(str::to_string), + ..Default::default() + }); + tenant + } + + fn sse_kms(key_id: &str) -> AdminEncryptionConfiguration { + AdminEncryptionConfiguration { + algorithm: AdminEncryptionAlgorithm::AwsKms, + kms_master_key_id: Some(key_id.to_string()), + bucket_key_enabled: false, + } + } + + fn sse_s3_secret_kube_context(key: &'static str, value: &'static str) -> Context { + let kube_service = service_fn(move |request: http::Request| async move { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/storage/secrets/sse-s3" + ); + let secret = Secret { + metadata: ObjectMeta { + name: Some("sse-s3".to_string()), + namespace: Some("storage".to_string()), + ..Default::default() + }, + data: Some(BTreeMap::from([( + key.to_string(), + ByteString(value.as_bytes().to_vec()), + )])), + ..Default::default() + }; + Ok::<_, Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(KubeBody::from( + serde_json::to_vec(&secret).expect("Secret should serialize"), + )) + .expect("response should build"), + ) + }); + Context::new(Client::new(kube_service, "default")) + } + + async fn encryption_requests(capture: &BucketControlCapture) -> usize { + capture + .requests + .lock() + .await + .iter() + .filter(|request| request.ends_with("encryption=")) + .count() + } + + fn previous_encryption_owner(hash: String) -> ProvisioningBucketStatus { + let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( + "secure-bucket", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + )); + previous.encryption_last_applied_hash = Some(hash); + previous + } + + async fn requests_for(capture: &BucketControlCapture, request: &str) -> usize { + capture + .requests + .lock() + .await + .iter() + .filter(|logged| logged.as_str() == request) + .count() + } + + #[tokio::test] + async fn sse_s3_bucket_encryption_is_applied_to_a_new_bucket_and_owned() { + let capture = BucketControlCapture::default(); + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let tenant = sse_s3_tenant(); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("RustFS bucket encryption was applied")) + ); + assert_eq!(capture.encryption.lock().await.clone(), Some(sse_s3())); + let owned = Some(hash_document(&sse_s3().to_xml())); + assert_eq!(item.encryption_desired_hash, owned); + assert_eq!(item.encryption_last_applied_hash, owned); + let put_requests = capture.put_requests.lock().await; + let put = put_requests + .iter() + .find(|request| request.query == "encryption=") + .expect("encryption PUT is captured"); + assert_eq!(put.body, sse_s3().to_xml()); + assert!(put.content_md5.is_some()); + assert!(put.authorization.as_deref().is_some_and(|authorization| { + authorization.contains( + "SignedHeaders=content-md5;content-type;host;x-amz-content-sha256;x-amz-date", + ) + })); + server.abort(); + } + + #[tokio::test] + async fn matching_live_bucket_encryption_is_adopted_without_a_write() { let capture = BucketControlCapture { existing: true, - versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), - object_lock: Arc::new(Mutex::new(Some(live))), + encryption: Arc::new(Mutex::new(Some(sse_s3()))), ..Default::default() }; let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = sse_s3_tenant(); let mut run = empty_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &object_lock_bucket()).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); assert_eq!( - item.object_lock_configuration_desired_hash, - item.object_lock_configuration_last_applied_hash - ); - assert!( - !capture - .requests - .lock() - .await - .iter() - .any(|request| request == "PUT object-lock=") + item.encryption_last_applied_hash, + Some(hash_document(&sse_s3().to_xml())) ); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 0); server.abort(); } #[tokio::test] - async fn omitted_object_lock_configuration_leaves_live_retention_unmanaged() { - let live = AdminObjectLockConfiguration { - default_retention: Some(AdminObjectLockDefaultRetention { - mode: AdminObjectLockRetentionMode::Governance, - period: AdminObjectLockRetentionPeriod::Days(7), - }), - }; + async fn omitted_bucket_encryption_makes_no_encryption_calls() { let capture = BucketControlCapture { existing: true, - versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), - object_lock: Arc::new(Mutex::new(Some(live.clone()))), + encryption: Arc::new(Mutex::new(Some(sse_s3()))), ..Default::default() }; let (client, server) = bucket_control_test_client(capture.clone()).await; @@ -6292,1018 +7342,2322 @@ mod tests { let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); - let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ - "name": "secure-bucket", - "objectLock": true - })) - .expect("bucket should deserialize"); + let bucket: ProvisioningBucket = + serde_json::from_value(serde_json::json!({"name": "secure-bucket"})) + .expect("bucket should deserialize"); let item = reconcile_bucket(&mut run, &client, &bucket).await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.object_lock.lock().await.as_ref(), Some(&live)); - assert!(item.object_lock_configuration_desired_hash.is_none()); assert!( !capture .requests .lock() .await .iter() - .any(|request| request == "PUT object-lock=") + .any(|request| request.ends_with("encryption=")) ); + assert_eq!(capture.encryption.lock().await.clone(), Some(sse_s3())); server.abort(); } #[tokio::test] - async fn owned_object_lock_default_retention_can_be_removed() { - let live = AdminObjectLockConfiguration { - default_retention: Some(AdminObjectLockDefaultRetention { - mode: AdminObjectLockRetentionMode::Compliance, - period: AdminObjectLockRetentionPeriod::Days(30), - }), + async fn unowned_live_bucket_encryption_is_not_replaced_or_removed() { + let foreign = AdminEncryptionConfiguration { + algorithm: AdminEncryptionAlgorithm::AwsKms, + kms_master_key_id: Some("foreign-key".to_string()), + bucket_key_enabled: false, }; + for spec in [ + serde_json::json!({"mode": "SSE-S3"}), + serde_json::json!({"state": "Absent"}), + ] { + let capture = BucketControlCapture { + existing: true, + encryption: Arc::new(Mutex::new(Some(foreign.clone()))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let tenant = sse_s3_tenant(); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &encryption_bucket(spec.clone())).await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str(), "{spec}"); + assert_eq!(item.reason, Reason::BucketEncryptionConflict.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("not owned by this status")) + ); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 0); + assert_eq!(requests_for(&capture, "DELETE encryption=").await, 0); + assert_eq!( + capture.encryption.lock().await.clone(), + Some(foreign.clone()) + ); + server.abort(); + } + } + + #[tokio::test] + async fn owned_bucket_encryption_follows_a_spec_change_to_sse_kms() { let capture = BucketControlCapture { existing: true, - versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), - object_lock: Arc::new(Mutex::new(Some(live.clone()))), + encryption: Arc::new(Mutex::new(Some(sse_s3()))), ..Default::default() }; let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = kms_tenant(Some("rustfs-default-key")); let mut run = empty_run(&ctx, &tenant); - let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( - "secure-bucket", - ProvisioningItemState::Ready, - Reason::ProvisioningConfigured.as_str(), - )); - previous.object_lock_configuration_last_applied_hash = Some(hash_document(&live.to_xml())); - run.previous.buckets.push(previous); - let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ - "name": "secure-bucket", - "objectLock": true, - "objectLockConfiguration": { - "state": "Absent" - } - })) - .expect("bucket should deserialize"); + run.previous + .buckets + .push(previous_encryption_owner(hash_document(&sse_s3().to_xml()))); - let item = reconcile_bucket(&mut run, &client, &bucket).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + let applied = AdminEncryptionConfiguration { + algorithm: AdminEncryptionAlgorithm::AwsKms, + kms_master_key_id: Some("rustfs-default-key".to_string()), + bucket_key_enabled: false, + }; assert_eq!( - capture.object_lock.lock().await.as_ref(), - Some(&AdminObjectLockConfiguration::enabled_without_default_retention()) + capture.encryption.lock().await.clone(), + Some(applied.clone()) ); assert_eq!( - item.object_lock_configuration_desired_hash, - item.object_lock_configuration_last_applied_hash + item.encryption_last_applied_hash, + Some(hash_document(&applied.to_xml())) ); assert_eq!( - capture - .requests - .lock() - .await - .iter() - .filter(|request| request.as_str() == "PUT object-lock=") - .count(), - 1 + item.encryption_desired_hash, + item.encryption_last_applied_hash ); + drop(run); + + // The default key is not written again on the next pass. + let mut run = empty_run(&ctx, &tenant); + run.previous.buckets.push(item.clone()); + let again = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; + assert_eq!(again.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 1); server.abort(); } #[tokio::test] - async fn object_lock_update_is_verified_after_a_success_response() { + async fn absent_bucket_encryption_removes_only_an_owned_configuration() { let capture = BucketControlCapture { existing: true, - ignore_object_lock_put: true, - versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), + encryption: Arc::new(Mutex::new(Some(sse_s3()))), ..Default::default() }; - let (client, server) = bucket_control_test_client(capture).await; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); + run.previous + .buckets + .push(previous_encryption_owner(hash_document(&sse_s3().to_xml()))); - let item = reconcile_bucket(&mut run, &client, &object_lock_bucket()).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"state": "Absent"})), + ) + .await; - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketObjectLockApplyFailed.as_str()); - assert_eq!(item.object_lock, Some(false)); - assert!(item.object_lock_configuration_last_applied_hash.is_none()); + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.encryption.lock().await.clone(), None); + assert_eq!(item.encryption_desired_hash, None); + assert_eq!(item.encryption_last_applied_hash, None); + assert_eq!(requests_for(&capture, "DELETE encryption=").await, 1); server.abort(); } - #[test] - fn object_lock_configuration_requires_explicit_object_lock_enablement() { - let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ - "name": "secure-bucket", - "objectLockConfiguration": { - "mode": "Compliance", - "days": 30 - } - })) - .expect("older CRDs can admit the structurally valid payload"); - - let (reason, message) = validate_bucket_configuration(&bucket) - .expect_err("runtime validation must protect clusters with an older CRD"); - - assert_eq!(reason, Reason::BucketObjectLockApplyFailed); - assert_eq!(message, "objectLockConfiguration requires objectLock: true"); - } - - #[test] - fn object_lock_configuration_maps_modes_and_enforces_runtime_boundaries() { - let configuration = |mode, days| { - serde_json::from_value(serde_json::json!({ - "mode": mode, - "days": days - })) - .expect("Object Lock configuration should deserialize") + #[tokio::test] + async fn changed_owned_bucket_encryption_is_a_conflict() { + let capture = BucketControlCapture { + existing: true, + encryption: Arc::new(Mutex::new(Some(AdminEncryptionConfiguration { + bucket_key_enabled: true, + ..sse_s3() + }))), + ..Default::default() }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let tenant = sse_s3_tenant(); + let mut run = empty_run(&ctx, &tenant); + run.previous + .buckets + .push(previous_encryption_owner(hash_document(&sse_s3().to_xml()))); - for (mode, expected_mode) in [ - ("Governance", AdminObjectLockRetentionMode::Governance), - ("Compliance", AdminObjectLockRetentionMode::Compliance), - ] { - for days in [1, MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS as i32] { - assert_eq!( - object_lock_configuration(&configuration(mode, days)).unwrap(), - AdminObjectLockConfiguration { - default_retention: Some(AdminObjectLockDefaultRetention { - mode: expected_mode, - period: AdminObjectLockRetentionPeriod::Days(days), - }), - } - ); - } - } - for days in [0, MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS as i32 + 1] { - let error = object_lock_configuration(&configuration("Governance", days)) - .expect_err("out-of-range retention must fail"); - assert!(error.contains("must be between")); - } - } + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; - #[test] - fn object_lock_reconcile_action_protects_owned_and_unmanaged_configurations() { - assert_eq!( - object_lock_configuration_reconcile_action(None, None, "desired", false), - ObjectLockConfigurationReconcileAction::Apply - ); - assert_eq!( - object_lock_configuration_reconcile_action(None, Some("desired"), "desired", false), - ObjectLockConfigurationReconcileAction::Ready( - "RustFS bucket Object Lock configuration matches spec" + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionConflict.as_str()); + assert!( + item.message.as_deref().is_some_and( + |message| message.contains("changed since the operator last applied it") ) ); assert_eq!( - object_lock_configuration_reconcile_action( - Some("owned"), - Some("owned"), - "changed", - false, - ), - ObjectLockConfigurationReconcileAction::Apply - ); - assert_eq!( - object_lock_configuration_reconcile_action( - None, - Some("enabled-without-retention"), - "desired", - true, - ), - ObjectLockConfigurationReconcileAction::Apply + item.encryption_last_applied_hash, + Some(hash_document(&sse_s3().to_xml())) ); - assert!(matches!( - object_lock_configuration_reconcile_action(None, Some("external"), "desired", false,), - ObjectLockConfigurationReconcileAction::Failed(_) - )); - assert!(matches!( - object_lock_configuration_reconcile_action( - Some("owned"), - Some("external-drift"), - "desired", - false, - ), - ObjectLockConfigurationReconcileAction::Failed(_) - )); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 0); + server.abort(); } #[tokio::test] - async fn create_bucket_503_marks_pending_and_retries() { - let router = Router::new().route( - "/app-data", - put(|| async { StatusCode::SERVICE_UNAVAILABLE }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + async fn sse_kms_bucket_encryption_requires_tenant_kms() { + let capture = BucketControlCapture { + existing: true, + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), + + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS", "kmsKeyId": "app-key"})), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionApplyFailed.as_str()); + assert!( + !capture + .requests + .lock() + .await + .iter() + .any(|request| request.ends_with("encryption=")) + ); + server.abort(); + } + + #[tokio::test] + async fn bucket_encryption_update_is_verified_after_a_success_response() { + let capture = BucketControlCapture { + existing: true, + ignore_encryption_put: true, ..Default::default() }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let tenant = sse_s3_tenant(); + let mut run = empty_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &bucket).await; - assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); - assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); - run.push_bucket(item); + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; - match run.finish().outcome { - ProvisioningOutcome::Retry { persist_status, .. } => assert!(persist_status), - _ => panic!("transient bucket create must retry"), - } + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionApplyFailed.as_str()); + assert_eq!(item.encryption_last_applied_hash, None); server.abort(); } #[tokio::test] - async fn create_bucket_400_fails_without_retry() { - let router = Router::new().route("/app-data", put(|| async { StatusCode::BAD_REQUEST })); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let ctx = empty_kube_context(); + async fn sse_s3_bucket_encryption_requires_an_encryption_provider() { let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), + let without_provider = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut invalid_key = sse_s3_tenant(); + invalid_key.spec.env = vec![sse_s3_master_key_env(Some("c2hvcnQ="), None)]; + for (tenant, expected) in [ + ( + without_provider, + "requires spec.encryption.enabled or RUSTFS_SSE_S3_MASTER_KEY", + ), + (invalid_key, "must be a base64-encoded 32-byte key"), + ] { + let capture = BucketControlCapture { + existing: true, + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionApplyFailed.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains(expected)), + "{:?}", + item.message + ); + assert_eq!(encryption_requests(&capture).await, 0); + server.abort(); + } + } + + #[tokio::test] + async fn sse_s3_bucket_encryption_accepts_tenant_kms_or_a_secret_master_key() { + let mut from_secret = sse_s3_tenant(); + from_secret.spec.env = vec![sse_s3_master_key_env(None, Some("master-key"))]; + for (tenant, ctx) in [ + (kms_tenant(None), empty_kube_context()), + ( + from_secret, + sse_s3_secret_kube_context("master-key", SSE_S3_TEST_MASTER_KEY), + ), + ] { + let capture = BucketControlCapture { + existing: true, + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.encryption.lock().await.clone(), Some(sse_s3())); + server.abort(); + } + } + + #[tokio::test] + async fn sse_s3_master_key_secret_without_the_key_fails_before_any_call() { + let capture = BucketControlCapture { + existing: true, ..Default::default() }; + let (client, server) = bucket_control_test_client(capture.clone()).await; + let ctx = sse_s3_secret_kube_context("other-key", SSE_S3_TEST_MASTER_KEY); + let mut tenant = sse_s3_tenant(); + tenant.spec.env = vec![sse_s3_master_key_env(None, Some("master-key"))]; + let mut run = empty_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &bucket).await; - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketCreateFailed.as_str()); - run.push_bucket(item); + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-S3"})), + ) + .await; - match run.finish().outcome { - ProvisioningOutcome::Failed { reason, .. } => { - assert_eq!(reason, Reason::BucketCreateFailed); - } - _ => panic!("permanent bucket create errors must not retry"), - } + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("has no key 'master-key'")) + ); + assert_eq!(encryption_requests(&capture).await, 0); server.abort(); } #[tokio::test] - async fn present_bucket_lifecycle_is_applied_when_live_configuration_is_missing() { - let capture = LifecycleCapture::default(); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn removing_an_explicit_kms_key_applies_the_tenant_default_key() { + let capture = BucketControlCapture { + existing: true, + encryption: Arc::new(Mutex::new(Some(sse_kms("old-explicit-key")))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = kms_tenant(Some("rustfs-default-key")); let mut run = empty_run(&ctx, &tenant); + run.previous + .buckets + .push(previous_encryption_owner(hash_document( + &sse_kms("old-explicit-key").to_xml(), + ))); - let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.get_count.load(Ordering::SeqCst), 1); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 1); - assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); assert_eq!( - capture.tier_count.load(Ordering::SeqCst), - 0, - "rules without transitions must not query RustFS tiers" + capture.encryption.lock().await.clone(), + Some(sse_kms("rustfs-default-key")) ); + let put_requests = capture.put_requests.lock().await; + let put = put_requests + .iter() + .find(|request| request.query == "encryption=") + .expect("encryption PUT is captured"); + assert_eq!(put.body, sse_kms("rustfs-default-key").to_xml()); assert_eq!( - item.lifecycle_desired_hash, - item.lifecycle_last_applied_hash + item.encryption_last_applied_hash, + Some(hash_document(&sse_kms("rustfs-default-key").to_xml())) ); - assert!(item.lifecycle_desired_hash.is_some()); - let body = capture.put_body.lock().await; - assert!(body.contains("expire-logs")); - assert!(body.contains("logs/")); - assert!(body.contains("30")); server.abort(); } #[tokio::test] - async fn present_bucket_lifecycle_refuses_to_replace_unmanaged_live_configuration() { - let capture = LifecycleCapture::default(); - *capture.live.lock().await = Some( - concat!( - "", - "7", - "externalEnabled" - ) - .to_string(), - ); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn tenant_default_kms_key_change_is_applied_to_an_owned_bucket() { + let capture = BucketControlCapture { + existing: true, + encryption: Arc::new(Mutex::new(Some(sse_kms("old-default-key")))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = kms_tenant(Some("new-default-key")); let mut run = empty_run(&ctx, &tenant); + run.previous + .buckets + .push(previous_encryption_owner(hash_document( + &sse_kms("old-default-key").to_xml(), + ))); - let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketLifecycleConflict.as_str()); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); - assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); - assert!(item.lifecycle_desired_hash.is_some()); - assert!(item.lifecycle_last_applied_hash.is_none()); + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!( + capture.encryption.lock().await.clone(), + Some(sse_kms("new-default-key")) + ); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 1); server.abort(); } #[tokio::test] - async fn matching_live_bucket_lifecycle_is_adopted_without_write() { - let bucket = present_lifecycle_bucket(30); - let live = lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle spec")) - .expect("valid lifecycle") - .expect("Present lifecycle") - .to_xml(); - let capture = LifecycleCapture::default(); - *capture.live.lock().await = Some(live); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn external_kms_key_change_is_a_conflict_that_keeps_the_ownership_record() { + let capture = BucketControlCapture { + existing: true, + encryption: Arc::new(Mutex::new(Some(sse_kms("foreign-key")))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = kms_tenant(Some("rustfs-default-key")); + let owned = hash_document(&sse_kms("rustfs-default-key").to_xml()); let mut run = empty_run(&ctx, &tenant); + run.previous + .buckets + .push(previous_encryption_owner(owned.clone())); - let item = reconcile_bucket(&mut run, &client, &bucket).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; - assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.get_count.load(Ordering::SeqCst), 1); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionConflict.as_str()); + assert_eq!(item.encryption_last_applied_hash, Some(owned)); + assert_eq!(requests_for(&capture, "PUT encryption=").await, 0); assert_eq!( - item.lifecycle_desired_hash, - item.lifecycle_last_applied_hash + capture.encryption.lock().await.clone(), + Some(sse_kms("foreign-key")) ); server.abort(); } #[tokio::test] - async fn transient_bucket_lifecycle_read_preserves_ownership_and_retries() { - let capture = LifecycleCapture::default(); - capture.get_error_status.store( - StatusCode::SERVICE_UNAVAILABLE.as_u16() as usize, - Ordering::SeqCst, - ); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn sse_kms_without_a_key_requires_the_tenant_default_key() { + let capture = BucketControlCapture { + existing: true, + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let tenant = kms_tenant(None); let mut run = empty_run(&ctx, &tenant); - let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( - "app-data", - ProvisioningItemState::Ready, - Reason::ProvisioningConfigured.as_str(), - )); - previous.lifecycle_last_applied_hash = Some("owned-live".to_string()); - run.previous.buckets.push(previous); - let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + let item = reconcile_bucket( + &mut run, + &client, + &encryption_bucket(serde_json::json!({"mode": "SSE-KMS"})), + ) + .await; - assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); - assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); - assert_eq!( - item.lifecycle_last_applied_hash.as_deref(), - Some("owned-live") + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketEncryptionApplyFailed.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("spec.encryption.defaultKeyId")) ); - assert!(run.retry.is_some()); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!(encryption_requests(&capture).await, 0); server.abort(); } - #[tokio::test] - async fn oversized_bucket_lifecycle_response_fails_without_write() { - let capture = LifecycleCapture::default(); - *capture.live.lock().await = Some(format!( - "{}", - "x".repeat(4 * 1024 * 1024) + #[test] + fn bucket_encryption_spec_is_validated_before_any_call() { + let spec = |value: serde_json::Value| -> BucketEncryptionSpec { + serde_json::from_value(value).expect("encryption spec deserializes") + }; + assert_eq!( + bucket_encryption_configuration(&spec(serde_json::json!({"mode": "SSE-S3"}))), + Ok(Some(sse_s3())) + ); + assert_eq!( + bucket_encryption_configuration(&spec(serde_json::json!({"state": "Absent"}))), + Ok(None) + ); + for invalid in [ + serde_json::json!({}), + serde_json::json!({"state": "Absent", "mode": "SSE-S3"}), + serde_json::json!({"mode": "SSE-S3", "kmsKeyId": "app-key"}), + serde_json::json!({"mode": "SSE-KMS", "kmsKeyId": "app key"}), + serde_json::json!({"mode": "SSE-KMS", "kmsKeyId": "app"}), + serde_json::json!({"mode": "SSE-KMS", "kmsKeyId": ""}), + ] { + assert!( + bucket_encryption_configuration(&spec(invalid.clone())).is_err(), + "{invalid} must be rejected" + ); + } + let bucket = encryption_bucket(serde_json::json!({"state": "Absent", "mode": "SSE-S3"})); + assert!(matches!( + validate_bucket_configuration(&bucket), + Err((Reason::BucketEncryptionApplyFailed, _)) )); - let (client, server) = lifecycle_test_client(capture.clone()).await; + } + + #[tokio::test] + async fn matching_live_object_lock_configuration_is_adopted_without_a_write() { + let live = AdminObjectLockConfiguration { + default_retention: Some(AdminObjectLockDefaultRetention { + mode: AdminObjectLockRetentionMode::Compliance, + period: AdminObjectLockRetentionPeriod::Days(30), + }), + }; + let capture = BucketControlCapture { + existing: true, + versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), + object_lock: Arc::new(Mutex::new(Some(live))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + let item = reconcile_bucket(&mut run, &client, &object_lock_bucket()).await; - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketLifecycleApplyFailed.as_str()); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); - assert!(run.retry.is_none()); + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!( + item.object_lock_configuration_desired_hash, + item.object_lock_configuration_last_applied_hash + ); + assert!( + !capture + .requests + .lock() + .await + .iter() + .any(|request| request == "PUT object-lock=") + ); server.abort(); } #[tokio::test] - async fn absent_bucket_lifecycle_deletes_only_matching_owned_configuration() { - let live = concat!( - "", - "30logs/", - "expire-logsEnabled" - ); - let live_hash = hash_document( - &canonicalize_bucket_lifecycle_xml(live).expect("live lifecycle should normalize"), - ); - let capture = LifecycleCapture::default(); - *capture.live.lock().await = Some(live.to_string()); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn omitted_object_lock_configuration_leaves_live_retention_unmanaged() { + let live = AdminObjectLockConfiguration { + default_retention: Some(AdminObjectLockDefaultRetention { + mode: AdminObjectLockRetentionMode::Governance, + period: AdminObjectLockRetentionPeriod::Days(7), + }), + }; + let capture = BucketControlCapture { + existing: true, + versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), + object_lock: Arc::new(Mutex::new(Some(live.clone()))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); - let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( - "app-data", - ProvisioningItemState::Ready, - Reason::ProvisioningConfigured.as_str(), - )); - previous.lifecycle_desired_hash = Some(live_hash.clone()); - previous.lifecycle_last_applied_hash = Some(live_hash); - run.previous.buckets.push(previous); + let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ + "name": "secure-bucket", + "objectLock": true + })) + .expect("bucket should deserialize"); - let item = reconcile_bucket(&mut run, &client, &absent_lifecycle_bucket()).await; + let item = reconcile_bucket(&mut run, &client, &bucket).await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.delete_count.load(Ordering::SeqCst), 1); - assert!(item.lifecycle_desired_hash.is_none()); - assert!(item.lifecycle_last_applied_hash.is_none()); + assert_eq!(capture.object_lock.lock().await.as_ref(), Some(&live)); + assert!(item.object_lock_configuration_desired_hash.is_none()); + assert!( + !capture + .requests + .lock() + .await + .iter() + .any(|request| request == "PUT object-lock=") + ); server.abort(); } #[tokio::test] - async fn omitted_bucket_lifecycle_performs_no_lifecycle_request_and_preserves_ownership() { - let capture = LifecycleCapture::default(); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn owned_object_lock_default_retention_can_be_removed() { + let live = AdminObjectLockConfiguration { + default_retention: Some(AdminObjectLockDefaultRetention { + mode: AdminObjectLockRetentionMode::Compliance, + period: AdminObjectLockRetentionPeriod::Days(30), + }), + }; + let capture = BucketControlCapture { + existing: true, + versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), + object_lock: Arc::new(Mutex::new(Some(live.clone()))), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture.clone()).await; let ctx = empty_kube_context(); let user = provisioning_user("app-user", "app-user-secret", "readwrite"); let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); let mut run = empty_run(&ctx, &tenant); let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( - "app-data", + "secure-bucket", ProvisioningItemState::Ready, Reason::ProvisioningConfigured.as_str(), )); - previous.lifecycle_desired_hash = Some("old-desired".to_string()); - previous.lifecycle_last_applied_hash = Some("owned-live".to_string()); + previous.object_lock_configuration_last_applied_hash = Some(hash_document(&live.to_xml())); run.previous.buckets.push(previous); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), - ..Default::default() - }; + let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ + "name": "secure-bucket", + "objectLock": true, + "objectLockConfiguration": { + "state": "Absent" + } + })) + .expect("bucket should deserialize"); let item = reconcile_bucket(&mut run, &client, &bucket).await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.get_count.load(Ordering::SeqCst), 0); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); - assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); - assert!(item.lifecycle_desired_hash.is_none()); assert_eq!( - item.lifecycle_last_applied_hash.as_deref(), - Some("owned-live") + capture.object_lock.lock().await.as_ref(), + Some(&AdminObjectLockConfiguration::enabled_without_default_retention()) + ); + assert_eq!( + item.object_lock_configuration_desired_hash, + item.object_lock_configuration_last_applied_hash + ); + assert_eq!( + capture + .requests + .lock() + .await + .iter() + .filter(|request| request.as_str() == "PUT object-lock=") + .count(), + 1 ); server.abort(); } + #[tokio::test] + async fn object_lock_update_is_verified_after_a_success_response() { + let capture = BucketControlCapture { + existing: true, + ignore_object_lock_put: true, + versioning: Arc::new(Mutex::new(BucketVersioningState::Enabled)), + ..Default::default() + }; + let (client, server) = bucket_control_test_client(capture).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &object_lock_bucket()).await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketObjectLockApplyFailed.as_str()); + assert_eq!(item.object_lock, Some(false)); + assert!(item.object_lock_configuration_last_applied_hash.is_none()); + server.abort(); + } + #[test] - fn lifecycle_reconcile_action_protects_owned_and_unmanaged_configurations() { + fn object_lock_configuration_requires_explicit_object_lock_enablement() { + let bucket: ProvisioningBucket = serde_json::from_value(serde_json::json!({ + "name": "secure-bucket", + "objectLockConfiguration": { + "mode": "Compliance", + "days": 30 + } + })) + .expect("older CRDs can admit the structurally valid payload"); + + let (reason, message) = validate_bucket_configuration(&bucket) + .expect_err("runtime validation must protect clusters with an older CRD"); + + assert_eq!(reason, Reason::BucketObjectLockApplyFailed); + assert_eq!(message, "objectLockConfiguration requires objectLock: true"); + } + + #[test] + fn object_lock_configuration_maps_modes_and_enforces_runtime_boundaries() { + let configuration = |mode, days| { + serde_json::from_value(serde_json::json!({ + "mode": mode, + "days": days + })) + .expect("Object Lock configuration should deserialize") + }; + + for (mode, expected_mode) in [ + ("Governance", AdminObjectLockRetentionMode::Governance), + ("Compliance", AdminObjectLockRetentionMode::Compliance), + ] { + for days in [1, MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS as i32] { + assert_eq!( + object_lock_configuration(&configuration(mode, days)).unwrap(), + AdminObjectLockConfiguration { + default_retention: Some(AdminObjectLockDefaultRetention { + mode: expected_mode, + period: AdminObjectLockRetentionPeriod::Days(days), + }), + } + ); + } + } + for days in [0, MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS as i32 + 1] { + let error = object_lock_configuration(&configuration("Governance", days)) + .expect_err("out-of-range retention must fail"); + assert!(error.contains("must be between")); + } + } + + #[test] + fn object_lock_reconcile_action_protects_owned_and_unmanaged_configurations() { assert_eq!( - lifecycle_reconcile_action(None, None, Some("desired")), - LifecycleReconcileAction::Apply + object_lock_configuration_reconcile_action(None, None, "desired", false), + ObjectLockConfigurationReconcileAction::Apply ); assert_eq!( - lifecycle_reconcile_action(None, Some("desired"), Some("desired")), - LifecycleReconcileAction::Ready("RustFS bucket lifecycle configuration matches spec") + object_lock_configuration_reconcile_action(None, Some("desired"), "desired", false), + ObjectLockConfigurationReconcileAction::Ready( + "RustFS bucket Object Lock configuration matches spec" + ) ); - assert!(matches!( - lifecycle_reconcile_action(None, Some("external"), Some("desired")), - LifecycleReconcileAction::Failed(_) - )); assert_eq!( - lifecycle_reconcile_action(Some("owned"), Some("owned"), Some("changed")), - LifecycleReconcileAction::Apply + object_lock_configuration_reconcile_action( + Some("owned"), + Some("owned"), + "changed", + false, + ), + ObjectLockConfigurationReconcileAction::Apply ); assert_eq!( - lifecycle_reconcile_action(Some("owned"), Some("owned"), None), - LifecycleReconcileAction::Delete + object_lock_configuration_reconcile_action( + None, + Some("enabled-without-retention"), + "desired", + true, + ), + ObjectLockConfigurationReconcileAction::Apply ); assert!(matches!( - lifecycle_reconcile_action(None, Some("external"), None), - LifecycleReconcileAction::Failed(_) + object_lock_configuration_reconcile_action(None, Some("external"), "desired", false,), + ObjectLockConfigurationReconcileAction::Failed(_) + )); + assert!(matches!( + object_lock_configuration_reconcile_action( + Some("owned"), + Some("external-drift"), + "desired", + false, + ), + ObjectLockConfigurationReconcileAction::Failed(_) )); + } - let mut bucket = present_lifecycle_bucket(30); - bucket.lifecycle.as_mut().expect("lifecycle").rules[0] - .filter - .prefix = "invalid\0prefix".to_string(); - assert!( - lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle")).is_err(), - "invalid XML characters must fail before an S3 request" + #[tokio::test] + async fn create_bucket_503_marks_pending_and_retries() { + let router = Router::new().route( + "/app-data", + put(|| async { StatusCode::SERVICE_UNAVAILABLE }), ); - } + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + ..Default::default() + }; - #[test] - fn lifecycle_configuration_validates_empty_and_exact_boundaries() { - let mut bucket = present_lifecycle_bucket(1); - let spec = bucket.lifecycle.as_mut().expect("lifecycle"); - let template = spec.rules[0].clone(); - spec.rules = (0..MAX_BUCKET_LIFECYCLE_RULES) - .map(|index| { - let mut rule = template.clone(); - rule.id = format!("rule-{index}"); - rule - }) - .collect(); - assert!(lifecycle_configuration(spec).is_ok()); - let mut overflow = template.clone(); - overflow.id = "overflow".to_string(); - spec.rules.push(overflow); - assert!(lifecycle_configuration(spec).is_err()); + let item = reconcile_bucket(&mut run, &client, &bucket).await; + assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); + run.push_bucket(item); - spec.rules = vec![template.clone()]; - spec.rules[0].id = "x".repeat(MAX_BUCKET_LIFECYCLE_RULE_ID_LENGTH as usize); - assert!(lifecycle_configuration(spec).is_ok()); - spec.rules[0].id.push('x'); - assert!(lifecycle_configuration(spec).is_err()); - spec.rules[0].id = "界".repeat(86); - assert!( - lifecycle_configuration(spec).is_err(), - "RustFS enforces the lifecycle rule ID limit in UTF-8 bytes" - ); + match run.finish().outcome { + ProvisioningOutcome::Retry { persist_status, .. } => assert!(persist_status), + _ => panic!("transient bucket create must retry"), + } + server.abort(); + } - spec.rules.clear(); - assert!(lifecycle_configuration(spec).is_err()); - spec.state = BucketLifecycleState::Absent; - assert_eq!(lifecycle_configuration(spec), Ok(None)); - spec.rules.push(template); - assert!(lifecycle_configuration(spec).is_err()); + #[tokio::test] + async fn create_bucket_400_fails_without_retry() { + let router = Router::new().route("/app-data", put(|| async { StatusCode::BAD_REQUEST })); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + ..Default::default() + }; - let zero_days = present_lifecycle_bucket(0); - assert!(lifecycle_configuration(zero_days.lifecycle.as_ref().expect("lifecycle")).is_err()); + let item = reconcile_bucket(&mut run, &client, &bucket).await; + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketCreateFailed.as_str()); + run.push_bucket(item); + + match run.finish().outcome { + ProvisioningOutcome::Failed { reason, .. } => { + assert_eq!(reason, Reason::BucketCreateFailed); + } + _ => panic!("permanent bucket create errors must not retry"), + } + server.abort(); } - #[test] - fn lifecycle_configuration_validates_transition_fields() { - let valid = transition_lifecycle_bucket(0, "COLD"); - let configuration = lifecycle_configuration(valid.lifecycle.as_ref().expect("lifecycle")) - .expect("zero-day transition is valid") - .expect("Present lifecycle"); + #[tokio::test] + async fn present_bucket_lifecycle_is_applied_when_live_configuration_is_missing() { + let capture = LifecycleCapture::default(); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.get_count.load(Ordering::SeqCst), 1); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 1); + assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); + assert_eq!( + capture.tier_count.load(Ordering::SeqCst), + 0, + "rules without transitions must not query RustFS tiers" + ); + assert_eq!( + item.lifecycle_desired_hash, + item.lifecycle_last_applied_hash + ); + assert!(item.lifecycle_desired_hash.is_some()); + let body = capture.put_body.lock().await; + assert!(body.contains("expire-logs")); + assert!(body.contains("logs/")); + assert!(body.contains("30")); + server.abort(); + } + + #[tokio::test] + async fn present_bucket_lifecycle_refuses_to_replace_unmanaged_live_configuration() { + let capture = LifecycleCapture::default(); + *capture.live.lock().await = Some( + concat!( + "", + "7", + "externalEnabled" + ) + .to_string(), + ); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketLifecycleConflict.as_str()); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); + assert!(item.lifecycle_desired_hash.is_some()); + assert!(item.lifecycle_last_applied_hash.is_none()); + server.abort(); + } + + #[tokio::test] + async fn matching_live_bucket_lifecycle_is_adopted_without_write() { + let bucket = present_lifecycle_bucket(30); + let live = lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle spec")) + .expect("valid lifecycle") + .expect("Present lifecycle") + .to_xml(); + let capture = LifecycleCapture::default(); + *capture.live.lock().await = Some(live); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.get_count.load(Ordering::SeqCst), 1); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!( + item.lifecycle_desired_hash, + item.lifecycle_last_applied_hash + ); + server.abort(); + } + + #[tokio::test] + async fn transient_bucket_lifecycle_read_preserves_ownership_and_retries() { + let capture = LifecycleCapture::default(); + capture.get_error_status.store( + StatusCode::SERVICE_UNAVAILABLE.as_u16() as usize, + Ordering::SeqCst, + ); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( + "app-data", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + )); + previous.lifecycle_last_applied_hash = Some("owned-live".to_string()); + run.previous.buckets.push(previous); + + let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + + assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); + assert_eq!( + item.lifecycle_last_applied_hash.as_deref(), + Some("owned-live") + ); + assert!(run.retry.is_some()); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + server.abort(); + } + + #[tokio::test] + async fn oversized_bucket_lifecycle_response_fails_without_write() { + let capture = LifecycleCapture::default(); + *capture.live.lock().await = Some(format!( + "{}", + "x".repeat(4 * 1024 * 1024) + )); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = reconcile_bucket(&mut run, &client, &present_lifecycle_bucket(30)).await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketLifecycleApplyFailed.as_str()); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert!(run.retry.is_none()); + server.abort(); + } + + #[tokio::test] + async fn absent_bucket_lifecycle_deletes_only_matching_owned_configuration() { + let live = concat!( + "", + "30logs/", + "expire-logsEnabled" + ); + let live_hash = hash_document( + &canonicalize_bucket_lifecycle_xml(live).expect("live lifecycle should normalize"), + ); + let capture = LifecycleCapture::default(); + *capture.live.lock().await = Some(live.to_string()); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( + "app-data", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + )); + previous.lifecycle_desired_hash = Some(live_hash.clone()); + previous.lifecycle_last_applied_hash = Some(live_hash); + run.previous.buckets.push(previous); + + let item = reconcile_bucket(&mut run, &client, &absent_lifecycle_bucket()).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.delete_count.load(Ordering::SeqCst), 1); + assert!(item.lifecycle_desired_hash.is_none()); + assert!(item.lifecycle_last_applied_hash.is_none()); + server.abort(); + } + + #[tokio::test] + async fn omitted_bucket_lifecycle_performs_no_lifecycle_request_and_preserves_ownership() { + let capture = LifecycleCapture::default(); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let mut previous = ProvisioningBucketStatus::new(ProvisioningItemStatus::new( + "app-data", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + )); + previous.lifecycle_desired_hash = Some("old-desired".to_string()); + previous.lifecycle_last_applied_hash = Some("owned-live".to_string()); + run.previous.buckets.push(previous); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + ..Default::default() + }; + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.get_count.load(Ordering::SeqCst), 0); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!(capture.delete_count.load(Ordering::SeqCst), 0); + assert!(item.lifecycle_desired_hash.is_none()); + assert_eq!( + item.lifecycle_last_applied_hash.as_deref(), + Some("owned-live") + ); + server.abort(); + } + + #[test] + fn lifecycle_reconcile_action_protects_owned_and_unmanaged_configurations() { + assert_eq!( + lifecycle_reconcile_action(None, None, Some("desired")), + LifecycleReconcileAction::Apply + ); + assert_eq!( + lifecycle_reconcile_action(None, Some("desired"), Some("desired")), + LifecycleReconcileAction::Ready("RustFS bucket lifecycle configuration matches spec") + ); + assert!(matches!( + lifecycle_reconcile_action(None, Some("external"), Some("desired")), + LifecycleReconcileAction::Failed(_) + )); + assert_eq!( + lifecycle_reconcile_action(Some("owned"), Some("owned"), Some("changed")), + LifecycleReconcileAction::Apply + ); + assert_eq!( + lifecycle_reconcile_action(Some("owned"), Some("owned"), None), + LifecycleReconcileAction::Delete + ); + assert!(matches!( + lifecycle_reconcile_action(None, Some("external"), None), + LifecycleReconcileAction::Failed(_) + )); + + let mut bucket = present_lifecycle_bucket(30); + bucket.lifecycle.as_mut().expect("lifecycle").rules[0] + .filter + .prefix = "invalid\0prefix".to_string(); + assert!( + lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle")).is_err(), + "invalid XML characters must fail before an S3 request" + ); + } + + #[test] + fn lifecycle_configuration_validates_empty_and_exact_boundaries() { + let mut bucket = present_lifecycle_bucket(1); + let spec = bucket.lifecycle.as_mut().expect("lifecycle"); + let template = spec.rules[0].clone(); + spec.rules = (0..MAX_BUCKET_LIFECYCLE_RULES) + .map(|index| { + let mut rule = template.clone(); + rule.id = format!("rule-{index}"); + rule + }) + .collect(); + assert!(lifecycle_configuration(spec).is_ok()); + let mut overflow = template.clone(); + overflow.id = "overflow".to_string(); + spec.rules.push(overflow); + assert!(lifecycle_configuration(spec).is_err()); + + spec.rules = vec![template.clone()]; + spec.rules[0].id = "x".repeat(MAX_BUCKET_LIFECYCLE_RULE_ID_LENGTH as usize); + assert!(lifecycle_configuration(spec).is_ok()); + spec.rules[0].id.push('x'); + assert!(lifecycle_configuration(spec).is_err()); + spec.rules[0].id = "界".repeat(86); + assert!( + lifecycle_configuration(spec).is_err(), + "RustFS enforces the lifecycle rule ID limit in UTF-8 bytes" + ); + + spec.rules.clear(); + assert!(lifecycle_configuration(spec).is_err()); + spec.state = BucketLifecycleState::Absent; + assert_eq!(lifecycle_configuration(spec), Ok(None)); + spec.rules.push(template); + assert!(lifecycle_configuration(spec).is_err()); + + let zero_days = present_lifecycle_bucket(0); + assert!(lifecycle_configuration(zero_days.lifecycle.as_ref().expect("lifecycle")).is_err()); + } + + #[test] + fn lifecycle_configuration_validates_transition_fields() { + let valid = transition_lifecycle_bucket(0, "COLD"); + let configuration = lifecycle_configuration(valid.lifecycle.as_ref().expect("lifecycle")) + .expect("zero-day transition is valid") + .expect("Present lifecycle"); + assert_eq!( + configuration.rules[0].transition, + Some(AdminLifecycleTransition { + days: 0, + storage_class: "COLD".to_string(), + }) + ); + + for (days, storage_class) in [ + (-1, "COLD"), + (1, ""), + (1, "cold"), + (1, "STANDARD"), + (1, "reduced_redundancy"), + (1, "BAD\u{0}TIER"), + ] { + let bucket = transition_lifecycle_bucket(days, storage_class); + assert!( + lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle")).is_err(), + "transition days={days} storageClass={storage_class:?} must be rejected" + ); + } + + let mut noncurrent = transition_lifecycle_bucket(1, "COLD"); + let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; + rule.transition = None; + rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { + noncurrent_days: -1, + storage_class: "COLD".to_string(), + }); + assert!( + lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")).is_err() + ); + let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; + rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { + noncurrent_days: 0, + storage_class: "COLD".to_string(), + }); + assert!( + lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")).is_ok(), + "RustFS accepts a zero-day noncurrent transition" + ); + let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; + rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { + noncurrent_days: 7, + storage_class: "COLD".to_string(), + }); + let configuration = + lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")) + .expect("noncurrent transition is valid") + .expect("Present lifecycle"); + assert_eq!( + configuration.rules[0].noncurrent_version_transition, + Some(AdminLifecycleNoncurrentVersionTransition { + noncurrent_days: 7, + storage_class: "COLD".to_string(), + }) + ); + } + + #[tokio::test] + async fn transition_bucket_lifecycle_is_applied_when_live_configuration_is_missing() { + let capture = LifecycleCapture::default(); + *capture.tiers.lock().await = r#"[{"type":"s3","s3":{"name":"COLD"}}]"#.to_string(); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.tier_count.load(Ordering::SeqCst), 1); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 1); + let body = capture.put_body.lock().await; + assert!(body.contains( + "Enabled1COLD" + )); + server.abort(); + } + + #[tokio::test] + async fn transition_bucket_lifecycle_waits_for_an_unregistered_tier() { + let capture = LifecycleCapture::default(); + *capture.tiers.lock().await = r#"[{"type":"s3","s3":{"name":"WARM"}}]"#.to_string(); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + + assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("COLD")) + ); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert!(run.retry.is_some(), "a missing tier must requeue"); + assert!(item.lifecycle_last_applied_hash.is_none()); + server.abort(); + } + + #[tokio::test] + async fn rustfs_shaped_live_transition_lifecycle_is_adopted_without_write() { + // Shape returned by RustFS GetBucketLifecycleConfiguration for an operator-applied rule. + let live = concat!( + "", + "", + "to-coldEnabled", + "1COLD", + "" + ); + let capture = LifecycleCapture::default(); + *capture.live.lock().await = Some(live.to_string()); + let (client, server) = lifecycle_test_client(capture.clone()).await; + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + + let item = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); + assert_eq!( + item.lifecycle_desired_hash, + item.lifecycle_last_applied_hash + ); + server.abort(); + } + + #[tokio::test] + async fn anonymous_download_policy_is_applied_when_live_policy_is_missing() { + let capture = PolicyApplyCapture::default(); + let route_capture = capture.clone(); + let router = Router::new().route( + "/app-data", + put({ + let capture = route_capture.clone(); + move |req: Request| { + let capture = capture.clone(); + async move { + if req.uri().query().unwrap_or("").contains("policy") { + let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) + .await + .expect("policy body"); + *capture.body.lock().await = + String::from_utf8(body_bytes.to_vec()).expect("utf8"); + } + StatusCode::OK + } + } + }) + .get(|req: Request| async move { + if req.uri().query().unwrap_or("").contains("policy") { + ( + StatusCode::NOT_FOUND, + r#"NoSuchBucketPolicy"#, + ) + } else { + (StatusCode::OK, "") + } + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + anonymous: Some(BucketAnonymousAccess::Download), + ..Default::default() + }; + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert!(item.last_applied_hash.is_some()); + let applied = capture.body.lock().await; + assert!(applied.contains("s3:GetObject")); + assert!(!applied.contains("s3:PutObject")); + server.abort(); + } + + #[tokio::test] + async fn explicit_private_removes_operator_managed_anonymous_policy() { + let live_policy = canned_anonymous_bucket_policy(BucketAnonymousAccess::Public, "app-data") + .expect("public policy"); + let live_hash = hash_document( + &normalize_policy_document(&live_policy).expect("public policy should normalize"), + ); + let delete_count = Arc::new(AtomicUsize::new(0)); + let route_delete_count = Arc::clone(&delete_count); + let route_live_policy = live_policy.clone(); + let router = Router::new().route( + "/app-data", + put(|| async { StatusCode::OK }) + .get(move |req: Request| { + let live_policy = route_live_policy.clone(); + async move { + assert_eq!(req.uri().query().unwrap_or(""), "policy="); + (StatusCode::OK, live_policy) + } + }) + .delete(move |req: Request| { + let delete_count = Arc::clone(&route_delete_count); + async move { + assert_eq!(req.uri().query().unwrap_or(""), "policy="); + delete_count.fetch_add(1, Ordering::SeqCst); + StatusCode::NO_CONTENT + } + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let mut previous = ProvisioningItemStatus::new( + "app-data", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + ); + previous.last_applied_hash = Some(live_hash.clone()); + run.previous.buckets.push(previous.into()); + let omitted_bucket = ProvisioningBucket { + name: "app-data".to_string(), + ..Default::default() + }; + + let omitted_item = reconcile_bucket(&mut run, &client, &omitted_bucket).await; + + assert_eq!(omitted_item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!( + omitted_item.last_applied_hash.as_deref(), + Some(live_hash.as_str()) + ); + let mut run = empty_run(&ctx, &tenant); + run.previous.buckets.push(omitted_item); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + anonymous: Some(BucketAnonymousAccess::Private), + ..Default::default() + }; + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(delete_count.load(Ordering::SeqCst), 1); + assert!(item.desired_hash.is_none()); + assert!(item.last_applied_hash.is_none()); + server.abort(); + } + + #[tokio::test] + async fn explicit_private_preserves_unmanaged_live_policy() { + let live_policy = canned_anonymous_bucket_policy(BucketAnonymousAccess::Public, "app-data") + .expect("public policy"); + let delete_count = Arc::new(AtomicUsize::new(0)); + let route_delete_count = Arc::clone(&delete_count); + let router = Router::new().route( + "/app-data", + put(|| async { StatusCode::OK }) + .get(move || { + let live_policy = live_policy.clone(); + async move { (StatusCode::OK, live_policy) } + }) + .delete(move || { + let delete_count = Arc::clone(&route_delete_count); + async move { + delete_count.fetch_add(1, Ordering::SeqCst); + StatusCode::NO_CONTENT + } + }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + anonymous: Some(BucketAnonymousAccess::Private), + ..Default::default() + }; + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketPolicyConflict.as_str()); + assert_eq!(delete_count.load(Ordering::SeqCst), 0); + server.abort(); + } + + #[tokio::test] + async fn mixed_anonymous_and_custom_policy_fails_before_rustfs_calls() { + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + let bucket = ProvisioningBucket { + name: "app-data".to_string(), + anonymous: Some(BucketAnonymousAccess::Public), + policy: Some(PolicyDocumentSource { + config_map_key_ref: ConfigMapKeyReference { + name: "bucket-policy".to_string(), + key: "policy.json".to_string(), + }, + }), + ..Default::default() + }; + + let item = reconcile_bucket(&mut run, &client, &bucket).await; + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::BucketPolicyConflict.as_str()); + } + + #[tokio::test] + async fn list_canned_policies_503_is_transient() { + let router = Router::new().route( + "/rustfs/admin/v3/list-canned-policies", + get(|| async { StatusCode::SERVICE_UNAVAILABLE }), + ); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + let client = + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + let tenant = Tenant { + metadata: ObjectMeta { + name: Some("tenant-a".to_string()), + namespace: Some("storage".to_string()), + ..Default::default() + }, + spec: crate::types::v1alpha1::tenant::TenantSpec { + policies: vec![ProvisioningPolicy { + name: "app-readwrite".to_string(), + document: PolicyDocumentSource { + config_map_key_ref: ConfigMapKeyReference { + name: "app-policy".to_string(), + key: "policy.json".to_string(), + }, + }, + ..Default::default() + }], + ..Default::default() + }, + status: None, + }; + + let error = load_live_policies(&client, &tenant) + .await + .expect_err("503 should fail the live policy load"); + assert!(error.transient); + assert!( + error.message.contains("503") + || error.message.contains("unavailable") + || error.message.contains("failed to list") + ); + server.abort(); + } + + #[tokio::test] + async fn user_stays_pending_when_referenced_policy_is_pending() { + let ctx = empty_kube_context(); + let user = provisioning_user("app-user", "app-user-secret", "readwrite"); + let tenant = provisioning_test_tenant(user.clone(), ProvisioningStatus::default()); + let mut run = empty_run(&ctx, &tenant); + let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); + let pending = BTreeSet::from(["readwrite".to_string()]); + let credentials = user_credentials("1"); + + let plan = prepare_user_reconcile( + &mut run, + &client, + &BTreeMap::new(), + &BTreeSet::new(), + &pending, + &user, + &credentials, + ) + .await; + + let UserReconcilePlan::Complete(item) = plan else { + panic!("pending referenced policy must not call RustFS"); + }; + assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("readwrite")) + ); + run.push_user(*item); + + match run.finish().outcome { + ProvisioningOutcome::Retry { persist_status, .. } => assert!(persist_status), + _ => panic!("pending referenced policy must requeue instead of failing the user"), + } + } + + #[derive(Clone, Default)] + struct TierCapture { + tiers: Arc>, + add_status: Arc, + list_status: Arc, + adds: Arc>>, + edits: Arc>>, + } + + async fn tier_admin_handler( + State(capture): State, + request: Request, + ) -> Response { + let method = request.method().clone(); + let path = request.uri().path().to_string(); + let query = request.uri().query().unwrap_or("").to_string(); + let body = axum::body::to_bytes(request.into_body(), usize::MAX) + .await + .expect("tier request body"); + let json = || serde_json::from_slice::(&body).expect("tier body is JSON"); + match method { + axum::http::Method::GET => { + let status = capture.list_status.load(Ordering::SeqCst); + if status != 0 { + return StatusCode::from_u16(status as u16) + .expect("configured test status is valid") + .into_response(); + } + (StatusCode::OK, capture.tiers.lock().await.clone()).into_response() + } + axum::http::Method::PUT => { + let added = json(); + capture.adds.lock().await.push((query, added.clone())); + let status = capture.add_status.load(Ordering::SeqCst); + if status != 0 { + return StatusCode::from_u16(status as u16) + .expect("configured test status is valid") + .into_response(); + } + let tier_type = added["type"].as_str().expect("tier type").to_string(); + let mut backend = added[&tier_type].clone(); + backend["secretKey"] = Value::from("REDACTED"); + let mut tiers: Vec = + serde_json::from_str(&capture.tiers.lock().await).unwrap_or_default(); + tiers.push(serde_json::json!({"type": tier_type, tier_type.clone(): backend})); + *capture.tiers.lock().await = Value::from(tiers).to_string(); + StatusCode::OK.into_response() + } + axum::http::Method::POST => { + capture.edits.lock().await.push((path, json())); + StatusCode::OK.into_response() + } + _ => StatusCode::METHOD_NOT_ALLOWED.into_response(), + } + } + + async fn tier_test_client( + capture: TierCapture, + lifecycle: Option, + ) -> (RustfsAdminClient, tokio::task::JoinHandle<()>) { + let mut router = Router::new() + .route("/rustfs/admin/v3/tier", any(tier_admin_handler)) + .route("/rustfs/admin/v3/tier/:name", any(tier_admin_handler)) + .with_state(capture); + if let Some(lifecycle) = lifecycle { + router = router.merge( + Router::new() + .route("/app-data", any(lifecycle_bucket_handler)) + .with_state(lifecycle), + ); + } + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("test server should bind"); + let addr = listener.local_addr().expect("listener should have address"); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("test server should serve") + }); + ( + RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"), + server, + ) + } + + fn cold_tier() -> ProvisioningTier { + serde_json::from_value(serde_json::json!({ + "name": "COLD", + "type": "S3", + "endpoint": "https://gateway.example.com", + "bucket": "cold", + "prefix": "tenant-a/", + "credsSecret": {"name": "cold-tier"} + })) + .expect("tier should deserialize") + } + + fn live_cold_tier(prefix: &str, access_key: &str) -> String { + serde_json::json!([{ + "type": "s3", + "s3": { + "name": "COLD", + "endpoint": "https://gateway.example.com", + "accessKey": access_key, + "secretKey": "REDACTED", + "bucket": "cold", + "prefix": prefix, + "region": "", + "storageClass": "" + } + }]) + .to_string() + } + + fn tier_credentials(resource_version: &str) -> TierCredentials { + TierCredentials { + access_key: "tier-access".to_string(), + secret_key: "tier-secret".to_string(), + secret_name: "cold-tier".to_string(), + resource_version: Some(resource_version.to_string()), + } + } + + fn owned_tier_status(resource_version: &str) -> ProvisioningItemStatus { + let mut item = ProvisioningItemStatus::new( + "COLD", + ProvisioningItemState::Ready, + Reason::ProvisioningConfigured.as_str(), + ); + let hash = tier_config_hash(&tier_config(&cold_tier())); + item.desired_hash = Some(hash.clone()); + item.last_applied_hash = Some(hash); + item.last_applied_generation = Some(2); + item.observed_secret_resource_version = Some(resource_version.to_string()); + item.observed_secret_name = Some("cold-tier".to_string()); + item.last_applied_access_key_hash = Some(access_key_hash("tier-access")); + item + } + + fn tier_test_tenant(previous: Option) -> Tenant { + Tenant { + metadata: ObjectMeta { + name: Some("tenant-a".to_string()), + namespace: Some("storage".to_string()), + generation: Some(3), + ..Default::default() + }, + spec: crate::types::v1alpha1::tenant::TenantSpec { + tiers: vec![cold_tier()], + ..Default::default() + }, + status: Some(crate::types::v1alpha1::status::Status { + provisioning: ProvisioningStatus { + tiers: previous.into_iter().collect(), + ..Default::default() + }, + ..Default::default() + }), + } + } + + fn tier_run<'a>(ctx: &'a Context, tenant: &'a Tenant) -> ProvisioningRun<'a> { + let mut run = empty_run(ctx, tenant); + run.previous = tenant + .status + .as_ref() + .map(|status| status.provisioning.clone()) + .unwrap_or_default(); + run + } + + fn tier_secret_kube_context(resource_version: &'static str) -> Context { + let kube_service = service_fn(move |request: http::Request| async move { + assert_eq!(request.method(), http::Method::GET); + assert_eq!( + request.uri().path(), + "/api/v1/namespaces/storage/secrets/cold-tier" + ); + let secret = Secret { + metadata: ObjectMeta { + name: Some("cold-tier".to_string()), + namespace: Some("storage".to_string()), + resource_version: Some(resource_version.to_string()), + ..Default::default() + }, + data: Some(BTreeMap::from([ + ("accesskey".to_string(), ByteString(b"tier-access".to_vec())), + ( + "secretkey".to_string(), + ByteString(b"tier-secret\n".to_vec()), + ), + ])), + ..Default::default() + }; + Ok::<_, Infallible>( + http::Response::builder() + .header("content-type", "application/json") + .body(KubeBody::from( + serde_json::to_vec(&secret).expect("Secret should serialize"), + )) + .expect("response should build"), + ) + }); + Context::new(Client::new(kube_service, "default")) + } + + #[tokio::test] + async fn missing_tier_is_registered_with_secret_credentials() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; + let ctx = empty_kube_context(); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); + + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + None, + Ok(tier_credentials("5")), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + let adds = capture.adds.lock().await.clone(); + assert_eq!(adds.len(), 1); + assert_eq!(adds[0].0, "force=false"); assert_eq!( - configuration.rules[0].transition, - Some(AdminLifecycleTransition { - days: 0, - storage_class: "COLD".to_string(), + adds[0].1, + serde_json::json!({ + "type": "s3", + "name": "COLD", + "s3": { + "name": "COLD", + "endpoint": "https://gateway.example.com", + "accessKey": "tier-access", + "secretKey": "tier-secret", + "bucket": "cold", + "prefix": "tenant-a/", + "region": "" + } }) ); + assert!(capture.edits.lock().await.is_empty()); + assert_eq!(item.observed_secret_resource_version.as_deref(), Some("5")); + assert_eq!(item.observed_secret_name.as_deref(), Some("cold-tier")); + assert_eq!(item.last_applied_hash, item.desired_hash); + assert_eq!(item.last_applied_generation, Some(3)); + server.abort(); + } - for (days, storage_class) in [ - (-1, "COLD"), - (1, ""), - (1, "cold"), - (1, "STANDARD"), - (1, "reduced_redundancy"), - (1, "BAD\u{0}TIER"), - ] { - let bucket = transition_lifecycle_bucket(days, storage_class); - assert!( - lifecycle_configuration(bucket.lifecycle.as_ref().expect("lifecycle")).is_err(), - "transition days={days} storageClass={storage_class:?} must be rejected" - ); - } + #[tokio::test] + async fn owned_tier_with_unchanged_secret_is_not_rewritten() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; + let ctx = empty_kube_context(); + let tenant = tier_test_tenant(Some(owned_tier_status("5"))); + let mut run = tier_run(&ctx, &tenant); + let live = parse_live_tier(&live_cold_tier("tenant-a/", "tier-access")); - let mut noncurrent = transition_lifecycle_bucket(1, "COLD"); - let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; - rule.transition = None; - rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { - noncurrent_days: -1, - storage_class: "COLD".to_string(), - }); - assert!( - lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")).is_err() - ); - let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; - rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { - noncurrent_days: 0, - storage_class: "COLD".to_string(), - }); - assert!( - lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")).is_ok(), - "RustFS accepts a zero-day noncurrent transition" - ); - let rule = &mut noncurrent.lifecycle.as_mut().expect("lifecycle").rules[0]; - rule.noncurrent_version_transition = Some(BucketLifecycleNoncurrentVersionTransition { - noncurrent_days: 7, - storage_class: "COLD".to_string(), - }); - let configuration = - lifecycle_configuration(noncurrent.lifecycle.as_ref().expect("lifecycle")) - .expect("noncurrent transition is valid") - .expect("Present lifecycle"); + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + Some(&live), + Ok(tier_credentials("5")), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert!(capture.adds.lock().await.is_empty()); + assert!(capture.edits.lock().await.is_empty()); + assert_eq!(item.last_applied_generation, Some(2)); + server.abort(); + } + + #[tokio::test] + async fn rotated_tier_secret_only_updates_credentials() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; + let ctx = empty_kube_context(); + let tenant = tier_test_tenant(Some(owned_tier_status("5"))); + let mut run = tier_run(&ctx, &tenant); + let live = parse_live_tier(&live_cold_tier("tenant-a/", "tier-access")); + + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + Some(&live), + Ok(tier_credentials("6")), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + assert!(capture.adds.lock().await.is_empty()); assert_eq!( - configuration.rules[0].noncurrent_version_transition, - Some(AdminLifecycleNoncurrentVersionTransition { - noncurrent_days: 7, - storage_class: "COLD".to_string(), - }) + capture.edits.lock().await.clone(), + vec![( + "/rustfs/admin/v3/tier/COLD".to_string(), + serde_json::json!({"access": "tier-access", "secret": "tier-secret"}) + )] ); + assert_eq!(item.observed_secret_resource_version.as_deref(), Some("6")); + assert_eq!(item.last_applied_generation, Some(3)); + server.abort(); } #[tokio::test] - async fn transition_bucket_lifecycle_is_applied_when_live_configuration_is_missing() { - let capture = LifecycleCapture::default(); - *capture.tiers.lock().await = r#"[{"type":"s3","s3":{"name":"COLD"}}]"#.to_string(); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn existing_tier_is_adopted_once_and_out_of_band_access_key_is_corrected() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); + let live = parse_live_tier(&live_cold_tier("tenant-a/", "tier-access")); - let item = - reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + let adopted = reconcile_tier( + &mut run, + &client, + &cold_tier(), + Some(&live), + Ok(tier_credentials("5")), + ) + .await; + + assert_eq!(adopted.state, ProvisioningItemState::Ready.as_str()); + assert!( + adopted + .message + .as_deref() + .is_some_and(|message| message.contains("adopted")) + ); + assert_eq!(capture.edits.lock().await.len(), 1); + + let tenant = tier_test_tenant(Some(adopted)); + let mut run = tier_run(&ctx, &tenant); + let changed = parse_live_tier(&live_cold_tier("tenant-a/", "someone-else")); + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + Some(&changed), + Ok(tier_credentials("5")), + ) + .await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.tier_count.load(Ordering::SeqCst), 1); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 1); - let body = capture.put_body.lock().await; - assert!(body.contains( - "Enabled1COLD" - )); + assert_eq!(capture.edits.lock().await.len(), 2); + assert!(capture.adds.lock().await.is_empty()); server.abort(); } #[tokio::test] - async fn transition_bucket_lifecycle_waits_for_an_unregistered_tier() { - let capture = LifecycleCapture::default(); - *capture.tiers.lock().await = r#"[{"type":"s3","s3":{"name":"WARM"}}]"#.to_string(); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn live_tier_with_different_settings_fails_without_writes() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); + let tenant = tier_test_tenant(Some(owned_tier_status("5"))); + let mut run = tier_run(&ctx, &tenant); + let live = parse_live_tier(&live_cold_tier("other/", "tier-access")); - let item = - reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + Some(&live), + Ok(tier_credentials("6")), + ) + .await; - assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::TierConflict.as_str()); assert!( item.message .as_deref() - .is_some_and(|message| message.contains("COLD")) + .is_some_and(|message| message.contains("prefix")) ); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); - assert!(run.retry.is_some(), "a missing tier must requeue"); - assert!(item.lifecycle_last_applied_hash.is_none()); + assert!(capture.adds.lock().await.is_empty()); + assert!(capture.edits.lock().await.is_empty()); + assert_eq!(item.observed_secret_resource_version.as_deref(), Some("5")); + assert!(run.retry.is_none()); server.abort(); } #[tokio::test] - async fn rustfs_shaped_live_transition_lifecycle_is_adopted_without_write() { - // Shape returned by RustFS GetBucketLifecycleConfiguration for an operator-applied rule. - let live = concat!( - "", - "", - "to-coldEnabled", - "1COLD", - "" - ); - let capture = LifecycleCapture::default(); - *capture.live.lock().await = Some(live.to_string()); - let (client, server) = lifecycle_test_client(capture.clone()).await; + async fn non_empty_remote_prefix_is_a_conflict_unless_forced() { + let capture = TierCapture::default(); + capture.add_status.store(409, Ordering::SeqCst); + let (client, server) = tier_test_client(capture.clone(), None).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); + + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + None, + Ok(tier_credentials("5")), + ) + .await; + + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::TierConflict.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("force: true")) + ); + assert!(item.observed_secret_resource_version.is_none()); + capture.add_status.store(0, Ordering::SeqCst); + let mut forced = cold_tier(); + forced.force = true; let item = - reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + reconcile_tier(&mut run, &client, &forced, None, Ok(tier_credentials("5"))).await; assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(capture.put_count.load(Ordering::SeqCst), 0); - assert_eq!( - item.lifecycle_desired_hash, - item.lifecycle_last_applied_hash - ); + assert_eq!(capture.adds.lock().await[1].0, "force=true"); server.abort(); } #[tokio::test] - async fn anonymous_download_policy_is_applied_when_live_policy_is_missing() { - let capture = PolicyApplyCapture::default(); - let route_capture = capture.clone(); - let router = Router::new().route( - "/app-data", - put({ - let capture = route_capture.clone(); - move |req: Request| { - let capture = capture.clone(); - async move { - if req.uri().query().unwrap_or("").contains("policy") { - let body_bytes = axum::body::to_bytes(req.into_body(), usize::MAX) - .await - .expect("policy body"); - *capture.body.lock().await = - String::from_utf8(body_bytes.to_vec()).expect("utf8"); - } - StatusCode::OK - } - } - }) - .get(|req: Request| async move { - if req.uri().query().unwrap_or("").contains("policy") { - ( - StatusCode::NOT_FOUND, - r#"NoSuchBucketPolicy"#, - ) - } else { - (StatusCode::OK, "") - } + async fn tier_name_and_secret_errors_fail_before_rustfs_writes() { + let capture = TierCapture::default(); + let (client, server) = tier_test_client(capture.clone(), None).await; + let ctx = empty_kube_context(); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); + + let mut lowercase = cold_tier(); + lowercase.name = "cold".to_string(); + let item = reconcile_tier( + &mut run, + &client, + &lowercase, + None, + Ok(tier_credentials("5")), + ) + .await; + assert_eq!(item.reason, Reason::TierApplyFailed.as_str()); + assert!( + item.message + .as_deref() + .is_some_and(|message| message.contains("uppercase")) + ); + + let item = reconcile_tier( + &mut run, + &client, + &cold_tier(), + None, + Err(SpecLoadError { + reason: Reason::TierSecretInvalid, + message: "tier Secret 'cold-tier' was not found".to_string(), + transient: false, }), + ) + .await; + assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(item.reason, Reason::TierSecretInvalid.as_str()); + assert!(capture.adds.lock().await.is_empty()); + assert!(capture.edits.lock().await.is_empty()); + server.abort(); + } + + #[test] + fn redacted_live_access_key_falls_back_to_the_secret_version() { + let previous = owned_tier_status("5"); + for live_access_key in ["", "REDACTED"] { + assert!(!tier_credentials_need_apply( + Some(&previous), + &tier_credentials("5"), + live_access_key + )); + assert!(tier_credentials_need_apply( + Some(&previous), + &tier_credentials("6"), + live_access_key + )); + } + assert!(tier_credentials_need_apply( + Some(&previous), + &tier_credentials("5"), + "someone-else" + )); + } + + #[test] + fn tier_names_follow_rustfs_rules() { + assert!(validate_tier_name("COLD_1").is_ok()); + assert!(validate_tier_name("STORJ-2").is_ok()); + for name in [ + "", + "Cold", + "STANDARD", + "REDUCED_REDUNDANCY", + "BAD\u{1}", + "ÄRCHIV", + "COLD TIER", + "COLD/1", + ] { + assert!( + validate_tier_name(name).is_err(), + "{name:?} must be rejected" + ); + } + } + + #[test] + fn tier_secret_values_are_trimmed_and_required() { + let data = BTreeMap::from([ + ( + "accesskey".to_string(), + ByteString(b" 8charkey \n".to_vec()), + ), + ("secretkey".to_string(), ByteString(b" ".to_vec())), + ("binary".to_string(), ByteString(vec![0xff])), + ("short".to_string(), ByteString(b" 7-chars\n".to_vec())), + ]); + + assert_eq!( + read_tier_secret_value(&data, "accesskey", "cold-tier").as_deref(), + Ok("8charkey") ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); + assert!( + read_tier_secret_value(&data, "short", "cold-tier") + .is_err_and(|message| message.contains("at least 8 characters")) + ); + assert!( + read_tier_secret_value(&data, "secretkey", "cold-tier") + .is_err_and(|message| message.contains("is empty")) + ); + assert!( + read_tier_secret_value(&data, "binary", "cold-tier") + .is_err_and(|message| message.contains("UTF-8")) + ); + assert!( + read_tier_secret_value(&data, "missing", "cold-tier") + .is_err_and(|message| message.contains("missing 'missing'")) + ); + } + + #[tokio::test] + async fn transient_tier_list_failure_keeps_ownership_and_retries() { + let capture = TierCapture::default(); + capture.list_status.store(503, Ordering::SeqCst); + let (client, server) = tier_test_client(capture.clone(), None).await; let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), - anonymous: Some(BucketAnonymousAccess::Download), - ..Default::default() - }; + let tenant = tier_test_tenant(Some(owned_tier_status("5"))); + let mut run = tier_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &bucket).await; - assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); + reconcile_tiers(&mut run, &client).await; + + let item = &run.status.tiers[0]; + assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(item.observed_secret_resource_version.as_deref(), Some("5")); assert!(item.last_applied_hash.is_some()); - let applied = capture.body.lock().await; - assert!(applied.contains("s3:GetObject")); - assert!(!applied.contains("s3:PutObject")); + assert!(run.retry.is_some()); server.abort(); } #[tokio::test] - async fn explicit_private_removes_operator_managed_anonymous_policy() { - let live_policy = canned_anonymous_bucket_policy(BucketAnonymousAccess::Public, "app-data") - .expect("public policy"); - let live_hash = hash_document( - &normalize_policy_document(&live_policy).expect("public policy should normalize"), - ); - let delete_count = Arc::new(AtomicUsize::new(0)); - let route_delete_count = Arc::clone(&delete_count); - let route_live_policy = live_policy.clone(); - let router = Router::new().route( - "/app-data", - put(|| async { StatusCode::OK }) - .get(move |req: Request| { - let live_policy = route_live_policy.clone(); - async move { - assert_eq!(req.uri().query().unwrap_or(""), "policy="); - (StatusCode::OK, live_policy) - } - }) - .delete(move |req: Request| { - let delete_count = Arc::clone(&route_delete_count); - async move { - assert_eq!(req.uri().query().unwrap_or(""), "policy="); - delete_count.fetch_add(1, Ordering::SeqCst); - StatusCode::NO_CONTENT - } - }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let mut previous = ProvisioningItemStatus::new( - "app-data", - ProvisioningItemState::Ready, - Reason::ProvisioningConfigured.as_str(), - ); - previous.last_applied_hash = Some(live_hash.clone()); - run.previous.buckets.push(previous.into()); - let omitted_bucket = ProvisioningBucket { - name: "app-data".to_string(), + async fn spec_tier_is_registered_before_a_bucket_lifecycle_references_it() { + let capture = TierCapture::default(); + *capture.tiers.lock().await = "[]".to_string(); + let lifecycle = LifecycleCapture { + tiers: capture.tiers.clone(), ..Default::default() }; + let (client, server) = tier_test_client(capture.clone(), Some(lifecycle.clone())).await; + let ctx = tier_secret_kube_context("7"); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); - let omitted_item = reconcile_bucket(&mut run, &client, &omitted_bucket).await; + reconcile_tiers(&mut run, &client).await; + let bucket = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; - assert_eq!(omitted_item.state, ProvisioningItemState::Ready.as_str()); assert_eq!( - omitted_item.last_applied_hash.as_deref(), - Some(live_hash.as_str()) + run.status.tiers[0].state, + ProvisioningItemState::Ready.as_str() ); - let mut run = empty_run(&ctx, &tenant); - run.previous.buckets.push(omitted_item); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), - anonymous: Some(BucketAnonymousAccess::Private), - ..Default::default() - }; - - let item = reconcile_bucket(&mut run, &client, &bucket).await; - - assert_eq!(item.state, ProvisioningItemState::Ready.as_str()); - assert_eq!(delete_count.load(Ordering::SeqCst), 1); - assert!(item.desired_hash.is_none()); - assert!(item.last_applied_hash.is_none()); + assert_eq!( + run.status.tiers[0] + .observed_secret_resource_version + .as_deref(), + Some("7") + ); + assert_eq!( + capture.adds.lock().await[0].1["s3"]["secretKey"], + "tier-secret" + ); + assert_eq!(bucket.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(lifecycle.put_count.load(Ordering::SeqCst), 1); + assert!(run.retry.is_none()); server.abort(); } #[tokio::test] - async fn explicit_private_preserves_unmanaged_live_policy() { - let live_policy = canned_anonymous_bucket_policy(BucketAnonymousAccess::Public, "app-data") - .expect("public policy"); - let delete_count = Arc::new(AtomicUsize::new(0)); - let route_delete_count = Arc::clone(&delete_count); - let router = Router::new().route( - "/app-data", - put(|| async { StatusCode::OK }) - .get(move || { - let live_policy = live_policy.clone(); - async move { (StatusCode::OK, live_policy) } - }) - .delete(move || { - let delete_count = Arc::clone(&route_delete_count); - async move { - delete_count.fetch_add(1, Ordering::SeqCst); - StatusCode::NO_CONTENT - } - }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), - anonymous: Some(BucketAnonymousAccess::Private), + async fn conflicting_declared_tier_blocks_the_bucket_lifecycle() { + let capture = TierCapture::default(); + *capture.tiers.lock().await = live_cold_tier("another-tenant/", "tier-access"); + let lifecycle = LifecycleCapture { + tiers: capture.tiers.clone(), ..Default::default() }; - - let item = reconcile_bucket(&mut run, &client, &bucket).await; - - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketPolicyConflict.as_str()); - assert_eq!(delete_count.load(Ordering::SeqCst), 0); + let (client, server) = tier_test_client(capture.clone(), Some(lifecycle.clone())).await; + let ctx = tier_secret_kube_context("7"); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); + + reconcile_tiers(&mut run, &client).await; + let bucket = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(0, "COLD")).await; + + assert_eq!(run.status.tiers[0].reason, Reason::TierConflict.as_str()); + assert!(capture.adds.lock().await.is_empty()); + assert!(capture.edits.lock().await.is_empty()); + assert_eq!(bucket.state, ProvisioningItemState::Failed.as_str()); + assert_eq!(bucket.reason, Reason::BucketLifecycleApplyFailed.as_str()); + assert!( + bucket + .message + .as_deref() + .is_some_and(|message| message.contains("COLD (Failed: TierConflict)")) + ); + assert_eq!(lifecycle.put_count.load(Ordering::SeqCst), 0); server.abort(); } #[tokio::test] - async fn mixed_anonymous_and_custom_policy_fails_before_rustfs_calls() { - let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user, ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); - let bucket = ProvisioningBucket { - name: "app-data".to_string(), - anonymous: Some(BucketAnonymousAccess::Public), - policy: Some(PolicyDocumentSource { - config_map_key_ref: ConfigMapKeyReference { - name: "bucket-policy".to_string(), - key: "policy.json".to_string(), - }, - }), + async fn pending_declared_tier_keeps_the_bucket_lifecycle_pending() { + let capture = TierCapture::default(); + capture.list_status.store(503, Ordering::SeqCst); + let lifecycle = LifecycleCapture { + tiers: capture.tiers.clone(), ..Default::default() }; + let (client, server) = tier_test_client(capture.clone(), Some(lifecycle.clone())).await; + let ctx = tier_secret_kube_context("7"); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); - let item = reconcile_bucket(&mut run, &client, &bucket).await; - assert_eq!(item.state, ProvisioningItemState::Failed.as_str()); - assert_eq!(item.reason, Reason::BucketPolicyConflict.as_str()); + reconcile_tiers(&mut run, &client).await; + let bucket = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "COLD")).await; + + assert_eq!( + run.status.tiers[0].state, + ProvisioningItemState::Pending.as_str() + ); + assert_eq!(bucket.state, ProvisioningItemState::Pending.as_str()); + assert_eq!(bucket.reason, Reason::ProvisioningPending.as_str()); + assert_eq!(lifecycle.put_count.load(Ordering::SeqCst), 0); + assert!(run.retry.is_some()); + server.abort(); } #[tokio::test] - async fn list_canned_policies_503_is_transient() { - let router = Router::new().route( - "/rustfs/admin/v3/list-canned-policies", - get(|| async { StatusCode::SERVICE_UNAVAILABLE }), - ); - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) - .await - .expect("test server should bind"); - let addr = listener.local_addr().expect("listener should have address"); - let server = tokio::spawn(async move { - axum::serve(listener, router) - .await - .expect("test server should serve") - }); - let client = - RustfsAdminClient::new_with_base_url(format!("http://{addr}"), "access", "secret"); - let tenant = Tenant { - metadata: ObjectMeta { - name: Some("tenant-a".to_string()), - namespace: Some("storage".to_string()), - ..Default::default() - }, - spec: crate::types::v1alpha1::tenant::TenantSpec { - policies: vec![ProvisioningPolicy { - name: "app-readwrite".to_string(), - document: PolicyDocumentSource { - config_map_key_ref: ConfigMapKeyReference { - name: "app-policy".to_string(), - key: "policy.json".to_string(), - }, - }, - ..Default::default() - }], - ..Default::default() - }, - status: None, + async fn undeclared_tier_still_only_needs_to_be_registered() { + let capture = TierCapture::default(); + *capture.tiers.lock().await = r#"[{"type":"s3","s3":{"name":"WARM"}}]"#.to_string(); + let lifecycle = LifecycleCapture { + tiers: capture.tiers.clone(), + ..Default::default() }; + let (client, server) = tier_test_client(capture.clone(), Some(lifecycle.clone())).await; + let ctx = tier_secret_kube_context("7"); + let tenant = tier_test_tenant(None); + let mut run = tier_run(&ctx, &tenant); - let error = load_live_policies(&client, &tenant) - .await - .expect_err("503 should fail the live policy load"); - assert!(error.transient); - assert!( - error.message.contains("503") - || error.message.contains("unavailable") - || error.message.contains("failed to list") - ); + reconcile_tiers(&mut run, &client).await; + let bucket = + reconcile_bucket(&mut run, &client, &transition_lifecycle_bucket(1, "WARM")).await; + + assert_eq!(bucket.state, ProvisioningItemState::Ready.as_str()); + assert_eq!(lifecycle.put_count.load(Ordering::SeqCst), 1); server.abort(); } #[tokio::test] - async fn user_stays_pending_when_referenced_policy_is_pending() { + async fn removed_tier_is_retained_with_its_ownership() { let ctx = empty_kube_context(); - let user = provisioning_user("app-user", "app-user-secret", "readwrite"); - let tenant = provisioning_test_tenant(user.clone(), ProvisioningStatus::default()); - let mut run = empty_run(&ctx, &tenant); - let client = RustfsAdminClient::new_with_base_url("http://127.0.0.1:1", "access", "secret"); - let pending = BTreeSet::from(["readwrite".to_string()]); - let credentials = user_credentials("1"); + let mut tenant = tier_test_tenant(Some(owned_tier_status("5"))); + tenant.spec.tiers.clear(); + tenant.spec.buckets = vec![ProvisioningBucket { + name: "app-data".to_string(), + ..Default::default() + }]; + let mut run = tier_run(&ctx, &tenant); - let plan = prepare_user_reconcile( - &mut run, - &client, - &BTreeMap::new(), - &BTreeSet::new(), - &pending, - &user, - &credentials, - ) - .await; + run.prepare_status(ProvisioningPhase::Ready); - let UserReconcilePlan::Complete(item) = plan else { - panic!("pending referenced policy must not call RustFS"); - }; - assert_eq!(item.state, ProvisioningItemState::Pending.as_str()); - assert_eq!(item.reason, Reason::ProvisioningPending.as_str()); - assert!( - item.message - .as_deref() - .is_some_and(|message| message.contains("readwrite")) + assert_eq!(run.status.tiers.len(), 1); + assert_eq!( + run.status.tiers[0].state, + ProvisioningItemState::Retained.as_str() ); - run.push_user(*item); - - match run.finish().outcome { - ProvisioningOutcome::Retry { persist_status, .. } => assert!(persist_status), - _ => panic!("pending referenced policy must requeue instead of failing the user"), + assert_eq!( + run.status.tiers[0] + .observed_secret_resource_version + .as_deref(), + Some("5") + ); + assert!(has_active_spec(&tier_test_tenant(None))); + } + + fn parse_live_tier(body: &str) -> RustfsTier { + let tier: Value = serde_json::from_str::>(body).expect("tier list")[0].clone(); + RustfsTier { + config: RustfsTierConfig { + tier_type: tier["type"].as_str().expect("type").to_string(), + name: tier["s3"]["name"].as_str().expect("name").to_string(), + endpoint: tier["s3"]["endpoint"] + .as_str() + .expect("endpoint") + .to_string(), + bucket: tier["s3"]["bucket"].as_str().expect("bucket").to_string(), + prefix: tier["s3"]["prefix"].as_str().expect("prefix").to_string(), + region: tier["s3"]["region"].as_str().expect("region").to_string(), + }, + access_key: tier["s3"]["accessKey"] + .as_str() + .expect("accessKey") + .to_string(), } } } diff --git a/src/sts/rustfs_client.rs b/src/sts/rustfs_client.rs index b758226..a99b1e8 100644 --- a/src/sts/rustfs_client.rs +++ b/src/sts/rustfs_client.rs @@ -21,6 +21,7 @@ use crate::Tenant; use crate::cluster_dns; pub use rustfs_admin::{ + BucketEncryptionAlgorithm, BucketEncryptionConfiguration, BucketLifecycleAbortIncompleteMultipartUpload, BucketLifecycleConfiguration, BucketLifecycleExpiration, BucketLifecycleNoncurrentVersionTransition, BucketLifecycleRule, BucketLifecycleRuleStatus, BucketLifecycleTransition, BucketObjectLockConfiguration, @@ -28,7 +29,8 @@ pub use rustfs_admin::{ BucketObjectLockRetentionPeriod, BucketVersioningState, BucketVersioningUpdate, CreateBucketResult, RustfsClientError, RustfsCredentials, RustfsErasureBackend, RustfsErasureSetInfo, RustfsPoolDecommissionInfo, RustfsPoolListItem, RustfsPoolStatus, - RustfsServerInfo, RustfsServerUsage, RustfsUserInfo, canonicalize_bucket_lifecycle_xml, + RustfsServerInfo, RustfsServerUsage, RustfsTier, RustfsTierConfig, RustfsUserInfo, + canonicalize_bucket_lifecycle_xml, }; /// Tenant-aware wrapper around the kube-agnostic RustFS admin client. diff --git a/src/tenant_reference_index.rs b/src/tenant_reference_index.rs index 0c6dd31..b346f4a 100644 --- a/src/tenant_reference_index.rs +++ b/src/tenant_reference_index.rs @@ -348,6 +348,16 @@ mod tests { policies: vec!["readwrite".to_string()], ..Default::default() }); + tenant + .spec + .tiers + .push(crate::types::v1alpha1::provisioning::ProvisioningTier { + name: "COLD".to_string(), + creds_secret: crate::types::v1alpha1::provisioning::TierCredentialsSecretRef { + name: "cold-tier-credentials".to_string(), + }, + ..Default::default() + }); tenant.spec.encryption = Some(EncryptionConfig { kms_secret: Some(local_ref("vault-token")), local: Some(LocalKmsConfig { @@ -404,6 +414,7 @@ mod tests { "oidc-extra-ca", "provisioned-user-credentials", "legacy-user", + "cold-tier-credentials", "vault-token", "local-master-key", "external-ca", diff --git a/src/types/v1alpha1.rs b/src/types/v1alpha1.rs index ec2051d..f659238 100755 --- a/src/types/v1alpha1.rs +++ b/src/types/v1alpha1.rs @@ -146,12 +146,53 @@ mod tenant_provisioning_crd_tests { MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS, MAX_POLICIES_PER_USER, }, tenant::{ - MAX_TENANT_BUCKETS, MAX_TENANT_POLICIES, MAX_TENANT_POOLS, MAX_TENANT_USERS, Tenant, + MAX_TENANT_BUCKETS, MAX_TENANT_POLICIES, MAX_TENANT_POOLS, MAX_TENANT_TIERS, + MAX_TENANT_USERS, Tenant, }, }; use kube::CustomResourceExt; use serde_json::json; + #[test] + fn tenant_crd_declares_tiers_with_rustfs_name_rules() { + let crd = serde_json::to_value(Tenant::crd()).expect("Tenant CRD serializes"); + let schema = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]; + let tiers = &schema["properties"]["spec"]["properties"]["tiers"]; + + assert_eq!(tiers["x-kubernetes-list-type"], json!("map")); + assert_eq!(tiers["x-kubernetes-list-map-keys"], json!(["name"])); + assert_eq!(tiers["maxItems"], json!(MAX_TENANT_TIERS)); + let tier = &tiers["items"]; + assert_eq!( + tier["required"], + json!(["bucket", "credsSecret", "endpoint", "name", "type"]) + ); + assert_eq!( + tier["properties"]["type"]["enum"], + json!(["S3", "RustFS", "MinIO"]) + ); + assert_eq!( + tier["properties"]["deletionPolicy"]["enum"], + json!(["Retain"]) + ); + assert_eq!( + tier["properties"]["name"]["x-kubernetes-validations"], + json!([{ + "message": "tier name must not be a reserved storage class", + "rule": "self != 'STANDARD' && self != 'REDUCED_REDUNDANCY'" + }]) + ); + assert_eq!( + tier["properties"]["name"]["pattern"], + json!("^[A-Z0-9_-]+$") + ); + assert_eq!( + schema["properties"]["status"]["properties"]["provisioning"]["properties"]["tiers"]["items"] + ["properties"]["observedSecretResourceVersion"]["type"], + json!("string") + ); + } + #[test] fn tenant_crd_includes_provisioning_spec_status_and_bounded_list_rules() { let crd = serde_json::to_value(Tenant::crd()).expect("Tenant CRD serializes"); @@ -465,6 +506,28 @@ mod tenant_provisioning_crd_tests { object_lock["x-kubernetes-validations"][0]["message"], json!("Present object lock configuration requires mode and days; Absent forbids them") ); + let encryption = &spec["properties"]["buckets"]["items"]["properties"]["encryption"]; + assert_eq!( + encryption["properties"]["mode"]["enum"], + json!(["SSE-S3", "SSE-KMS", null]) + ); + assert_eq!( + encryption["properties"]["kmsKeyId"]["pattern"], + json!("^[A-Za-z0-9._:/+=@-]+$") + ); + let encryption_messages: Vec<_> = encryption["x-kubernetes-validations"] + .as_array() + .expect("encryption validations are present") + .iter() + .map(|validation| validation["message"].clone()) + .collect(); + assert_eq!( + encryption_messages, + vec![ + json!("Present bucket encryption requires mode; Absent forbids mode and kmsKeyId"), + json!("kmsKeyId requires mode SSE-KMS"), + ] + ); let bucket_validations = spec["properties"]["buckets"]["items"]["x-kubernetes-validations"] .as_array() .expect("bucket validations are present"); @@ -492,6 +555,10 @@ mod tenant_provisioning_crd_tests { bucket_status["properties"]["objectLockConfigurationLastAppliedHash"]["nullable"], json!(true) ); + assert_eq!( + bucket_status["properties"]["encryptionLastAppliedHash"]["nullable"], + json!(true) + ); } } diff --git a/src/types/v1alpha1/provisioning.rs b/src/types/v1alpha1/provisioning.rs index 69b8260..ac014e2 100644 --- a/src/types/v1alpha1/provisioning.rs +++ b/src/types/v1alpha1/provisioning.rs @@ -30,6 +30,9 @@ pub(crate) const MAX_BUCKET_NAME_LENGTH: u32 = 63; pub(crate) const MAX_BUCKET_LIFECYCLE_RULES: u32 = 1_000; pub(crate) const MAX_BUCKET_LIFECYCLE_RULE_ID_LENGTH: u32 = 255; pub(crate) const MAX_BUCKET_OBJECT_LOCK_RETENTION_DAYS: u32 = 36_500; +pub(crate) const MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH: u32 = 1024; +pub(crate) const MAX_PROVISIONING_TIER_NAME_LENGTH: u32 = 253; +pub(crate) const MAX_TIER_CREDENTIALS_SECRET_NAME_LENGTH: u32 = 253; #[derive(Deserialize, Serialize, Clone, Debug, JsonSchema, ToSchema, Default, PartialEq, Eq)] #[serde(rename_all = "PascalCase")] @@ -124,6 +127,83 @@ pub(crate) fn duplicate_user_credentials_secret_names( .collect() } +/// Remote backend of a tier. All three use the S3 API with static credentials. +#[derive( + Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, ToSchema, Default, PartialEq, Eq, +)] +pub enum ProvisioningTierType { + #[default] + #[serde(rename = "S3")] + S3, + #[serde(rename = "RustFS")] + RustFS, + #[serde(rename = "MinIO")] + MinIO, +} + +impl ProvisioningTierType { + /// Backend type as named by the RustFS tier admin API. + pub const fn rustfs_type(self) -> &'static str { + match self { + Self::S3 => "s3", + Self::RustFS => "rustfs", + Self::MinIO => "minio", + } + } +} + +/// Reference to a tier credentials Secret in the Tenant namespace. +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, ToSchema, Default, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct TierCredentialsSecretRef { + #[schemars(length(min = 1, max = MAX_TIER_CREDENTIALS_SECRET_NAME_LENGTH))] + pub name: String, +} + +/// Remote storage tier registered in RustFS for bucket lifecycle transitions. +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, ToSchema, Default, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ProvisioningTier { + /// Tier name, referenced as `storageClass` by bucket lifecycle transitions. Uppercase ASCII + /// letters, digits, `_` and `-` only: RustFS looks up a tier by the raw URL path segment. + #[schemars( + length(min = 1, max = MAX_PROVISIONING_TIER_NAME_LENGTH), + regex(pattern = r"^[A-Z0-9_-]+$") + )] + #[x_kube(validation = Rule::new("self != 'STANDARD' && self != 'REDUCED_REDUNDANCY'").message("tier name must not be a reserved storage class"))] + pub name: String, + + #[serde(rename = "type")] + pub tier_type: ProvisioningTierType, + + /// Remote S3 endpoint URL, for example `https://gateway.storjshare.io`. + #[schemars(length(min = 1))] + pub endpoint: String, + + /// Remote bucket that receives transitioned objects. + #[schemars(length(min = 1))] + pub bucket: String, + + /// Object prefix inside the remote bucket. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub prefix: String, + + #[serde(default, skip_serializing_if = "String::is_empty")] + pub region: String, + + /// Secret with `accesskey` and `secretkey` for the remote bucket. Updating the Secret + /// rotates the tier credentials in RustFS. + pub creds_secret: TierCredentialsSecretRef, + + /// Register the tier even when the remote prefix already holds objects, for example a + /// prefix written by this Tenant before a restore. RustFS refuses such a prefix otherwise. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub force: bool, + + #[serde(default, skip_serializing_if = "is_retain")] + pub deletion_policy: ProvisioningDeletionPolicy, +} + #[derive( Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, ToSchema, Default, PartialEq, Eq, )] @@ -187,6 +267,51 @@ impl BucketObjectLockConfiguration { } } +#[derive(Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "PascalCase")] +pub enum BucketEncryptionState { + Present, + Absent, +} + +#[derive(Deserialize, Serialize, Clone, Copy, Debug, JsonSchema, ToSchema, PartialEq, Eq)] +pub enum BucketEncryptionMode { + #[serde(rename = "SSE-S3")] + SseS3, + #[serde(rename = "SSE-KMS")] + SseKms, +} + +/// Default server-side encryption applied by RustFS to new objects in the bucket. +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +#[x_kube(validation = Rule::new("(!has(self.state) || self.state == 'Present') ? has(self.mode) : !has(self.mode) && !has(self.kmsKeyId)").message("Present bucket encryption requires mode; Absent forbids mode and kmsKeyId"))] +#[x_kube(validation = Rule::new("!has(self.kmsKeyId) || (has(self.mode) && self.mode == 'SSE-KMS')").message("kmsKeyId requires mode SSE-KMS"))] +pub struct BucketEncryptionSpec { + /// Omission is equivalent to `Present`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, + + /// `SSE-S3` stores S3 `AES256` and requires Tenant KMS or `RUSTFS_SSE_S3_MASTER_KEY` in + /// `spec.env`; `SSE-KMS` stores `aws:kms` and requires `spec.encryption.enabled` on the Tenant. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + + /// KMS key for `SSE-KMS`. Omission uses the Tenant `spec.encryption.defaultKeyId`. + #[schemars( + length(min = 1, max = MAX_BUCKET_ENCRYPTION_KMS_KEY_ID_LENGTH), + regex(pattern = r"^[A-Za-z0-9._:/+=@-]+$") + )] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub kms_key_id: Option, +} + +impl BucketEncryptionSpec { + pub(crate) fn state(&self) -> BucketEncryptionState { + self.state.unwrap_or(BucketEncryptionState::Present) + } +} + #[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, ToSchema, Default, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct BucketLifecycleRuleFilter { @@ -317,6 +442,11 @@ pub struct ProvisioningBucket { #[serde(default, skip_serializing_if = "Option::is_none")] pub policy: Option, + /// Default bucket encryption. Omission leaves the live configuration unmanaged; `Absent` + /// removes only a configuration previously owned by the operator. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption: Option, + /// Declarative S3 bucket lifecycle configuration. Omission leaves the live lifecycle /// configuration unmanaged; `Absent` removes only a configuration previously owned by the /// operator. @@ -521,6 +651,35 @@ mod tests { assert!(rule.expiration.is_none()); } + #[test] + fn tier_deserializes_issue_yaml_shape_and_omits_defaults() { + let tier: super::ProvisioningTier = serde_json::from_value(serde_json::json!({ + "name": "STORJ", + "type": "S3", + "endpoint": "https://gateway.storjshare.io", + "bucket": "rustfs-ilm", + "prefix": "tenant-a/", + "credsSecret": {"name": "storj-tier"} + })) + .expect("tier should deserialize"); + + assert_eq!(tier.tier_type, super::ProvisioningTierType::S3); + assert_eq!(tier.tier_type.rustfs_type(), "s3"); + assert!(tier.region.is_empty()); + assert!(!tier.force); + let value = serde_json::to_value(&tier).expect("tier serializes"); + assert!(value.get("region").is_none()); + assert!(value.get("force").is_none()); + assert!(value.get("deletionPolicy").is_none()); + assert_eq!(value["type"], "S3"); + + for (name, rustfs_type) in [("RustFS", "rustfs"), ("MinIO", "minio")] { + let tier_type: super::ProvisioningTierType = + serde_json::from_value(serde_json::json!(name)).expect("tier type deserializes"); + assert_eq!(tier_type.rustfs_type(), rustfs_type); + } + } + #[test] fn bucket_object_lock_configuration_preserves_issue_yaml_shape() { let bucket: super::ProvisioningBucket = serde_json::from_value(serde_json::json!({ diff --git a/src/types/v1alpha1/status.rs b/src/types/v1alpha1/status.rs index 0ca12f6..8401376 100755 --- a/src/types/v1alpha1/status.rs +++ b/src/types/v1alpha1/status.rs @@ -189,8 +189,13 @@ pub enum Reason { BucketObjectLockConfigurationConflict, BucketPolicyApplyFailed, BucketPolicyConflict, + BucketEncryptionApplyFailed, + BucketEncryptionConflict, BucketLifecycleApplyFailed, BucketLifecycleConflict, + TierSecretInvalid, + TierApplyFailed, + TierConflict, KubernetesApiError, StatusPatchFailed, ObservedGenerationStale, @@ -273,8 +278,13 @@ impl Reason { Self::BucketObjectLockConfigurationConflict => "BucketObjectLockConfigurationConflict", Self::BucketPolicyApplyFailed => "BucketPolicyApplyFailed", Self::BucketPolicyConflict => "BucketPolicyConflict", + Self::BucketEncryptionApplyFailed => "BucketEncryptionApplyFailed", + Self::BucketEncryptionConflict => "BucketEncryptionConflict", Self::BucketLifecycleApplyFailed => "BucketLifecycleApplyFailed", Self::BucketLifecycleConflict => "BucketLifecycleConflict", + Self::TierSecretInvalid => "TierSecretInvalid", + Self::TierApplyFailed => "TierApplyFailed", + Self::TierConflict => "TierConflict", Self::KubernetesApiError => "KubernetesApiError", Self::StatusPatchFailed => "StatusPatchFailed", Self::ObservedGenerationStale => "ObservedGenerationStale", @@ -560,8 +570,13 @@ pub fn is_blocked_reason(reason: &str) -> bool { | "BucketObjectLockConfigurationConflict" | "BucketPolicyApplyFailed" | "BucketPolicyConflict" + | "BucketEncryptionApplyFailed" + | "BucketEncryptionConflict" | "BucketLifecycleApplyFailed" | "BucketLifecycleConflict" + | "TierSecretInvalid" + | "TierApplyFailed" + | "TierConflict" ) } @@ -679,10 +694,17 @@ pub fn next_actions_for_reason(reason: &str) -> Vec<&'static str> { } "BucketPolicyApplyFailed" => vec!["fixBucketPolicy", "inspectOperatorLogs"], "BucketPolicyConflict" => vec!["inspectLiveBucketPolicy", "updateBucketSpec"], + "BucketEncryptionApplyFailed" => vec!["fixBucketEncryption", "inspectOperatorLogs"], + "BucketEncryptionConflict" => { + vec!["inspectLiveBucketEncryption", "updateBucketSpec"] + } "BucketLifecycleApplyFailed" => vec!["fixBucketLifecycle", "inspectOperatorLogs"], "BucketLifecycleConflict" => { vec!["inspectLiveBucketLifecycle", "updateBucketSpec"] } + "TierSecretInvalid" => vec!["fixTierSecret"], + "TierApplyFailed" => vec!["fixTierSpec", "inspectOperatorLogs"], + "TierConflict" => vec!["inspectLiveTier", "updateTierSpec"], "KubernetesApiError" => vec!["retry", "inspectOperatorLogs"], "ObservedGenerationStale" => vec!["waitForReconcile"], _ => Vec::new(), @@ -800,6 +822,23 @@ mod tests { next_actions_for_reason("BucketLifecycleConflict"), vec!["inspectLiveBucketLifecycle", "updateBucketSpec"] ); + assert!(is_blocked_reason("BucketEncryptionApplyFailed")); + assert!(is_blocked_reason("BucketEncryptionConflict")); + assert_eq!( + next_actions_for_reason("BucketEncryptionApplyFailed"), + vec!["fixBucketEncryption", "inspectOperatorLogs"] + ); + assert_eq!( + next_actions_for_reason("BucketEncryptionConflict"), + vec!["inspectLiveBucketEncryption", "updateBucketSpec"] + ); + for reason in ["TierSecretInvalid", "TierApplyFailed", "TierConflict"] { + assert!(is_blocked_reason(reason), "{reason} must block"); + } + assert_eq!( + next_actions_for_reason("TierConflict"), + vec!["inspectLiveTier", "updateTierSpec"] + ); } #[test] diff --git a/src/types/v1alpha1/status/provisioning.rs b/src/types/v1alpha1/status/provisioning.rs index 9512af4..070c646 100644 --- a/src/types/v1alpha1/status/provisioning.rs +++ b/src/types/v1alpha1/status/provisioning.rs @@ -34,11 +34,17 @@ pub struct ProvisioningStatus { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub buckets: Vec, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tiers: Vec, } impl ProvisioningStatus { pub fn is_empty(&self) -> bool { - self.policies.is_empty() && self.users.is_empty() && self.buckets.is_empty() + self.policies.is_empty() + && self.users.is_empty() + && self.buckets.is_empty() + && self.tiers.is_empty() } } @@ -147,6 +153,12 @@ pub struct ProvisioningBucketStatus { #[serde(default, skip_serializing_if = "Option::is_none")] pub object_lock_configuration_last_applied_hash: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption_desired_hash: Option, + + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encryption_last_applied_hash: Option, } impl ProvisioningBucketStatus { @@ -158,6 +170,8 @@ impl ProvisioningBucketStatus { versioning: None, object_lock_configuration_desired_hash: None, object_lock_configuration_last_applied_hash: None, + encryption_desired_hash: None, + encryption_last_applied_hash: None, } } } diff --git a/src/types/v1alpha1/tenant.rs b/src/types/v1alpha1/tenant.rs index 90cf298..5a387e7 100755 --- a/src/types/v1alpha1/tenant.rs +++ b/src/types/v1alpha1/tenant.rs @@ -19,7 +19,7 @@ use crate::types::v1alpha1::network::NetworkConfig; use crate::types::v1alpha1::pool::{Pool, validate_pool_collection}; use crate::types::v1alpha1::pool_lifecycle::PoolLifecycleSpec; use crate::types::v1alpha1::provisioning::{ - ProvisioningBucket, ProvisioningPolicy, ProvisioningUser, + ProvisioningBucket, ProvisioningPolicy, ProvisioningTier, ProvisioningUser, }; use crate::types::v1alpha1::security_context::PodSecurityContextOverride; use crate::types::v1alpha1::tls::TlsConfig; @@ -44,6 +44,7 @@ pub(crate) const MAX_TENANT_POOLS: u32 = 32; pub(crate) const MAX_TENANT_POLICIES: u32 = 256; pub(crate) const MAX_TENANT_USERS: u32 = 256; pub(crate) const MAX_TENANT_BUCKETS: u32 = 1024; +pub(crate) const MAX_TENANT_TIERS: u32 = 64; pub(crate) const RUSTFS_TENANT_LABEL: &str = "rustfs.tenant"; /// Reference to one key in a namespaced Kubernetes Secret. @@ -283,6 +284,15 @@ pub struct TenantSpec { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub buckets: Vec, + /// Remote storage tiers that bucket lifecycle transitions can reference by name. + /// Tiers are reconciled before buckets. + #[schemars( + length(max = MAX_TENANT_TIERS), + extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["name"]) + )] + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tiers: Vec, + /// Encryption / KMS configuration for server-side encryption. /// When enabled, the operator injects KMS environment variables and mounts /// secrets into RustFS pods so the in-process `rustfs-kms` library is configured. @@ -344,6 +354,9 @@ impl TenantSpec { for user in &self.users { insert(user.credentials_secret_name()); } + for tier in &self.tiers { + insert(&tier.creds_secret.name); + } } if let Some(tls) = &self.tls { @@ -640,6 +653,13 @@ mod tests { "policies": ["readwrite"] } ], + "tiers": [{ + "name": "COLD", + "type": "S3", + "endpoint": "https://s3.example.com", + "bucket": "cold", + "credsSecret": { "name": "cold-tier-creds" } + }], "tls": { "certManager": { "secretName": "default-tls", @@ -676,6 +696,7 @@ mod tests { spec.referenced_secret_names(), BTreeSet::from([ "admin-creds".to_string(), + "cold-tier-creds".to_string(), "default-ca".to_string(), "default-tls".to_string(), "env-secret".to_string(),