diff --git a/.github/config/test-matrix.yaml b/.github/config/test-matrix.yaml index 96560a0..091bf4a 100644 --- a/.github/config/test-matrix.yaml +++ b/.github/config/test-matrix.yaml @@ -12,17 +12,25 @@ # Structure: # overlays: # : -# condition-overrides: "= ..." +# condition-overrides: "= ..." or "/= ..." # # Fields: -# condition-overrides Space-separated list of apiGroup=Condition -# pairs. During verification, custom resources -# belonging to the given API group will be -# checked for the specified condition instead -# of the default "Ready". +# condition-overrides Space-separated list of overrides, each either +# apiGroup=Condition or the more specific +# apiGroup/Kind=Condition (which takes precedence +# when both match a resource). During verification, +# custom resources matching the given API group (and +# Kind, if specified) will be checked for the given +# condition instead of the default "Ready". overlays: metrics: condition-overrides: "monitoring.coreos.com=Available" mcp-metrics: condition-overrides: "monitoring.coreos.com=Available" + kroxylicious: + condition-overrides: >- + kroxylicious.io/KafkaProxyIngress=Accepted + kroxylicious.io/KafkaService=ResolvedRefs + kroxylicious.io/VirtualKafkaCluster=Accepted + kroxylicious.io/KafkaProtocolFilter=ResolvedRefs diff --git a/.github/scripts/VerifyInstall.java b/.github/scripts/VerifyInstall.java index 6dd78f6..a3013c4 100644 --- a/.github/scripts/VerifyInstall.java +++ b/.github/scripts/VerifyInstall.java @@ -33,7 +33,9 @@ *

Environment variables: *

    *
  • {@code OVERLAY} — overlay name (default: "core")
    • - *
  • {@code CONDITION_OVERRIDES} — space-separated apiGroup=Condition pairs (default condition: Ready)
    • + *
  • {@code CONDITION_OVERRIDES} — space-separated overrides (default condition: Ready). Each entry is + * either {@code apiGroup=Condition} or the more specific {@code apiGroup/Kind=Condition}, which takes + * precedence when both match a resource.
    • *
  • {@code TIMEOUT} — wait timeout with unit suffix, e.g. "300s", "5m" (default: "600s")
    • *
*/ @@ -105,7 +107,8 @@ public static void main(String[] args) { // Step 4: Wait for custom resources to be ready for (ScriptUtils.ResourceRef ref : expectedCRs) { - String condition = conditionOverrides.getOrDefault(ref.apiGroup(), "Ready"); + String condition = conditionOverrides.getOrDefault(ref.apiGroup() + "/" + ref.kind, + conditionOverrides.getOrDefault(ref.apiGroup(), "Ready")); System.out.println("--- " + ref.kind + "/" + ref.name + " (" + ref.namespace + ") waiting for condition=" + condition + " ---"); diff --git a/components/kroxylicious/base/kustomization.yaml b/components/kroxylicious/base/kustomization.yaml new file mode 100644 index 0000000..9576367 --- /dev/null +++ b/components/kroxylicious/base/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - https://github.com/kroxylicious/kroxylicious/releases/download/v0.25.0/kroxylicious-operator-0.25.0-install.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: streamshub-developer-quickstart + includeSelectors: false + +patches: + # Upstream requests a full CPU, which starves the single-node dev cluster + # (the Kafka entity-operator fails to schedule). Trim it to dev-sized values. + - target: + kind: Deployment + name: kroxylicious-operator + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kroxylicious-operator + spec: + template: + spec: + containers: + - name: operator + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/components/kroxylicious/stack/kroxylicious/kroxylicious.yaml b/components/kroxylicious/stack/kroxylicious/kroxylicious.yaml new file mode 100644 index 0000000..35cdd4b --- /dev/null +++ b/components/kroxylicious/stack/kroxylicious/kroxylicious.yaml @@ -0,0 +1,62 @@ +apiVersion: kroxylicious.io/v1alpha1 +kind: KafkaProxy +metadata: + name: kroxylicious +spec: + infrastructure: + proxyContainer: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: kroxylicious.io/v1alpha1 +kind: KafkaProxyIngress +metadata: + name: cluster-ip +spec: + proxyRef: + name: kroxylicious + clusterIP: + protocol: TCP +--- +apiVersion: kroxylicious.io/v1alpha1 +kind: KafkaService +metadata: + name: dev-cluster +spec: + bootstrapServers: dev-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092 + nodeIdRanges: + - start: 0 + end: 0 +--- +apiVersion: kroxylicious.io/v1alpha1 +kind: KafkaProtocolFilter +metadata: + name: protocol-logger +spec: + type: ProtocolLogger + configTemplate: + logLevel: INFO + apiKeyNames: + - PRODUCE + - FETCH + - METADATA +--- +apiVersion: kroxylicious.io/v1alpha1 +kind: VirtualKafkaCluster +metadata: + name: dev-cluster +spec: + proxyRef: + name: kroxylicious + targetKafkaServiceRef: + name: dev-cluster + ingresses: + - ingressRef: + name: cluster-ip + filterRefs: + - name: protocol-logger diff --git a/components/kroxylicious/stack/kroxylicious/kustomization.yaml b/components/kroxylicious/stack/kroxylicious/kustomization.yaml new file mode 100644 index 0000000..8bce9de --- /dev/null +++ b/components/kroxylicious/stack/kroxylicious/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kroxylicious + +resources: + - namespace.yaml + - kroxylicious.yaml + +labels: + - pairs: + app.kubernetes.io/part-of: streamshub-developer-quickstart + includeSelectors: false diff --git a/components/kroxylicious/stack/kroxylicious/namespace.yaml b/components/kroxylicious/stack/kroxylicious/namespace.yaml new file mode 100644 index 0000000..83445e4 --- /dev/null +++ b/components/kroxylicious/stack/kroxylicious/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kroxylicious diff --git a/components/kroxylicious/stack/kustomization.yaml b/components/kroxylicious/stack/kustomization.yaml new file mode 100644 index 0000000..9a4769f --- /dev/null +++ b/components/kroxylicious/stack/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - kroxylicious diff --git a/docs/overlays/_index.md b/docs/overlays/_index.md index 17cea70..9d0ffed 100644 --- a/docs/overlays/_index.md +++ b/docs/overlays/_index.md @@ -26,6 +26,7 @@ curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/ | [metrics](metrics.md) | Adds Prometheus Operator, a Prometheus instance, and Kafka metrics collection via PodMonitors. Wires the Console to display metrics. | | [mcp](mcp.md) | Adds the StreamsHub MCP server, giving AI assistants read-only access to the Strimzi-managed Kafka cluster. | | [mcp-metrics](mcp-metrics.md) | Combines Prometheus monitoring with the StreamsHub MCP server. The MCP server uses Prometheus as its centralized metrics provider. | +| [kroxylicious](kroxylicious.md) | Adds a Kroxylicious proxy in front of the Kafka cluster, demonstrating protocol-level request/response filtering. | ## Developing Overlays diff --git a/docs/overlays/kroxylicious.md b/docs/overlays/kroxylicious.md new file mode 100644 index 0000000..2a49c04 --- /dev/null +++ b/docs/overlays/kroxylicious.md @@ -0,0 +1,168 @@ ++++ +title = 'Kroxylicious' +weight = 4 +cpu_total = '3 CPU cores' +memory_total = '5 GiB' ++++ + +The Kroxylicious overlay extends the core stack with a [Kroxylicious](https://kroxylicious.io/) proxy deployed in front of the Strimzi-managed Kafka cluster, demonstrating request/response interception via a protocol filter. + +## Quick-Start Install + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/install.sh | OVERLAY=kroxylicious bash +``` + +## Manual Install + +If you prefer step-by-step control, the Kroxylicious overlay uses `overlays/kroxylicious` instead of `overlays/core`: + +```shell +# Phase 1: Operators and CRDs +kubectl apply --server-side --force-conflicts -k 'https://github.com/streamshub/developer-quickstart//overlays/kroxylicious/base?ref=main' + +# Optionally, wait for the operators to be ready +kubectl wait --for=condition=Available deployment/strimzi-cluster-operator -n strimzi --timeout=120s +kubectl wait --for=condition=Available deployment/kroxylicious-operator -n kroxylicious-operator --timeout=120s + +# Phase 2: Operands and virtual cluster +kubectl apply -k 'https://github.com/streamshub/developer-quickstart//overlays/kroxylicious/stack?ref=main' +``` + +## Verify the Installation + +Confirm the operator and proxy resources are ready: + +```shell +kubectl get deployment -n kroxylicious-operator kroxylicious-operator +kubectl get kafkaproxy,kafkaproxyingress,kafkaservice,virtualkafkacluster,kafkaprotocolfilter -n kroxylicious +``` + +Check that the `VirtualKafkaCluster` reports `Accepted=True`: + +```shell +kubectl get virtualkafkacluster dev-cluster -n kroxylicious -o yaml +``` + +Then follow [Trying It Out](#trying-it-out) above to confirm traffic actually flows through the proxy and the +`ProtocolLogger` filter is logging it. + + +## Uninstall + +```shell +curl -sL https://raw.githubusercontent.com/streamshub/developer-quickstart/main/uninstall.sh | OVERLAY=kroxylicious bash +``` + +> **Note:** `uninstall.sh`'s shared-CRD detection (which retains CRDs still in use by other CRs on the +> cluster) only applies to Strimzi, Apicurio Registry, and the StreamsHub Console, since their CRDs are +> commonly used by workloads outside this quick-start. Kroxylicious's CRDs are always fully removed when +> the `kroxylicious` overlay is uninstalled, regardless of whether Strimzi/Apicurio/Console resources are +> shared, the same behavior the `metrics` overlay uses for Prometheus. If you have Kroxylicious resources +> managed outside this quick-start on the same cluster, back them up before uninstalling. + +## What Gets Added + +On top of the components provided by the core overlay, the Kroxylicious overlay adds: + +| Component | Namespace | Description | +|----------------------------------------|--------------------------|-----------------------------------------------------------------------------------| +| Kroxylicious operator | `kroxylicious-operator` | Reconciles `KafkaProxy`, `KafkaService`, `VirtualKafkaCluster`, and related CRs | +| KafkaProxy `kroxylicious` | `kroxylicious` | The running proxy instance | +| KafkaProxyIngress `cluster-ip` | `kroxylicious` | Exposes the proxy via a ClusterIP Service | +| KafkaService `dev-cluster` | `kroxylicious` | References the target Strimzi `dev-cluster` Kafka cluster | +| VirtualKafkaCluster `dev-cluster` | `kroxylicious` | The virtual cluster clients connect through | +| KafkaProtocolFilter `protocol-logger` | `kroxylicious` | Logs PRODUCE, FETCH, and METADATA requests/responses at INFO level | + +## Resource Requirements + +The Kroxylicious overlay requires at least {{< param cpu_total >}} and {{< param memory_total >}} of allocatable cluster resources. +This includes the resources for the [core](core.md) stack plus the Kroxylicious components listed above. + +## Accessing the Virtual Cluster + +From inside the cluster, Kafka clients can connect through the virtual cluster at: + +``` +dev-cluster-cluster-ip-bootstrap.kroxylicious.svc.cluster.local:9292 +``` + +To access it from outside the cluster, port-forward the Service: + +```shell +kubectl port-forward -n kroxylicious svc/dev-cluster-cluster-ip-bootstrap 9292:9292 +``` + +## What the Protocol Filter Does + +The overlay attaches a single [`ProtocolLogger`](https://kroxylicious.io/) filter (`KafkaProtocolFilter/protocol-logger`) +to the virtual cluster. `ProtocolLogger` doesn't modify traffic: it's a read-only filter that logs the full +JSON representation of each Kafka request and response as it passes through the proxy, for `PRODUCE`, `FETCH`, +and `METADATA` API calls (configured via `spec.configTemplate.apiKeyNames`). It's a simple way to see, +concretely, that the proxy sits in the data path and is intercepting real protocol traffic between your +client and the Strimzi-managed `dev-cluster`, before reaching for a filter that actually changes behavior +(e.g. `RecordEncryption`, `Authorization`, or `RecordValidation`; see the +[Kroxylicious filter docs](https://kroxylicious.io/kroxylicious-filters/) for the full catalogue). + +Each log entry is a JSON object with a `header` (`type`: `REQUEST`/`RESPONSE`, `apiKey`, `apiVersion`, +`correlationId`, `clientId`) followed by the decoded message body, so you can see, for example, the exact +topic names and partition assignments inside a `PRODUCE` request, or the record batch metadata in its +corresponding response. + +### Trying It Out + +Run a throwaway Kafka client pod inside the cluster and talk to the virtual cluster's bootstrap address +instead of the real Kafka cluster directly: + +```shell +kubectl run kafka-client --restart=Never --image=quay.io/strimzi/kafka:1.1.0-kafka-4.3.0 -n kroxylicious \ + --command -- sleep 600 +kubectl wait --for=condition=Ready pod/kafka-client -n kroxylicious --timeout=90s + +# Create a topic through the proxy +kubectl exec -n kroxylicious kafka-client -- bin/kafka-topics.sh \ + --bootstrap-server dev-cluster-cluster-ip-bootstrap.kroxylicious.svc.cluster.local:9292 \ + --create --topic demo-topic --partitions 1 --replication-factor 1 + +# Produce a message through the proxy +kubectl exec -n kroxylicious kafka-client -- bash -c \ + 'echo hello-kroxylicious | bin/kafka-console-producer.sh \ + --bootstrap-server dev-cluster-cluster-ip-bootstrap.kroxylicious.svc.cluster.local:9292 --topic demo-topic' + +# Consume it back through the proxy +kubectl exec -n kroxylicious kafka-client -- bin/kafka-console-consumer.sh \ + --bootstrap-server dev-cluster-cluster-ip-bootstrap.kroxylicious.svc.cluster.local:9292 \ + --topic demo-topic --from-beginning --max-messages 1 --timeout-ms 20000 +``` + +Then look at the proxy's logs to see the intercepted traffic: + +```shell +kubectl logs -n kroxylicious deployment/kroxylicious | grep -A20 "ProtocolLoggerFilter" +``` + +You should see paired `REQUEST`/`RESPONSE` JSON log entries for the `PRODUCE` call (and `METADATA`/`FETCH` +calls made by the client/consumer as part of connecting). Clean up the test pod when you're done: + +```shell +kubectl delete pod kafka-client -n kroxylicious +``` + +## Troubleshooting + +### Proxy Resources Not Becoming Ready + +```shell +# Check operator logs +kubectl logs -n kroxylicious-operator deployment/kroxylicious-operator + +# Check conditions on each CR +kubectl get kafkaservice dev-cluster -n kroxylicious -o jsonpath='{.status.conditions}' +kubectl get virtualkafkacluster dev-cluster -n kroxylicious -o jsonpath='{.status.conditions}' +``` + +**Common causes:** + +- Strimzi Kafka cluster not yet ready: the `KafkaService` requires `dev-cluster-kafka-bootstrap.kafka.svc.cluster.local:9092` to be reachable +- Kroxylicious CRDs not installed: verify with `kubectl get crd | grep kroxylicious.io` +- Insufficient cluster resources: check node resource availability diff --git a/overlays/kroxylicious/base/kustomization.yaml b/overlays/kroxylicious/base/kustomization.yaml new file mode 100644 index 0000000..8bdfb06 --- /dev/null +++ b/overlays/kroxylicious/base/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../core/base + +components: + - ../../../components/kroxylicious/base diff --git a/overlays/kroxylicious/stack/console-virtual-cluster-patch.yaml b/overlays/kroxylicious/stack/console-virtual-cluster-patch.yaml new file mode 100644 index 0000000..85dad21 --- /dev/null +++ b/overlays/kroxylicious/stack/console-virtual-cluster-patch.yaml @@ -0,0 +1,10 @@ +- op: replace + path: /spec/kafkaClusters + value: + - name: dev-cluster + namespace: kroxylicious + kind: virtualkafkaclusters.kroxylicious.io + properties: + values: + - name: bootstrap.servers + value: dev-cluster-cluster-ip-bootstrap.kroxylicious.svc.cluster.local:9292 diff --git a/overlays/kroxylicious/stack/kustomization.yaml b/overlays/kroxylicious/stack/kustomization.yaml new file mode 100644 index 0000000..946a652 --- /dev/null +++ b/overlays/kroxylicious/stack/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../components/core/stack + - ../../../components/kroxylicious/stack + +patches: + - target: + group: console.streamshub.github.com + version: v1alpha1 + kind: Console + name: streamshub-console + path: console-virtual-cluster-patch.yaml diff --git a/uninstall.sh b/uninstall.sh index 5cc38b8..3662911 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -491,6 +491,14 @@ main() { prom_url=$(kustomize_url "components/metrics/base/prometheus-operator") kubectl delete -k "${prom_url}" --ignore-not-found=true 2>/dev/null || true fi + + # Remove kroxylicious-operator if kroxylicious overlay was used + if [ "${OVERLAY}" = "kroxylicious" ]; then + info " Removing Kroxylicious operator..." + local kroxylicious_url + kroxylicious_url=$(kustomize_url "components/kroxylicious/base") + kubectl delete -k "${kroxylicious_url}" --ignore-not-found=true 2>/dev/null || true + fi fi echo ""