diff --git a/CHANGELOG.md b/CHANGELOG.md index 323d05c6..7d34355e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ All notable changes to this project will be documented in this file. - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#960], [#961]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#968]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#971]). +- Removed the product-config based configuration validation. Config and environment overrides are + now merged directly from the CRD into the validated cluster, the Java-properties writer is + vendored locally, and the `product-config` crate dependency is dropped. The `--product-config` + CLI flag is now a no-op ([#976]). [#953]: https://github.com/stackabletech/kafka-operator/pull/953 [#960]: https://github.com/stackabletech/kafka-operator/pull/960 @@ -26,6 +30,7 @@ All notable changes to this project will be documented in this file. [#968]: https://github.com/stackabletech/kafka-operator/pull/968 [#971]: https://github.com/stackabletech/kafka-operator/pull/971 [#973]: https://github.com/stackabletech/kafka-operator/pull/973 +[#976]: https://github.com/stackabletech/kafka-operator/pull/976 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.lock b/Cargo.lock index 5357bf1f..1390fe52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1473,6 +1473,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", "thiserror 1.0.69", @@ -1517,11 +1518,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1842,9 +1843,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1856,9 +1857,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1868,9 +1869,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1881,9 +1882,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1895,14 +1896,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1913,21 +1914,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2210,6 +2212,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2349,9 +2360,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2367,9 +2378,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2812,11 +2820,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2844,9 +2852,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2889,7 +2897,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2901,7 +2909,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2920,12 +2928,11 @@ dependencies = [ "const_format", "futures", "indoc", - "product-config", "rstest", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2935,7 +2942,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -2947,6 +2954,7 @@ dependencies = [ "futures", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2959,7 +2967,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2971,12 +2979,14 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -2986,8 +2996,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -2996,15 +3006,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3015,7 +3025,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3028,21 +3038,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3060,7 +3070,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3076,7 +3086,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3412,6 +3422,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3523,9 +3544,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3641,6 +3662,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3936,9 +3967,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index ff2e1a37..321c6fe5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4690,6 +4690,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4705,6 +4710,10 @@ rec { } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4716,7 +4725,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4842,8 +4851,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "k8s_version"; @@ -4862,7 +4871,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -6043,9 +6052,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6082,24 +6091,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6142,18 +6151,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6189,16 +6195,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6259,10 +6265,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6287,16 +6292,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6309,27 +6317,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6373,30 +6381,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6404,9 +6411,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6432,6 +6439,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6456,10 +6470,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6476,15 +6498,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -6993,7 +7014,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7261,6 +7282,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7690,9 +7739,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7709,7 +7758,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7729,62 +7778,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7795,27 +7826,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7824,17 +7855,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7843,33 +7874,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7881,40 +7906,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9288,29 +9310,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9378,11 +9396,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9408,7 +9426,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9516,8 +9534,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_certs"; @@ -9576,7 +9594,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9649,10 +9667,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "serde"; packageId = "serde"; @@ -9664,7 +9678,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9711,8 +9725,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_operator"; @@ -9763,6 +9777,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9770,6 +9788,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9819,7 +9838,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9873,12 +9892,21 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9891,8 +9919,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -9922,12 +9950,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_shared"; @@ -9972,7 +10000,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10003,12 +10031,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_telemetry"; @@ -10052,7 +10080,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10060,7 +10088,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10117,8 +10145,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_versioned"; @@ -10152,7 +10180,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10167,8 +10195,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -10235,8 +10263,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_webhook"; @@ -10309,7 +10337,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11400,6 +11428,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.5"; + edition = "2021"; + sha256 = "16bk1cxi2m0xgaabf98nnj7dn9j16ymkh27jq4s3shjm4a85m1ra"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11856,9 +11911,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -12260,6 +12315,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -13918,9 +14033,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index 8620a9ef..02c687e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/kafka-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.1", features = ["crds", "webhook"] } anyhow = "1.0" @@ -30,5 +29,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/crate-hashes.json b/crate-hashes.json index 86f2b840..5b0037c5 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/deploy/helm/kafka-operator/configs/properties.yaml b/deploy/helm/kafka-operator/configs/properties.yaml index b6f80cdd..9bd8c3b2 100644 --- a/deploy/helm/kafka-operator/configs/properties.yaml +++ b/deploy/helm/kafka-operator/configs/properties.yaml @@ -1,152 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$" - - - unit: &unitUrl - name: "url" - regex: "^((https?|ftp|file)://)?[-a-zA-Z0-9+&@#}/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]" - examples: - - "https://www.stackable.de/blog/" - - - unit: &unitCapacity - name: "capacity" - regex: "^[1-9]\\d*$" - - - unit: &unitMilliseconds - name: "milliseconds" - regex: "^[1-9]\\d*$" - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - - name: "controller" - required: true - asOfVersion: "0.0.0" - comment: "TTL for domain names that cannot be resolved." - description: "TTL for domain names that cannot be resolved." - - - property: &opaAuthorizerClassName - propertyNames: - - name: "authorizer.class.name" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "com.bisnode.kafka.authorization.OpaAuthorizer" - - fromVersion: "3.0.0" - value: "org.openpolicyagent.kafka.OpaAuthorizer" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer class name" - - - property: &opaAuthorizerUrl - propertyNames: - - name: "opa.authorizer.url" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "string" - unit: *unitUrl - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer URL" - - - property: &opaAuthorizerInitialCacheCapacity - propertyNames: - - name: "opa.authorizer.cache.initial.capacity" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA Authorizer initial cache capacity" - - - property: &opaAuthorizerMaxCacheSize - propertyNames: - - name: "opa.authorizer.cache.maximum.size" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "OPA authorizer max cache size" - - - property: &opaAuthorizerCacheExpireAfterSeconds - propertyNames: - - name: "opa.authorizer.cache.expire.after.seconds" - kind: - type: "file" - file: "broker.properties" - datatype: - type: "integer" - unit: *unitCapacity - defaultValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: false - asOfVersion: "0.0.0" - description: "The number of seconds after which the OPA authorizer cache expires" + units: [] +properties: [] diff --git a/docs/modules/kafka/pages/reference/commandline-parameters.adoc b/docs/modules/kafka/pages/reference/commandline-parameters.adoc index 9059a960..3c66dc41 100644 --- a/docs/modules/kafka/pages/reference/commandline-parameters.adoc +++ b/docs/modules/kafka/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-kafka-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/kafka/pages/reference/environment-variables.adoc b/docs/modules/kafka/pages/reference/environment-variables.adoc index cc7dd3a2..d2271300 100644 --- a/docs/modules/kafka/pages/reference/environment-variables.adoc +++ b/docs/modules/kafka/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/kafka-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-kafka-operator run ----- - -or via docker: - ----- -docker run \ - --name kafka-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/kafka-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces diff --git a/extra/crds.yaml b/extra/crds.yaml index a2c3b7d9..9a83de14 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -541,6 +541,7 @@ spec: properties: broker.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -551,6 +552,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1158,6 +1160,7 @@ spec: properties: broker.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1168,6 +1171,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1786,6 +1790,7 @@ spec: properties: controller.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1796,6 +1801,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2235,6 +2241,7 @@ spec: properties: controller.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2245,6 +2252,7 @@ spec: type: object security.properties: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index f2903572..b9d9d40f 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true indoc.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a064c1ac..8f495905 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,9 +1,8 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::KafkaCluster`]. -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use const_format::concatcp; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, @@ -26,6 +25,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; +pub(crate) mod build; mod dereference; mod validate; @@ -39,7 +39,6 @@ use crate::{ discovery::{self, build_discovery_configmap}, operations::pdb::add_pdbs, resource::{ - configmap::build_rolegroup_config_map, listener::build_broker_rolegroup_bootstrap_listener, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, statefulset::{build_broker_rolegroup_statefulset, build_controller_rolegroup_statefulset}, @@ -51,7 +50,6 @@ pub const KAFKA_FULL_CONTROLLER_NAME: &str = concatcp!(KAFKA_CONTROLLER_NAME, '. pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -114,9 +112,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::role::Error }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -156,9 +151,6 @@ pub enum Error { #[snafu(display("KafkaCluster object is misconfigured"))] MisconfiguredKafkaCluster { source: crd::Error }, - #[snafu(display("failed to parse role: {source}"))] - ParseRole { source: strum::ParseError }, - #[snafu(display("failed to build statefulset"))] BuildStatefulset { source: crate::resource::statefulset::Error, @@ -166,7 +158,7 @@ pub enum Error { #[snafu(display("failed to build configmap"))] BuildConfigMap { - source: crate::resource::configmap::Error, + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to build service"))] @@ -198,7 +190,6 @@ impl ReconcilerError for Error { Error::ApplyDiscoveryConfig { .. } => None, Error::DeleteOrphans { .. } => None, Error::CreateClusterResources { .. } => None, - Error::FailedToResolveConfig { .. } => None, Error::ApplyServiceAccount { .. } => None, Error::ApplyRoleBinding { .. } => None, Error::ApplyStatus { .. } => None, @@ -207,7 +198,6 @@ impl ReconcilerError for Error { Error::GetRequiredLabels { .. } => None, Error::InvalidKafkaCluster { .. } => None, Error::MisconfiguredKafkaCluster { .. } => None, - Error::ParseRole { .. } => None, Error::BuildStatefulset { .. } => None, Error::BuildConfigMap { .. } => None, Error::BuildService { .. } => None, @@ -238,18 +228,13 @@ pub async fn reconcile_kafka( .context(DereferenceSnafu)?; // validate (no client required) - let validate::ValidatedInputs { + let validate::ValidatedKafkaCluster { authorization_config, image, kafka_security, - role_config: validated_config, - } = validate::validate( - kafka, - dereferenced_objects, - &ctx.operator_environment, - &ctx.product_config, - ) - .context(ValidateClusterSnafu)?; + role_groups, + } = validate::validate(kafka, dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let opa_connect = authorization_config .as_ref() @@ -295,15 +280,9 @@ pub async fn reconcile_kafka( let mut bootstrap_listeners = Vec::::new(); - for (kafka_role_str, role_config) in &validated_config { - let kafka_role = KafkaRole::from_str(kafka_role_str).context(ParseRoleSnafu)?; - - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup_ref = kafka.rolegroup_ref(&kafka_role, rolegroup_name); - - let merged_config = kafka_role - .merged_config(kafka, &rolegroup_ref.role_group) - .context(FailedToResolveConfigSnafu)?; + for (kafka_role, rg_map) in &role_groups { + for (rolegroup_name, validated_rg) in rg_map { + let rolegroup_ref = kafka.rolegroup_ref(kafka_role, rolegroup_name); let rg_headless_service = build_rolegroup_headless_service(kafka, &image, &rolegroup_ref, &kafka_security) @@ -328,13 +307,14 @@ pub async fn reconcile_kafka( ) .context(BuildPodDescriptorsSnafu)?; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = build::config_map::build_rolegroup_config_map( kafka, &image, &kafka_security, &rolegroup_ref, - rolegroup_config, - &merged_config, + validated_rg.config_file_overrides.clone(), + validated_rg.jvm_security_overrides.clone(), + &validated_rg.merged_config, &kafka_listeners, &pod_descriptors, opa_connect.as_deref(), @@ -344,37 +324,37 @@ pub async fn reconcile_kafka( let rg_statefulset = match kafka_role { KafkaRole::Broker => build_broker_rolegroup_statefulset( kafka, - &kafka_role, + kafka_role, &image, &rolegroup_ref, - rolegroup_config, + &validated_rg.env_overrides, &kafka_security, - &merged_config, + &validated_rg.merged_config, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, KafkaRole::Controller => build_controller_rolegroup_statefulset( kafka, - &kafka_role, + kafka_role, &image, &rolegroup_ref, - rolegroup_config, + &validated_rg.env_overrides, &kafka_security, - &merged_config, + &validated_rg.merged_config, &rbac_sa, &client.kubernetes_cluster_info, ) .context(BuildStatefulsetSnafu)?, }; - if let AnyConfig::Broker(broker_config) = merged_config { + if let AnyConfig::Broker(broker_config) = &validated_rg.merged_config { let rg_bootstrap_listener = build_broker_rolegroup_bootstrap_listener( kafka, &image, &kafka_security, &rolegroup_ref, - &broker_config, + broker_config, ) .context(BuildListenerSnafu)?; bootstrap_listeners.push( @@ -417,12 +397,12 @@ pub async fn reconcile_kafka( ); } - let role_config = kafka.role_config(&kafka_role); + let role_cfg = kafka.role_config(kafka_role); if let Some(GenericRoleConfig { pod_disruption_budget: pdb, - }) = role_config + }) = role_cfg { - add_pdbs(pdb, kafka, &kafka_role, client, &mut cluster_resources) + add_pdbs(pdb, kafka, kafka_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; } diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..2c1d1767 --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,233 @@ +use std::collections::BTreeMap; + +use indoc::formatdoc; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::ConfigMap, + role_utils::RoleGroupRef, + v2::config_file_writer::{PropertiesWriterError, to_java_properties_string}, +}; + +use crate::{ + controller::KAFKA_CONTROLLER_NAME, + crd::{ + JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, + STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, + listener::{KafkaListenerConfig, node_address_cmd}, + role::AnyConfig, + security::KafkaTlsSecurity, + v1alpha1, + }, + product_logging::extend_role_group_config_map, + utils::build_recommended_labels, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("invalid metadata manager"))] + InvalidMetadataManager { source: crate::crd::Error }, + + #[snafu(display("failed to build ConfigMap for {}", rolegroup))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display( + "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", + rolegroup + ))] + JvmSecurityProperties { + source: PropertiesWriterError, + rolegroup: String, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to serialize config for {rolegroup}"))] + SerializeConfig { + source: PropertiesWriterError, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to build properties for {rolegroup}"))] + BuildProperties { + source: crate::controller::build::properties::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] + BuildJaasConfig { rolegroup: String }, +} + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] +pub fn build_rolegroup_config_map( + kafka: &v1alpha1::KafkaCluster, + resolved_product_image: &ResolvedProductImage, + kafka_security: &KafkaTlsSecurity, + rolegroup: &RoleGroupRef, + config_file_overrides: BTreeMap, + jvm_security_overrides: BTreeMap, + merged_config: &AnyConfig, + listener_config: &KafkaListenerConfig, + pod_descriptors: &[KafkaPodDescriptor], + opa_connect_string: Option<&str>, +) -> Result { + let kafka_config_file_name = merged_config.config_file_name(); + + let metadata_manager = kafka + .effective_metadata_manager() + .context(InvalidMetadataManagerSnafu)?; + + let kafka_config = match merged_config { + AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( + kafka_security, + listener_config, + pod_descriptors, + opa_connect_string, + metadata_manager == MetadataManager::KRaft, + kafka + .spec + .cluster_config + .broker_id_pod_config_map_name + .is_some(), + config_file_overrides, + ), + AnyConfig::Controller(_) => { + crate::controller::build::properties::controller_properties::build( + kafka_security, + listener_config, + pod_descriptors, + metadata_manager == MetadataManager::KRaft, + config_file_overrides, + ) + } + } + .with_context(|_| BuildPropertiesSnafu { + rolegroup: rolegroup.clone(), + })?; + + let kafka_config = kafka_config + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>(); + + let jvm_sec_props: BTreeMap> = jvm_security_overrides + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect(); + + let mut cm_builder = ConfigMapBuilder::new(); + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(kafka) + .name(rolegroup.object_name()) + .ownerreference_from_resource(kafka, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + kafka, + KAFKA_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(), + ) + .add_data( + kafka_config_file_name, + to_java_properties_string(kafka_config.iter().map(|(k, v)| (k, v))).with_context( + |_| SerializeConfigSnafu { + rolegroup: rolegroup.clone(), + }, + )?, + ) + .add_data( + JVM_SECURITY_PROPERTIES_FILE, + to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { + JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + } + })?, + ) + .add_data( + "client.properties", + to_java_properties_string( + kafka_security + .client_properties() + .iter() + .map(|(k, v)| (k, v)), + ) + .with_context(|_| JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + })?, + ) + // This file contains the JAAS configuration for Kerberos authentication + // It has the ".properties" extension but is not a Java properties file. + // It is processed by `config-utils` to substitute "env:" and "file:" variables + // and this tool currently doesn't support the JAAS login configuration format. + .add_data( + "jaas.properties", + jaas_config_file(kafka_security.has_kerberos_enabled()), + ); + + tracing::debug!(?kafka_config, "Applied kafka config"); + tracing::debug!(?jvm_sec_props, "Applied JVM config"); + + extend_role_group_config_map( + &resolved_product_image.product_version, + rolegroup, + merged_config, + &mut cm_builder, + ); + + cm_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} + +// Generate JAAS configuration file for Kerberos authentication +// or an empty string if Kerberos is not enabled. +// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html +fn jaas_config_file(is_kerberos_enabled: bool) -> String { + match is_kerberos_enabled { + false => String::new(), + true => formatdoc! {" + bootstrap.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; + }}; + + client.KafkaServer {{ + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + isInitiator=false + keyTab=\"/stackable/kerberos/keytab\" + principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; + }}; + + ", + bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), + broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), + }, + } +} diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..b8c4c422 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,4 @@ +//! Builders that assemble Kubernetes resources for kafka rolegroups. + +pub mod config_map; +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/broker_properties.rs b/rust/operator-binary/src/controller/build/properties/broker_properties.rs new file mode 100644 index 00000000..5d43ab79 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/broker_properties.rs @@ -0,0 +1,118 @@ +use std::collections::BTreeMap; + +use snafu::OptionExt; + +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; +use crate::{ + crd::{ + KafkaPodDescriptor, + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, + KAFKA_PROCESS_ROLES, KafkaRole, + }, + security::KafkaTlsSecurity, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +pub fn build( + kafka_security: &KafkaTlsSecurity, + listener_config: &KafkaListenerConfig, + pod_descriptors: &[KafkaPodDescriptor], + opa_connect_string: Option<&str>, + kraft_mode: bool, + disable_broker_id_generation: bool, + overrides: BTreeMap, +) -> Result, Error> { + let kraft_controllers = kraft_controllers(pod_descriptors); + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/topicdata".to_string(), + ), + (KAFKA_LISTENERS.to_string(), listener_config.listeners()), + ( + KAFKA_ADVERTISED_LISTENERS.to_string(), + listener_config.advertised_listeners(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config.listener_security_protocol_map(), + ), + ( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ), + ]); + + if kraft_mode { + let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; + + // Running in KRaft mode + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ( + KAFKA_PROCESS_ROLES.to_string(), + KafkaRole::Broker.to_string(), + ), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ]); + } else { + // Running with ZooKeeper enabled + result.extend([( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + )]); + // We are in zookeeper mode and the user has defined a broker id mapping + // so we disable automatic id generation. + // This check ensures that existing clusters running in ZooKeeper mode do not + // suddenly break after the introduction of this change. + if disable_broker_id_generation { + result.extend([ + ( + "broker.id.generation.enable".to_string(), + "false".to_string(), + ), + (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), + ]); + } + } + + // Enable OPA authorization + if opa_connect_string.is_some() { + result.extend([ + ( + "authorizer.class.name".to_string(), + "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), + ), + ( + "opa.authorizer.metrics.enabled".to_string(), + "true".to_string(), + ), + ( + "opa.authorizer.url".to_string(), + opa_connect_string.unwrap_or_default().to_string(), + ), + ]); + } + + result.extend(kafka_security.broker_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + Ok(result) +} diff --git a/rust/operator-binary/src/controller/build/properties/controller_properties.rs b/rust/operator-binary/src/controller/build/properties/controller_properties.rs new file mode 100644 index 00000000..6a8172a8 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/controller_properties.rs @@ -0,0 +1,76 @@ +use std::collections::BTreeMap; + +use snafu::OptionExt; + +use super::{Error, NoKraftControllersFoundSnafu, kraft_controllers}; +use crate::{ + crd::{ + KafkaPodDescriptor, + listener::{KafkaListenerConfig, KafkaListenerName}, + role::{ + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, + KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, + }, + security::KafkaTlsSecurity, + }, + operations::graceful_shutdown::graceful_shutdown_config_properties, +}; + +pub fn build( + kafka_security: &KafkaTlsSecurity, + listener_config: &KafkaListenerConfig, + pod_descriptors: &[KafkaPodDescriptor], + kraft_mode: bool, + overrides: BTreeMap, +) -> Result, Error> { + let kraft_controllers = + kraft_controllers(pod_descriptors).context(NoKraftControllersFoundSnafu)?; + + let mut result = BTreeMap::from([ + ( + KAFKA_LOG_DIRS.to_string(), + "/stackable/data/kraft".to_string(), + ), + (KAFKA_PROCESS_ROLES.to_string(), KafkaRole::Controller.to_string()), + ( + "controller.listener.names".to_string(), + KafkaListenerName::Controller.to_string(), + ), + ( + KAFKA_NODE_ID.to_string(), + "${env:REPLICA_ID}".to_string(), + ), + ( + KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), + kraft_controllers.clone(), + ), + ( + KAFKA_LISTENERS.to_string(), + "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), + ), + ( + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), + listener_config + .listener_security_protocol_map_for_controller()), + ]); + + result.insert( + "inter.broker.listener.name".to_string(), + KafkaListenerName::Internal.to_string(), + ); + + // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. + // It is not needed once the controller is fully running in KRaft mode. + if !kraft_mode { + result.insert( + "zookeeper.connect".to_string(), + "${env:ZOOKEEPER}".to_string(), + ); + } + + result.extend(kafka_security.controller_config_settings()); + result.extend(graceful_shutdown_config_properties()); + result.extend(overrides); + + Ok(result) +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..adcdcaae --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,35 @@ +//! Property-file builders for Kafka rolegroup ConfigMaps. + +pub mod broker_properties; +pub mod controller_properties; + +use snafu::Snafu; + +use crate::crd::{KafkaPodDescriptor, role::KafkaRole}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("no Kraft controllers found to build"))] + NoKraftControllersFound, +} + +pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { + let result = pod_descriptors + .iter() + .filter(|pd| pd.role == KafkaRole::Controller.to_string()) + .map(|desc| { + format!( + "{fqdn}:{client_port}", + fqdn = desc.fqdn(), + client_port = desc.client_port + ) + }) + .collect::>() + .join(","); + + if result.is_empty() { + None + } else { + Some(result) + } +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a3e441a0..72d6fa6a 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,28 +1,23 @@ //! The validate step in the KafkaCluster controller. //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_kafka`. +//! [`ValidatedKafkaCluster`], consumed by the rest of `reconcile_kafka`. -use std::collections::HashMap; +use std::collections::BTreeMap; -use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, - }, }; use crate::{ controller::dereference::DereferencedObjects, crd::{ - self, CONTAINER_IMAGE_BASE_NAME, JVM_SECURITY_PROPERTIES_FILE, + self, CONTAINER_IMAGE_BASE_NAME, authentication::{self}, authorization::KafkaAuthorizationConfig, - role::{KafkaRole, broker::BROKER_PROPERTIES_FILE, controller::CONTROLLER_PROPERTIES_FILE}, + role::{AnyConfig, KafkaRole}, security::{self, KafkaTlsSecurity}, v1alpha1, }, @@ -44,25 +39,39 @@ pub enum Error { #[snafu(display("cluster object defines no '{role}' role"))] MissingKafkaRole { source: crd::Error, role: KafkaRole }, - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, + #[snafu(display("failed to resolve merged config for rolegroup"))] + ResolveMergedConfig { source: crate::crd::role::Error }, } type Result = std::result::Result; -/// Synchronous inputs the rest of `reconcile_kafka` needs after dereferencing. -pub struct ValidatedInputs { - pub authorization_config: Option, +/// The validated cluster. Carries everything the build steps need, resolved once +/// here so downstream code never re-derives it or touches the raw spec. +pub struct ValidatedKafkaCluster { pub image: ResolvedProductImage, pub kafka_security: KafkaTlsSecurity, - pub role_config: ValidatedRoleConfigByPropertyKind, + // DESIGN DECISION: the dereferenced authorization config is folded into the + // validated cluster (read from here downstream). The other dereferenced input, + // the authentication classes, is intentionally NOT stored: it is fully consumed + // here to build `kafka_security`. Alternative: also store the resolved auth + // classes — rejected because nothing downstream needs them beyond kafka_security. + pub authorization_config: Option, + pub role_groups: BTreeMap>, +} + +pub struct ValidatedRoleGroupConfig { + pub merged_config: AnyConfig, + // DESIGN DECISION: overrides are resolved into flat maps HERE rather than stored + // as the typed KeyValueConfigOverrides and resolved in the per-file builders (the + // hdfs-operator pattern). Reason: broker and controller use different override + // struct types (KafkaBrokerConfigOverrides vs KafkaControllerConfigOverrides), so a + // single typed field would require an enum. Resolving here keeps the build/properties + // builders taking plain `BTreeMap`. Alternative: an enum over the two + // override types threaded to builders that call resolved_overrides() — more types for + // no behavioural gain. + pub config_file_overrides: BTreeMap, + pub jvm_security_overrides: BTreeMap, + pub env_overrides: BTreeMap, } /// Validates the cluster spec and the dereferenced inputs. @@ -70,8 +79,7 @@ pub fn validate( kafka: &v1alpha1::KafkaCluster, dereferenced_objects: DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, -) -> Result { +) -> Result { let image = kafka .spec .image @@ -99,81 +107,242 @@ pub fn validate( .validate_authentication_methods() .context(FailedToValidateAuthenticationMethodSnafu)?; - let role_config = validated_product_config(kafka, &image.product_version, product_config)?; + // DESIGN DECISION: build the per-rolegroup config (merged config + resolved overrides) + // here, so reconcile reads a fully-typed ValidatedKafkaCluster instead of re-deriving + // merged_config in the loop and threading a product-config HashMap. Alternative: keep + // deriving merged_config in the reconcile loop — rejected; validation is the right place + // to prove every rolegroup resolves before any resource is built. + let mut role_groups: BTreeMap> = + BTreeMap::new(); - Ok(ValidatedInputs { - authorization_config: dereferenced_objects.authorization_config, + // Brokers always exist. + let broker_role = kafka + .broker_role() + .cloned() + .context(MissingKafkaRoleSnafu { + role: KafkaRole::Broker, + })?; + + let mut broker_groups: BTreeMap = BTreeMap::new(); + for rolegroup_name in broker_role.role_groups.keys() { + let merged_config = KafkaRole::Broker + .merged_config(kafka, rolegroup_name) + .context(ResolveMergedConfigSnafu)?; + let (config_file_overrides, jvm_security_overrides, env_overrides) = + collect_broker_role_group_overrides(kafka, &broker_role, rolegroup_name); + broker_groups.insert( + rolegroup_name.clone(), + ValidatedRoleGroupConfig { + merged_config, + config_file_overrides, + jvm_security_overrides, + env_overrides, + }, + ); + } + role_groups.insert(KafkaRole::Broker, broker_groups); + + // We need this guard because controller_role() returns an error if controllers is None, + // which would stop reconciliation for ZooKeeper-mode clusters. + if kafka.spec.controllers.is_some() { + let controller_role = kafka + .controller_role() + .cloned() + .context(MissingKafkaRoleSnafu { + role: KafkaRole::Controller, + })?; + + let mut controller_groups: BTreeMap = BTreeMap::new(); + for rolegroup_name in controller_role.role_groups.keys() { + let merged_config = KafkaRole::Controller + .merged_config(kafka, rolegroup_name) + .context(ResolveMergedConfigSnafu)?; + let (config_file_overrides, jvm_security_overrides, env_overrides) = + collect_controller_role_group_overrides(kafka, &controller_role, rolegroup_name); + controller_groups.insert( + rolegroup_name.clone(), + ValidatedRoleGroupConfig { + merged_config, + config_file_overrides, + jvm_security_overrides, + env_overrides, + }, + ); + } + role_groups.insert(KafkaRole::Controller, controller_groups); + } + + Ok(ValidatedKafkaCluster { image, kafka_security, - role_config, + authorization_config: dereferenced_objects.authorization_config, + role_groups, }) } -fn validated_product_config( +// DESIGN DECISION: role-group overrides are merged role-level first, then role-group +// extended on top so role-group wins — identical to the precedent product-config used. +// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather +// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly +// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: +// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. +fn collect_broker_role_group_overrides( kafka: &v1alpha1::KafkaCluster, - product_version: &str, - product_config: &ProductConfigManager, -) -> Result { - let mut role_config = HashMap::new(); - - let broker_role = [( - KafkaRole::Broker.to_string(), - ( - vec![ - PropertyNameKind::File(BROKER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .broker_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Broker, - })? - .erase(), - ), - )] - .into(); - - let broker_role_config = - transform_all_roles_to_config(kafka, &broker_role).context(GenerateProductConfigSnafu)?; - - role_config.extend(broker_role_config); - - // We need this because controller_role() raises an error if non-existent, - // which would stop reconciliation. - if kafka.spec.controllers.is_some() { - let controller_role = [( - KafkaRole::Controller.to_string(), - ( - vec![ - PropertyNameKind::File(CONTROLLER_PROPERTIES_FILE.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - kafka - .controller_role() - .cloned() - .context(MissingKafkaRoleSnafu { - role: KafkaRole::Controller, - })? - .erase(), - ), - )] - .into(); - - let controller_role_config = transform_all_roles_to_config(kafka, &controller_role) - .context(GenerateProductConfigSnafu)?; - - role_config.extend(controller_role_config); + broker_role: &crate::crd::BrokerRole, + rolegroup_name: &str, +) -> ( + BTreeMap, + BTreeMap, + BTreeMap, +) { + // --- broker.properties overrides --- + let role_broker_overrides: BTreeMap> = broker_role + .config + .config_overrides + .broker_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_broker_overrides: BTreeMap> = broker_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.broker_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_broker = role_broker_overrides; + merged_broker.extend(rg_broker_overrides); + let config_file_overrides: BTreeMap = merged_broker + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- security.properties overrides --- + let role_security_overrides: BTreeMap> = broker_role + .config + .config_overrides + .security_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_security_overrides: BTreeMap> = broker_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_security = role_security_overrides; + merged_security.extend(rg_security_overrides); + let jvm_security_overrides: BTreeMap = merged_security + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- env overrides --- + // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides + // (role then role-group) are extended on top, so a user override of the same key wins. + // This mirrors product-config's old merge of compute_env() output with user envOverrides. + // Alternative: inject after user overrides (operator wins) — rejected to preserve the + // previous precedence. + // + // KAFKA_CLUSTER_ID injection moved here from crd/role/broker.rs::Configuration::compute_env. + let mut env_overrides: BTreeMap = BTreeMap::new(); + if let Some(cluster_id) = kafka.cluster_id() { + env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); + } + let role_env: &std::collections::HashMap = &broker_role.config.env_overrides; + env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + if let Some(rg) = broker_role.role_groups.get(rolegroup_name) { + env_overrides.extend( + rg.config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); + } + + (config_file_overrides, jvm_security_overrides, env_overrides) +} + +// DESIGN DECISION: role-group overrides are merged role-level first, then role-group +// extended on top so role-group wins — identical to the precedent product-config used. +// We read the v2 KeyValueConfigOverrides `.overrides` map and BTreeMap::extend rather +// than using its `Merge` impl, because plain extend reproduces the old behaviour exactly +// (last-writer-wins per key) and avoids depending on Merge semantics. Alternative: +// KeyValueConfigOverrides::merge — equivalent here but an unnecessary semantic dependency. +fn collect_controller_role_group_overrides( + kafka: &v1alpha1::KafkaCluster, + controller_role: &crate::crd::ControllerRole, + rolegroup_name: &str, +) -> ( + BTreeMap, + BTreeMap, + BTreeMap, +) { + // --- controller.properties overrides --- + let role_controller_overrides: BTreeMap> = controller_role + .config + .config_overrides + .controller_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_controller_overrides: BTreeMap> = controller_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.controller_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_controller = role_controller_overrides; + merged_controller.extend(rg_controller_overrides); + let config_file_overrides: BTreeMap = merged_controller + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- security.properties overrides --- + let role_security_overrides: BTreeMap> = controller_role + .config + .config_overrides + .security_properties + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let rg_security_overrides: BTreeMap> = controller_role + .role_groups + .get(rolegroup_name) + .and_then(|rg| rg.config.config_overrides.security_properties.as_ref()) + .map(|o| o.overrides.clone()) + .unwrap_or_default(); + let mut merged_security = role_security_overrides; + merged_security.extend(rg_security_overrides); + let jvm_security_overrides: BTreeMap = merged_security + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + // --- env overrides --- + // DESIGN DECISION: KAFKA_CLUSTER_ID is injected first, then the user env overrides + // (role then role-group) are extended on top, so a user override of the same key wins. + // This mirrors product-config's old merge of compute_env() output with user envOverrides. + // Alternative: inject after user overrides (operator wins) — rejected to preserve the + // previous precedence. + // + // KAFKA_CLUSTER_ID injection moved here from crd/role/controller.rs::Configuration::compute_env. + let mut env_overrides: BTreeMap = BTreeMap::new(); + if let Some(cluster_id) = kafka.cluster_id() { + env_overrides.insert("KAFKA_CLUSTER_ID".to_string(), cluster_id.to_string()); + } + let role_env: &std::collections::HashMap = + &controller_role.config.env_overrides; + env_overrides.extend(role_env.iter().map(|(k, v)| (k.clone(), v.clone()))); + if let Some(rg) = controller_role.role_groups.get(rolegroup_name) { + env_overrides.extend( + rg.config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())), + ); } - validate_all_roles_and_groups_config( - product_version, - &role_config, - product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu) + (config_file_overrides, jvm_security_overrides, env_overrides) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d662de30..fa5b7498 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -16,13 +16,13 @@ use stackable_operator::{ cluster_operation::ClusterOperation, networking::DomainName, product_image_selection::ProductImage, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, deep_merger::ObjectOverrides, kube::{CustomResource, runtime::reflector::ObjectRef}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, utils::cluster_info::KubernetesClusterInfo, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -240,6 +240,8 @@ pub mod versioned { pub broker_id_pod_config_map_name: Option, } + // Uses the v2 KeyValueConfigOverrides (Merge-capable, `nullable` values) to match + // trino/hdfs. Resolution into flat maps happens in controller/validate.rs. #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaBrokerConfigOverrides { @@ -291,32 +293,6 @@ impl Default for v1alpha1::KafkaClusterConfig { } } -impl KeyValueOverridesProvider for v1alpha1::KafkaBrokerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::broker::BROKER_PROPERTIES_FILE => self.broker_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - -impl KeyValueOverridesProvider for v1alpha1::KafkaControllerConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - role::controller::CONTROLLER_PROPERTIES_FILE => self.controller_properties.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() - } -} - impl HasStatusCondition for v1alpha1::KafkaCluster { fn conditions(&self) -> Vec { match &self.status { diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index 70ac85d0..674b9feb 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,16 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; pub const BROKER_PROPERTIES_FILE: &str = "broker.properties"; @@ -102,38 +96,4 @@ impl BrokerConfig { } } -impl Configuration for BrokerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} +// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_broker_role_group_overrides. diff --git a/rust/operator-binary/src/crd/role/controller.rs b/rust/operator-binary/src/crd/role/controller.rs index bf1468b6..27756ee1 100644 --- a/rust/operator-binary/src/crd/role/controller.rs +++ b/rust/operator-binary/src/crd/role/controller.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::resources::{ @@ -8,16 +6,12 @@ use stackable_operator::{ }, config::{fragment::Fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, schemars::{self, JsonSchema}, }; use strum::{Display, EnumIter}; -use crate::crd::{ - role::commons::{CommonConfig, Storage, StorageFragment}, - v1alpha1, -}; +use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; pub const CONTROLLER_PROPERTIES_FILE: &str = "controller.properties"; @@ -92,38 +86,4 @@ impl ControllerConfig { } } -impl Configuration for ControllerConfigFragment { - type Configurable = v1alpha1::KafkaCluster; - - fn compute_env( - &self, - resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - if let Some(cluster_id) = resource.cluster_id() { - result.insert("KAFKA_CLUSTER_ID".to_string(), Some(cluster_id.to_string())); - } - Ok(result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } -} +// KAFKA_CLUSTER_ID injection moved to controller/validate.rs::collect_controller_role_group_overrides. diff --git a/rust/operator-binary/src/crd/role/mod.rs b/rust/operator-binary/src/crd/role/mod.rs index e08474ed..6c7bac4f 100644 --- a/rust/operator-binary/src/crd/role/mod.rs +++ b/rust/operator-binary/src/crd/role/mod.rs @@ -98,7 +98,9 @@ pub enum Error { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index c074f8af..25362c54 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -80,9 +80,9 @@ async fn main() -> anyhow::Result<()> { RunArguments { operator_environment, watch_namespace, - product_config, maintenance, common, + .. }, .. }) => { @@ -127,11 +127,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/kafka-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -188,7 +183,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background diff --git a/rust/operator-binary/src/resource/configmap.rs b/rust/operator-binary/src/resource/configmap.rs deleted file mode 100644 index 473120b3..00000000 --- a/rust/operator-binary/src/resource/configmap.rs +++ /dev/null @@ -1,421 +0,0 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; - -use indoc::formatdoc; -use product_config::{types::PropertyNameKind, writer::to_java_properties_string}; -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, - k8s_openapi::api::core::v1::ConfigMap, - role_utils::RoleGroupRef, -}; - -use crate::{ - controller::KAFKA_CONTROLLER_NAME, - crd::{ - JVM_SECURITY_PROPERTIES_FILE, KafkaPodDescriptor, MetadataManager, - STACKABLE_LISTENER_BOOTSTRAP_DIR, STACKABLE_LISTENER_BROKER_DIR, - listener::{KafkaListenerConfig, KafkaListenerName, node_address_cmd}, - role::{ - AnyConfig, KAFKA_ADVERTISED_LISTENERS, KAFKA_BROKER_ID, - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS, KAFKA_LISTENER_SECURITY_PROTOCOL_MAP, - KAFKA_LISTENERS, KAFKA_LOG_DIRS, KAFKA_NODE_ID, KAFKA_PROCESS_ROLES, KafkaRole, - }, - security::KafkaTlsSecurity, - v1alpha1, - }, - operations::graceful_shutdown::graceful_shutdown_config_properties, - product_logging::extend_role_group_config_map, - utils::build_recommended_labels, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("invalid metadata manager"))] - InvalidMetadataManager { source: crate::crd::Error }, - - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, - - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - JvmSecurityProperties { - source: product_config::writer::PropertiesWriterError, - rolegroup: String, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to serialize config for {rolegroup}"))] - SerializeConfig { - source: product_config::writer::PropertiesWriterError, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("no Kraft controllers found to build"))] - NoKraftControllersFound, - - #[snafu(display("unknown Kafka role [{name}]"))] - UnknownKafkaRole { - source: strum::ParseError, - name: String, - }, - - #[snafu(display("failed to build jaas configuration file for {rolegroup}"))] - BuildJaasConfig { rolegroup: String }, -} - -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -pub fn build_rolegroup_config_map( - kafka: &v1alpha1::KafkaCluster, - resolved_product_image: &ResolvedProductImage, - kafka_security: &KafkaTlsSecurity, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_config: &AnyConfig, - listener_config: &KafkaListenerConfig, - pod_descriptors: &[KafkaPodDescriptor], - opa_connect_string: Option<&str>, -) -> Result { - let kafka_config_file_name = merged_config.config_file_name(); - - let metadata_manager = kafka - .effective_metadata_manager() - .context(InvalidMetadataManagerSnafu)?; - - let mut kafka_config = server_properties_file( - metadata_manager == MetadataManager::KRaft, - &rolegroup.role, - pod_descriptors, - listener_config, - opa_connect_string, - kafka - .spec - .cluster_config - .broker_id_pod_config_map_name - .is_some(), - )?; - - match merged_config { - AnyConfig::Broker(_) => kafka_config.extend(kafka_security.broker_config_settings()), - AnyConfig::Controller(_) => { - kafka_config.extend(kafka_security.controller_config_settings()) - } - } - - kafka_config.extend(graceful_shutdown_config_properties()); - - // Need to call this to get configOverrides :( - kafka_config.extend( - rolegroup_config - .get(&PropertyNameKind::File(kafka_config_file_name.to_string())) - .cloned() - .unwrap_or_default(), - ); - - let kafka_config = kafka_config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>(); - - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - - let mut cm_builder = ConfigMapBuilder::new(); - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(kafka) - .name(rolegroup.object_name()) - .ownerreference_from_resource(kafka, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - kafka, - KAFKA_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - kafka_config_file_name, - to_java_properties_string(kafka_config.iter().map(|(k, v)| (k, v))).with_context( - |_| SerializeConfigSnafu { - rolegroup: rolegroup.clone(), - }, - )?, - ) - .add_data( - JVM_SECURITY_PROPERTIES_FILE, - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - } - })?, - ) - .add_data( - "client.properties", - to_java_properties_string( - kafka_security - .client_properties() - .iter() - .map(|(k, v)| (k, v)), - ) - .with_context(|_| JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - })?, - ) - // This file contains the JAAS configuration for Kerberos authentication - // It has the ".properties" extension but is not a Java properties file. - // It is processed by `config-utils` to substitute "env:" and "file:" variables - // and this tool currently doesn't support the JAAS login configuration format. - .add_data( - "jaas.properties", - jaas_config_file(kafka_security.has_kerberos_enabled()), - ); - - tracing::debug!(?kafka_config, "Applied kafka config"); - tracing::debug!(?jvm_sec_props, "Applied JVM config"); - - extend_role_group_config_map( - &resolved_product_image.product_version, - rolegroup, - merged_config, - &mut cm_builder, - ); - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -// Generate the content of both broker.properties and controller.properties files. -fn server_properties_file( - kraft_mode: bool, - role: &str, - pod_descriptors: &[KafkaPodDescriptor], - listener_config: &KafkaListenerConfig, - opa_connect_string: Option<&str>, - disable_broker_id_generation: bool, -) -> Result, Error> { - let kraft_controllers = kraft_controllers(pod_descriptors); - - let role = KafkaRole::from_str(role).context(UnknownKafkaRoleSnafu { - name: role.to_string(), - })?; - - match role { - KafkaRole::Controller => { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/kraft".to_string(), - ), - (KAFKA_PROCESS_ROLES.to_string(), role.to_string()), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_NODE_ID.to_string(), - "${env:REPLICA_ID}".to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ( - KAFKA_LISTENERS.to_string(), - "CONTROLLER://${env:POD_NAME}.${env:ROLEGROUP_HEADLESS_SERVICE_NAME}.${env:NAMESPACE}.svc.${env:CLUSTER_DOMAIN}:${env:KAFKA_CLIENT_PORT}".to_string(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config - .listener_security_protocol_map_for_controller()), - ]); - - result.insert( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ); - - // The ZooKeeper connection is needed for migration from ZooKeeper to KRaft mode. - // It is not needed once the controller is fully running in KRaft mode. - if !kraft_mode { - result.insert( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - ); - } - Ok(result) - } - KafkaRole::Broker => { - let mut result = BTreeMap::from([ - ( - KAFKA_LOG_DIRS.to_string(), - "/stackable/data/topicdata".to_string(), - ), - (KAFKA_LISTENERS.to_string(), listener_config.listeners()), - ( - KAFKA_ADVERTISED_LISTENERS.to_string(), - listener_config.advertised_listeners(), - ), - ( - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP.to_string(), - listener_config.listener_security_protocol_map(), - ), - ( - "inter.broker.listener.name".to_string(), - KafkaListenerName::Internal.to_string(), - ), - ]); - - if kraft_mode { - let kraft_controllers = kraft_controllers.context(NoKraftControllersFoundSnafu)?; - - // Running in KRaft mode - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_NODE_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ( - KAFKA_PROCESS_ROLES.to_string(), - KafkaRole::Broker.to_string(), - ), - ( - "controller.listener.names".to_string(), - KafkaListenerName::Controller.to_string(), - ), - ( - KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS.to_string(), - kraft_controllers.clone(), - ), - ]); - } else { - // Running with ZooKeeper enabled - result.extend([( - "zookeeper.connect".to_string(), - "${env:ZOOKEEPER}".to_string(), - )]); - // We are in zookeeper mode and the user has defined a broker id mapping - // so we disable automatic id generation. - // This check ensures that existing clusters running in ZooKeeper mode do not - // suddenly break after the introduction of this change. - if disable_broker_id_generation { - result.extend([ - ( - "broker.id.generation.enable".to_string(), - "false".to_string(), - ), - (KAFKA_BROKER_ID.to_string(), "${env:REPLICA_ID}".to_string()), - ]); - } - } - - // Enable OPA authorization - if opa_connect_string.is_some() { - result.extend([ - ( - "authorizer.class.name".to_string(), - "org.openpolicyagent.kafka.OpaAuthorizer".to_string(), - ), - ( - "opa.authorizer.metrics.enabled".to_string(), - "true".to_string(), - ), - ( - "opa.authorizer.url".to_string(), - opa_connect_string.unwrap_or_default().to_string(), - ), - ]); - } - - Ok(result) - } - } -} - -fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Option { - let result = pod_descriptors - .iter() - .filter(|pd| pd.role == KafkaRole::Controller.to_string()) - .map(|desc| { - format!( - "{fqdn}:{client_port}", - fqdn = desc.fqdn(), - client_port = desc.client_port - ) - }) - .collect::>() - .join(","); - - if result.is_empty() { - None - } else { - Some(result) - } -} - -// Generate JAAS configuration file for Kerberos authentication -// or an empty string if Kerberos is not enabled. -// See https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/LoginConfigFile.html -fn jaas_config_file(is_kerberos_enabled: bool) -> String { - match is_kerberos_enabled { - false => String::new(), - true => formatdoc! {" - bootstrap.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{bootstrap_address}@${{env:KERBEROS_REALM}}\"; - }}; - - client.KafkaServer {{ - com.sun.security.auth.module.Krb5LoginModule required - useKeyTab=true - storeKey=true - isInitiator=false - keyTab=\"/stackable/kerberos/keytab\" - principal=\"kafka/{broker_address}@${{env:KERBEROS_REALM}}\"; - }}; - - ", - bootstrap_address = node_address_cmd(STACKABLE_LISTENER_BOOTSTRAP_DIR), - broker_address = node_address_cmd(STACKABLE_LISTENER_BROKER_DIR), - }, - } -} diff --git a/rust/operator-binary/src/resource/mod.rs b/rust/operator-binary/src/resource/mod.rs index a79483f8..514d0adb 100644 --- a/rust/operator-binary/src/resource/mod.rs +++ b/rust/operator-binary/src/resource/mod.rs @@ -1,4 +1,3 @@ -pub mod configmap; pub mod listener; pub mod service; pub mod statefulset; diff --git a/rust/operator-binary/src/resource/statefulset.rs b/rust/operator-binary/src/resource/statefulset.rs index 5cb262f0..7f232793 100644 --- a/rust/operator-binary/src/resource/statefulset.rs +++ b/rust/operator-binary/src/resource/statefulset.rs @@ -1,9 +1,5 @@ -use std::{ - collections::{BTreeMap, HashMap}, - ops::Deref, -}; +use std::{collections::BTreeMap, ops::Deref}; -use product_config::types::PropertyNameKind; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -171,7 +167,7 @@ pub fn build_broker_rolegroup_statefulset( kafka_role: &KafkaRole, resolved_product_image: &ResolvedProductImage, rolegroup_ref: &RoleGroupRef, - broker_config: &HashMap>, + env_overrides: &BTreeMap, kafka_security: &KafkaTlsSecurity, merged_config: &AnyConfig, service_account: &ServiceAccount, @@ -249,10 +245,8 @@ pub fn build_broker_rolegroup_statefulset( .context(AddKerberosConfigSnafu)?; } - let mut env = broker_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() + let mut env = env_overrides + .iter() .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()), @@ -581,7 +575,7 @@ pub fn build_controller_rolegroup_statefulset( kafka_role: &KafkaRole, resolved_product_image: &ResolvedProductImage, rolegroup_ref: &RoleGroupRef, - controller_config: &HashMap>, + env_overrides: &BTreeMap, kafka_security: &KafkaTlsSecurity, merged_config: &AnyConfig, service_account: &ServiceAccount, @@ -603,10 +597,8 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let mut env = controller_config - .get(&PropertyNameKind::Env) - .into_iter() - .flatten() + let mut env = env_overrides + .iter() .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()),