diff --git a/CHANGELOG.md b/CHANGELOG.md index d4c6baec..eec63e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,14 @@ All notable changes to this project will be documented in this file. ### Changed -- Updated stackable image versions ([#271]) -- `operator-rs` `0.25.2` → `0.30.1` ([#274], [#283]). +- Updated stackable image versions ([#271]). +- `operator-rs` `0.25.2` → `0.32.1` ([#274], [#283], [#292], [#298]). - Consolidated security context user, group and fs group ([#277]). - [BREAKING] Use Product image selection instead of version. `spec.version` has been replaced by `spec.image` ([#280]). - Fix role group node selector ([#283]). - [BREAKING] Moved `database` specification from role / role-group level to top-level `clusterConfig` ([#292]). - [BREAKING] Moved `s3`, `serviceType` and `hdfs` discovery to top-level `clusterConfig` ([#292]). -- `operator-rs` `0.30.1` -> `0.30.2` ([#292]) +- Enable logging ([#298]). [#271]: https://github.com/stackabletech/hive-operator/pull/271 [#274]: https://github.com/stackabletech/hive-operator/pull/274 @@ -21,18 +21,19 @@ All notable changes to this project will be documented in this file. [#280]: https://github.com/stackabletech/hive-operator/pull/280 [#283]: https://github.com/stackabletech/hive-operator/pull/283 [#292]: https://github.com/stackabletech/hive-operator/pull/292 +[#298]: https://github.com/stackabletech/hive-operator/pull/298 ## [0.8.0] - 2022-11-07 ### Added - PVCs for data storage, cpu and memory limits are now configurable ([#242]). -- Orphaned resources are deleted ([#254]) -- Support HDFS connections ([#264]) +- Orphaned resources are deleted ([#254]). +- Support HDFS connections ([#264]). ### Changed -- `operator-rs` `0.22.0` -> `0.25.2` ([#254]) +- `operator-rs` `0.22.0` -> `0.25.2` ([#254]). [#242]: https://github.com/stackabletech/hive-operator/pull/242 [#254]: https://github.com/stackabletech/hive-operator/pull/254 diff --git a/Cargo.lock b/Cargo.lock index ab2e0e59..ad0d7d9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" [[package]] name = "async-trait" -version = "0.1.60" +version = "0.1.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" +checksum = "689894c2db1ea643a50834b999abf1c110887402542955ff5451dab8f861f9ed" dependencies = [ "proc-macro2", "quote", @@ -72,6 +72,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "bit-set" version = "0.5.3" @@ -106,9 +112,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -164,9 +170,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.30" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656ad1e55e23d287773f7d8192c300dc715c3eeded93b3da651d11c42cfd74d2" +checksum = "4ec7a4128863c188deefe750ac1d1dfe66c236909f845af04beed823638dc1b2" dependencies = [ "bitflags", "clap_derive", @@ -179,9 +185,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.21" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", "proc-macro-error", @@ -192,9 +198,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] @@ -256,9 +262,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e" dependencies = [ "cc", "cxxbridge-flags", @@ -268,9 +274,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200" dependencies = [ "cc", "codespan-reporting", @@ -283,15 +289,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e" dependencies = [ "proc-macro2", "quote", @@ -828,9 +834,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", "windows-sys", @@ -885,12 +891,13 @@ dependencies = [ [[package]] name = "json-patch" -version = "0.2.7" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce" +checksum = "e712e62827c382a77b87f590532febb1f8b2fdbc3eefa1ee37fe7281687075ef" dependencies = [ "serde", "serde_json", + "thiserror", "treediff", ] @@ -907,11 +914,11 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9455388f4977de4d0934efa9f7d36296295537d774574113a20f6082de03da" +checksum = "3d1985030683a2bac402cbda61222195de80d3f66b4c87ab56e5fea379bd98c3" dependencies = [ - "base64", + "base64 0.20.0", "bytes", "chrono", "schemars", @@ -922,9 +929,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf241a3a42bca4a2d1c21f2f34a659655032a7858270c7791ad4433aa8d79cb" +checksum = "53ee2ba94546e32a5aef943e5831c6ac25592ff8dcfa8b2a06e0aaea90c69188" dependencies = [ "k8s-openapi", "kube-client", @@ -935,11 +942,11 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e442b4e6d55c4b3d0c0c70d79a8865bf17e2c33725f9404bfcb8a29ee002ffe" +checksum = "7c9ca1f597bd48ed26f45f601bf2fa3aaa0933b8d1652d883b8444519b72af4a" dependencies = [ - "base64", + "base64 0.20.0", "bytes", "chrono", "dirs-next", @@ -970,9 +977,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca2e1b1528287ba61602bbd17d0aa717fbb4d0fb257f4fa3a5fa884116ef778" +checksum = "61f2c6d1a2d1584859499eb05a41c5a44713818041621fa7515cfdbdf4769ea7" dependencies = [ "chrono", "form_urlencoded", @@ -988,9 +995,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1af50996adb7e1251960d278859772fa30df99879dc154d792e36832209637cb" +checksum = "3e1dfe288fd87029f87c5713ddf585a4221e1b5be8f8c7c02ba28f5211f2a6d7" dependencies = [ "darling", "proc-macro2", @@ -1001,9 +1008,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.76.0" +version = "0.78.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9b312c38884a3f41d67e2f7580824b6f45d360b98497325b5630664b3a359d" +checksum = "7995aaf15656bf3694cd455578d59119acc3ca55f71e9a1b579a34a47d17ecce" dependencies = [ "ahash", "backoff", @@ -1031,15 +1038,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libgit2-sys" -version = "0.14.0+1.5.0" +version = "0.14.1+1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47a00859c70c8a4f7218e6d1cc32875c4b55f6799445b842b0d8ed5e4c3d959b" +checksum = "4a07fb2692bc3593bda59de45a502bb3071659f2c515e28c71e728306b038e17" dependencies = [ "cc", "libc", @@ -1176,9 +1183,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "openssl" @@ -1335,9 +1342,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" dependencies = [ "cfg-if", "libc", @@ -1348,11 +1355,11 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" dependencies = [ - "base64", + "base64 0.13.1", ] [[package]] @@ -1431,9 +1438,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" dependencies = [ "unicode-ident", ] @@ -1515,9 +1522,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1541,9 +1548,9 @@ checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ "bitflags", "errno", @@ -1622,9 +1629,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] @@ -1641,9 +1648,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -1687,9 +1694,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.16" +version = "0.9.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92b5b431e8907b50339b51223b97d102db8d987ced36f6e4d03621db9316c834" +checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567" dependencies = [ "indexmap", "itoa", @@ -1799,8 +1806,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.30.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.30.2#133db3918fb3af191a4203106a5056d77cc9579f" +version = "0.32.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.32.1#832fa2abd5599cd71548fa1db6038d7b407fd046" dependencies = [ "chrono", "clap", @@ -1820,7 +1827,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "serde_yaml 0.9.16", + "serde_yaml 0.9.17", "snafu", "stackable-operator-derive", "strum", @@ -1833,8 +1840,8 @@ dependencies = [ [[package]] name = "stackable-operator-derive" -version = "0.30.2" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.30.2#133db3918fb3af191a4203106a5056d77cc9579f" +version = "0.32.1" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.32.1#832fa2abd5599cd71548fa1db6038d7b407fd046" dependencies = [ "darling", "proc-macro2", @@ -1883,9 +1890,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -1958,9 +1965,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.1" +version = "1.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" +checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" dependencies = [ "autocfg", "bytes", @@ -2037,9 +2044,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ "serde", ] @@ -2067,7 +2074,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ - "base64", + "base64 0.13.1", "bitflags", "bytes", "futures-core", @@ -2172,24 +2179,24 @@ dependencies = [ [[package]] name = "treediff" -version = "3.0.2" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff" +checksum = "52984d277bdf2a751072b5df30ec0377febdb02f7696d64c2d7d54630bac4303" dependencies = [ "serde_json", ] [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "0046be40136ef78dc325e0edefccf84ccddacd0afcc1ca54103fa3c61bbdab1d" [[package]] name = "unicode-ident" @@ -2371,45 +2378,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "xml-rs" diff --git a/deploy/external/log4j.properties b/deploy/external/log4j.properties deleted file mode 100644 index 48160ac6..00000000 --- a/deploy/external/log4j.properties +++ /dev/null @@ -1,341 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Define some default values that can be overridden by system properties -hadoop.root.logger=INFO,console -hadoop.log.dir=. -hadoop.log.file=hadoop.log - -# Define the root logger to the system property "hadoop.root.logger". -log4j.rootLogger=${hadoop.root.logger}, EventCounter - -# Logging Threshold -log4j.threshold=ALL - -# Null Appender -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# Rolling File Appender - cap space usage at 5gb. -# -hadoop.log.maxfilesize=256MB -hadoop.log.maxbackupindex=20 -log4j.appender.RFA=org.apache.log4j.RollingFileAppender -log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} - -log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} -log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} - -log4j.appender.RFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# Daily Rolling File Appender -# - -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} - -# Rollover at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - - -# -# console -# Add "console" to rootlogger above if you want to use this -# - -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n - -# -# TaskLog Appender -# -log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender - -log4j.appender.TLA.layout=org.apache.log4j.PatternLayout -log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# -# HDFS block state change log from block manager -# -# Uncomment the following to log normal block state change -# messages from BlockManager in NameNode. -#log4j.logger.BlockStateChange=DEBUG - -# -#Security appender -# -hadoop.security.logger=INFO,NullAppender -hadoop.security.log.maxfilesize=256MB -hadoop.security.log.maxbackupindex=20 -log4j.category.SecurityLogger=${hadoop.security.logger} -hadoop.security.log.file=SecurityAuth-${user.name}.audit -log4j.appender.RFAS=org.apache.log4j.RollingFileAppender -log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} -log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} - -# -# Daily Rolling Security appender -# -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd - -# -# hadoop configuration logging -# - -# Uncomment the following line to turn off configuration deprecation warnings. -# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN - -# -# hive audit logging -# -hive.audit.logger=INFO,NullAppender -hive.audit.log.maxfilesize=256MB -hive.audit.log.maxbackupindex=20 -log4j.logger.org.apache.hadoop.hive.server.namenode.FSNamesystem.audit=${hive.audit.logger} -log4j.additivity.org.apache.hadoop.hive.server.namenode.FSNamesystem.audit=false -log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hive-audit.log -log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.RFAAUDIT.MaxFileSize=${hive.audit.log.maxfilesize} -log4j.appender.RFAAUDIT.MaxBackupIndex=${hive.audit.log.maxbackupindex} - -# -# NameNode metrics logging. -# The default is to retain two namenode-metrics.log files up to 64MB each. -# -namenode.metrics.logger=INFO,NullAppender -log4j.logger.NameNodeMetricsLog=${namenode.metrics.logger} -log4j.additivity.NameNodeMetricsLog=false -log4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender -log4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log -log4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout -log4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n -log4j.appender.NNMETRICSRFA.MaxBackupIndex=1 -log4j.appender.NNMETRICSRFA.MaxFileSize=64MB - -# -# DataNode metrics logging. -# The default is to retain two datanode-metrics.log files up to 64MB each. -# -datanode.metrics.logger=INFO,NullAppender -log4j.logger.DataNodeMetricsLog=${datanode.metrics.logger} -log4j.additivity.DataNodeMetricsLog=false -log4j.appender.DNMETRICSRFA=org.apache.log4j.RollingFileAppender -log4j.appender.DNMETRICSRFA.File=${hadoop.log.dir}/datanode-metrics.log -log4j.appender.DNMETRICSRFA.layout=org.apache.log4j.PatternLayout -log4j.appender.DNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n -log4j.appender.DNMETRICSRFA.MaxBackupIndex=1 -log4j.appender.DNMETRICSRFA.MaxFileSize=64MB - -# Custom Logging levels - -#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG -#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG -#log4j.logger.org.apache.hadoop.hive.server.namenode.FSNamesystem.audit=DEBUG - - -# AWS SDK & S3A FileSystem -#log4j.logger.com.amazonaws=ERROR -log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR -#log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN - -# -# Event Counter Appender -# Sends counts of logging messages at different severity levels to Hadoop Metrics. -# -log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter - -# -# Job Summary Appender -# -# Use following logger to send summary to separate file defined by -# hadoop.mapreduce.jobsummary.log.file : -# hadoop.mapreduce.jobsummary.logger=INFO,JSA -# -hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} -hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log -hadoop.mapreduce.jobsummary.log.maxfilesize=256MB -hadoop.mapreduce.jobsummary.log.maxbackupindex=20 -log4j.appender.JSA=org.apache.log4j.RollingFileAppender -log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} -log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} -log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} -log4j.appender.JSA.layout=org.apache.log4j.PatternLayout -log4j.appender.JSA.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} -log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false - -# -# shuffle connection log from shuffleHandler -# Uncomment the following line to enable logging of shuffle connections -# log4j.logger.org.apache.hadoop.mapred.ShuffleHandler.audit=DEBUG - -# -# Yarn ResourceManager Application Summary Log -# -# Set the ResourceManager summary log filename -yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log -# Set the ResourceManager summary log level and appender -yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} -#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY - -# To enable AppSummaryLogging for the RM, -# set yarn.server.resourcemanager.appsummary.logger to -# ,RMSUMMARY in hadoop-env.sh - -# Appender for ResourceManager Application Summary Log -# Requires the following properties to be set -# - hadoop.log.dir (Hadoop Log directory) -# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) -# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) - -log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} -log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false -log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender -log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} -log4j.appender.RMSUMMARY.MaxFileSize=256MB -log4j.appender.RMSUMMARY.MaxBackupIndex=20 -log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout -log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n - -# -# YARN ResourceManager audit logging -# -rm.audit.logger=INFO,NullAppender -rm.audit.log.maxfilesize=256MB -rm.audit.log.maxbackupindex=20 -log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger} -log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false -log4j.appender.RMAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.RMAUDIT.File=${hadoop.log.dir}/rm-audit.log -log4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -log4j.appender.RMAUDIT.MaxFileSize=${rm.audit.log.maxfilesize} -log4j.appender.RMAUDIT.MaxBackupIndex=${rm.audit.log.maxbackupindex} - -# -# YARN NodeManager audit logging -# -nm.audit.logger=INFO,NullAppender -nm.audit.log.maxfilesize=256MB -nm.audit.log.maxbackupindex=20 -log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger} -log4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false -log4j.appender.NMAUDIT=org.apache.log4j.RollingFileAppender -log4j.appender.NMAUDIT.File=${hadoop.log.dir}/nm-audit.log -log4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout -log4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601}%p %c{2}: %m%n -log4j.appender.NMAUDIT.MaxFileSize=${nm.audit.log.maxfilesize} -log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex} - -# HS audit log configs -#mapreduce.hs.audit.logger=INFO,HSAUDIT -#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} -#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false -#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender -#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log -#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout -#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n -#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd - -# Http Server Request Logs -#log4j.logger.http.requests.namenode=INFO,namenoderequestlog -#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log -#log4j.appender.namenoderequestlog.RetainDays=3 - -#log4j.logger.http.requests.datanode=INFO,datanoderequestlog -#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log -#log4j.appender.datanoderequestlog.RetainDays=3 - -#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog -#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log -#log4j.appender.resourcemanagerrequestlog.RetainDays=3 - -#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog -#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log -#log4j.appender.jobhistoryrequestlog.RetainDays=3 - -#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog -#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender -#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log -#log4j.appender.nodemanagerrequestlog.RetainDays=3 - - -# WebHive request log on datanodes -# Specify -Ddatanode.webhive.logger=INFO,HTTPDRFA on datanode startup to -# direct the log to a separate file. -#datanode.webhive.logger=INFO,console -#log4j.logger.datanode.webhive=${datanode.webhive.logger} -#log4j.appender.HTTPDRFA=org.apache.log4j.DailyRollingFileAppender -#log4j.appender.HTTPDRFA.File=${hadoop.log.dir}/hadoop-datanode-webhive.log -#log4j.appender.HTTPDRFA.layout=org.apache.log4j.PatternLayout -#log4j.appender.HTTPDRFA.layout.ConversionPattern=%d{ISO8601} %m%n -#log4j.appender.HTTPDRFA.DatePattern=.yyyy-MM-dd - - -# Appender for viewing information for errors and warnings -yarn.ewma.cleanupInterval=300 -yarn.ewma.messageAgeLimitSeconds=86400 -yarn.ewma.maxUniqueMessages=250 -log4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender -log4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval} -log4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds} -log4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages} - -# -# Fair scheduler state dump -# -# Use following logger to dump the state to a separate file - -#log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=DEBUG,FSSTATEDUMP -#log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler.statedump=false -#log4j.appender.FSSTATEDUMP=org.apache.log4j.RollingFileAppender -#log4j.appender.FSSTATEDUMP.File=${hadoop.log.dir}/fairscheduler-statedump.log -#log4j.appender.FSSTATEDUMP.layout=org.apache.log4j.PatternLayout -#log4j.appender.FSSTATEDUMP.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -#log4j.appender.FSSTATEDUMP.MaxFileSize=${hadoop.log.maxfilesize} -#log4j.appender.FSSTATEDUMP.MaxBackupIndex=${hadoop.log.maxbackupindex} - -# Log levels of third-party libraries -log4j.logger.org.apache.commons.beanutils=WARN diff --git a/deploy/helm/hive-operator/crds/crds.yaml b/deploy/helm/hive-operator/crds/crds.yaml index 98d30e9d..ebe6d27a 100644 --- a/deploy/helm/hive-operator/crds/crds.yaml +++ b/deploy/helm/hive-operator/crds/crds.yaml @@ -166,6 +166,10 @@ spec: - ClusterIP nullable: true type: string + vectorAggregatorConfigMapName: + description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true + type: string required: - database type: object @@ -223,8 +227,93 @@ spec: config: default: {} properties: + logging: + default: + enableVectorAgent: null + containers: {} + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Fragment derived from `ContainerLogConfigChoice` + properties: + console: + nullable: true + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + nullable: true + type: string + type: object + file: + nullable: true + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + type: object + type: object + type: object + enableVectorAgent: + nullable: true + type: boolean + type: object resources: - nullable: true + default: + memory: + limit: null + runtimeLimits: {} + cpu: + min: null + max: null + storage: + data: + capacity: null properties: cpu: default: @@ -232,111 +321,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -349,38 +345,7 @@ spec: capacity: null properties: capacity: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string selectors: @@ -421,7 +386,6 @@ spec: type: object type: object warehouseDir: - description: The location of default database for the Hive warehouse. Maps to the `hive.metastore.warehouse.dir` setting. nullable: true type: string type: object @@ -448,8 +412,93 @@ spec: config: default: {} properties: + logging: + default: + enableVectorAgent: null + containers: {} + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Fragment derived from `ContainerLogConfigChoice` + properties: + console: + nullable: true + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + nullable: true + type: string + type: object + file: + nullable: true + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + properties: + level: + description: Log levels + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + type: object + type: object + type: object + enableVectorAgent: + nullable: true + type: boolean + type: object resources: - nullable: true + default: + memory: + limit: null + runtimeLimits: {} + cpu: + min: null + max: null + storage: + data: + capacity: null properties: cpu: default: @@ -457,111 +506,18 @@ spec: max: null properties: max: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string min: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string type: object memory: properties: limit: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string runtimeLimits: @@ -574,38 +530,7 @@ spec: capacity: null properties: capacity: - description: |- - Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors. - - The serialization format is: - - ::= - (Note that may be empty, from the "" case in .) - ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= "+" | "-" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei - (International System of units; See: http://physics.nist.gov/cuu/Units/binary.html) - ::= m | "" | k | M | G | T | P | E - (Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.) - ::= "e" | "E" - - No matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities. - - When a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized. - - Before serializing, Quantity will be put in "canonical form". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that: - a. No precision is lost - b. No fractional digits will be emitted - c. The exponent (or suffix) is as large as possible. - The sign will be omitted unless the number is negative. - - Examples: - 1.5 will be serialized as "1500m" - 1.5Gi will be serialized as "1536Mi" - - Note that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise. - - Non-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.) - - This format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation. + description: "Quantity is a fixed-point representation of a number. It provides convenient marshaling/unmarshaling in JSON and YAML, in addition to String() and AsInt64() accessors.\n\nThe serialization format is:\n\n``` ::= \n\n\t(Note that may be empty, from the \"\" case in .)\n\n ::= 0 | 1 | ... | 9 ::= | ::= | . | . | . ::= \"+\" | \"-\" ::= | ::= | | ::= Ki | Mi | Gi | Ti | Pi | Ei\n\n\t(International System of units; See: http://physics.nist.gov/cuu/Units/binary.html)\n\n ::= m | \"\" | k | M | G | T | P | E\n\n\t(Note that 1024 = 1Ki but 1000 = 1k; I didn't choose the capitalization.)\n\n ::= \"e\" | \"E\" ```\n\nNo matter which of the three exponent forms is used, no quantity may represent a number greater than 2^63-1 in magnitude, nor may it have more than 3 decimal places. Numbers larger or more precise will be capped or rounded up. (E.g.: 0.1m will rounded up to 1m.) This may be extended in the future if we require larger or smaller quantities.\n\nWhen a Quantity is parsed from a string, it will remember the type of suffix it had, and will use the same type again when it is serialized.\n\nBefore serializing, Quantity will be put in \"canonical form\". This means that Exponent/suffix will be adjusted up or down (with a corresponding increase or decrease in Mantissa) such that:\n\n- No precision is lost - No fractional digits will be emitted - The exponent (or suffix) is as large as possible.\n\nThe sign will be omitted unless the number is negative.\n\nExamples:\n\n- 1.5 will be serialized as \"1500m\" - 1.5Gi will be serialized as \"1536Mi\"\n\nNote that the quantity will NEVER be internally represented by a floating point number. That is the whole point of this exercise.\n\nNon-canonical values will still parse as long as they are well formed, but will be re-emitted in their canonical form. (So always use canonical form, or don't diff.)\n\nThis format is intended to make it difficult to use these numbers without writing some sort of special handling code in the hopes that that will cause implementors to also use a fixed point implementation." nullable: true type: string selectors: @@ -646,7 +571,6 @@ spec: type: object type: object warehouseDir: - description: The location of default database for the Hive warehouse. Maps to the `hive.metastore.warehouse.dir` setting. nullable: true type: string type: object diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 88f099ad..1445350f 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -42,6 +42,25 @@ clusterConfig: The managed Hive instances are automatically configured to export Prometheus metrics. See xref:home:operators:monitoring.adoc[] for more details. +== Log aggregation + +The logs can be forwarded to a Vector log aggregator by providing a discovery +ConfigMap for the aggregator and by enabling the log agent: + +[source,yaml] +---- +spec: + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery + metastore: + config: + logging: + enableVectorAgent: true +---- + +Further information on how to configure logging, can be found in +xref:home:concepts:logging.adoc[]. + == Configuration & Environment Overrides The cluster definition also supports overriding configuration properties and environment variables, either per role or per role group, where the more specific override (role group) has precedence over the less specific one (role). @@ -186,7 +205,7 @@ metadata: spec: image: productVersion: 3.1.3 - stackableVersion: 0.2.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/metastore_db;create=true @@ -229,7 +248,7 @@ metadata: spec: image: productVersion: 3.1.3 - stackableVersion: 0.2.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:derby:;databaseName=/stackable/metastore_db;create=true @@ -299,7 +318,7 @@ metadata: spec: image: productVersion: 3.1.3 - stackableVersion: 0.2.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:postgresql://hive-postgresql.default.svc.cluster.local:5432/hive diff --git a/docs/modules/getting_started/examples/code/hive-postgres-s3.yaml b/docs/modules/getting_started/examples/code/hive-postgres-s3.yaml index 2035d1be..37ccc4f3 100644 --- a/docs/modules/getting_started/examples/code/hive-postgres-s3.yaml +++ b/docs/modules/getting_started/examples/code/hive-postgres-s3.yaml @@ -6,7 +6,7 @@ metadata: spec: image: productVersion: 3.1.3 - stackableVersion: 0.2.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:postgresql://postgresql:5432/hive diff --git a/examples/simple-hive-cluster-postgres-s3.yaml b/examples/simple-hive-cluster-postgres-s3.yaml index 46644092..7e3c45a0 100644 --- a/examples/simple-hive-cluster-postgres-s3.yaml +++ b/examples/simple-hive-cluster-postgres-s3.yaml @@ -18,7 +18,7 @@ metadata: spec: image: productVersion: 3.1.3 - stackableVersion: 0.2.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true diff --git a/examples/simple-hive-cluster.yaml b/examples/simple-hive-cluster.yaml index 9006e09b..2f22a70b 100644 --- a/examples/simple-hive-cluster.yaml +++ b/examples/simple-hive-cluster.yaml @@ -6,7 +6,7 @@ metadata: spec: image: productVersion: 2.3.9 - stackableVersion: 0.6.0 + stackableVersion: 0.prerelease clusterConfig: database: connString: jdbc:derby:;databaseName=/tmp/hive;create=true diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 9acc7f26..9b8cb304 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -13,6 +13,6 @@ indoc = "1.0.8" serde = "1.0" serde_json = "1.0" snafu = "0.7" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.32.1" } strum = { version = "0.24", features = ["derive"] } tracing = "0.1" diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index b13924dc..0c509460 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -14,26 +14,33 @@ use stackable_operator::{ k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::{runtime::reflector::ObjectRef, CustomResource}, product_config_utils::{ConfigError, Configuration}, + product_logging::{self, spec::Logging}, role_utils::{Role, RoleGroupRef}, schemars::{self, JsonSchema}, }; use std::collections::BTreeMap; -use strum::{Display, EnumString}; +use strum::{Display, EnumIter, EnumString}; pub const APP_NAME: &str = "hive"; - +// directories pub const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; -pub const STACKABLE_RW_CONFIG_DIR: &str = "/stackable/rwconfig"; +pub const STACKABLE_CONFIG_DIR_NAME: &str = "config"; +pub const STACKABLE_CONFIG_MOUNT_DIR: &str = "/stackable/mount/config"; +pub const STACKABLE_CONFIG_MOUNT_DIR_NAME: &str = "config-mount"; +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +pub const STACKABLE_LOG_DIR_NAME: &str = "log"; +pub const STACKABLE_LOG_CONFIG_MOUNT_DIR: &str = "/stackable/mount/log-config"; +pub const STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME: &str = "log-config-mount"; // config file names pub const HIVE_SITE_XML: &str = "hive-site.xml"; pub const HIVE_ENV_SH: &str = "hive-env.sh"; -pub const LOG_4J_PROPERTIES: &str = "log4j.properties"; +pub const HIVE_LOG4J2_PROPERTIES: &str = "hive-log4j2.properties"; // default ports pub const HIVE_PORT_NAME: &str = "hive"; pub const HIVE_PORT: u16 = 9083; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9084; -// Certificates and trust stores +// certificates and trust stores pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts"; pub const SYSTEM_TRUST_STORE_PASSWORD: &str = "changeit"; pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; @@ -48,7 +55,7 @@ pub const JVM_HEAP_FACTOR: f32 = 0.8; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("no metastore role configuration provided"))] - NoMetaStoreRole, + MissingMetaStoreRole, #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, } @@ -75,7 +82,7 @@ pub struct HiveClusterSpec { /// The Hive metastore image to use pub image: ProductImage, #[serde(default, skip_serializing_if = "Option::is_none")] - pub metastore: Option>, + pub metastore: Option>, /// Emergency stop button, if `true` then all pods are stopped without affecting configuration (as setting `replicas` to `0` would) #[serde(default, skip_serializing_if = "Option::is_none")] pub stopped: Option, @@ -97,6 +104,10 @@ pub struct HiveClusterConfig { /// Use with caution. #[serde(default, skip_serializing_if = "Option::is_none")] pub service_type: Option, + /// Name of the Vector aggregator discovery ConfigMap. + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -120,7 +131,7 @@ impl HiveRole { vec![ "bin/start-metastore".to_string(), "--config".to_string(), - STACKABLE_RW_CONFIG_DIR.to_string(), + STACKABLE_CONFIG_DIR.to_string(), "--db-type".to_string(), db_type.to_string(), "--hive-bin-dir".to_string(), @@ -130,7 +141,7 @@ impl HiveRole { vec![ "/bin/hive".to_string(), "--config".to_string(), - STACKABLE_RW_CONFIG_DIR.to_string(), + STACKABLE_CONFIG_DIR.to_string(), "--service".to_string(), "metastore".to_string(), ] @@ -138,6 +149,26 @@ impl HiveRole { } } +#[derive( + Clone, + Debug, + Deserialize, + Display, + Eq, + EnumIter, + JsonSchema, + Ord, + PartialEq, + PartialOrd, + Serialize, +)] +#[serde(rename_all = "kebab-case")] +#[strum(serialize_all = "kebab-case")] +pub enum Container { + Hive, + Vector, +} + #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] #[fragment_attrs( derive( @@ -157,13 +188,28 @@ pub struct MetastoreStorageConfig { pub data: PvcConfig, } -#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] +#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] +#[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") +)] pub struct MetaStoreConfig { /// The location of default database for the Hive warehouse. /// Maps to the `hive.metastore.warehouse.dir` setting. pub warehouse_dir: Option, - pub resources: Option>, + #[fragment_attrs(serde(default))] + pub resources: Resources, + #[fragment_attrs(serde(default))] + pub logging: Logging, } impl MetaStoreConfig { @@ -182,23 +228,27 @@ impl MetaStoreConfig { pub const S3_SSL_ENABLED: &'static str = "fs.s3a.connection.ssl.enabled"; pub const S3_PATH_STYLE_ACCESS: &'static str = "fs.s3a.path.style.access"; - fn default_resources() -> ResourcesFragment { - ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("200m".to_owned())), - max: Some(Quantity("4".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("2Gi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: MetastoreStorageConfigFragment { - data: PvcConfigFragment { - capacity: Some(Quantity("2Gi".to_owned())), - storage_class: None, - selectors: None, + fn default_config() -> MetaStoreConfigFragment { + MetaStoreConfigFragment { + warehouse_dir: None, + resources: ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("200m".to_owned())), + max: Some(Quantity("4".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("2Gi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: MetastoreStorageConfigFragment { + data: PvcConfigFragment { + capacity: Some(Quantity("2Gi".to_owned())), + storage_class: None, + selectors: None, + }, }, }, + logging: product_logging::spec::default_logging(), } } } @@ -269,7 +319,7 @@ pub struct DatabaseConnectionSpec { pub db_type: DbType, } -impl Configuration for MetaStoreConfig { +impl Configuration for MetaStoreConfigFragment { type Configurable = HiveCluster; fn compute_env( @@ -299,7 +349,7 @@ impl Configuration for MetaStoreConfig { ) -> Result>, ConfigError> { let mut result = BTreeMap::new(); result.insert( - Self::DB_TYPE_CLI.to_string(), + MetaStoreConfig::DB_TYPE_CLI.to_string(), Some(hive.spec.cluster_config.database.db_type.to_string()), ); Ok(result) @@ -309,44 +359,50 @@ impl Configuration for MetaStoreConfig { &self, hive: &Self::Configurable, _role_name: &str, - _file: &str, + file: &str, ) -> Result>, ConfigError> { let mut result = BTreeMap::new(); - if let Some(warehouse_dir) = &self.warehouse_dir { - result.insert( - Self::METASTORE_WAREHOUSE_DIR.to_string(), - Some(warehouse_dir.to_string()), - ); + match file { + HIVE_SITE_XML => { + if let Some(warehouse_dir) = &self.warehouse_dir { + result.insert( + MetaStoreConfig::METASTORE_WAREHOUSE_DIR.to_string(), + Some(warehouse_dir.to_string()), + ); + } + result.insert( + MetaStoreConfig::CONNECTION_URL.to_string(), + Some(hive.spec.cluster_config.database.conn_string.clone()), + ); + result.insert( + MetaStoreConfig::CONNECTION_USER_NAME.to_string(), + Some(hive.spec.cluster_config.database.user.clone()), + ); + result.insert( + MetaStoreConfig::CONNECTION_PASSWORD.to_string(), + Some(hive.spec.cluster_config.database.password.clone()), + ); + result.insert( + MetaStoreConfig::CONNECTION_DRIVER_NAME.to_string(), + Some( + hive.spec + .cluster_config + .database + .db_type + .get_jdbc_driver_class() + .to_string(), + ), + ); + + result.insert( + MetaStoreConfig::METASTORE_METRICS_ENABLED.to_string(), + Some("true".to_string()), + ); + } + HIVE_ENV_SH => {} + _ => {} } - result.insert( - Self::CONNECTION_URL.to_string(), - Some(hive.spec.cluster_config.database.conn_string.clone()), - ); - result.insert( - Self::CONNECTION_USER_NAME.to_string(), - Some(hive.spec.cluster_config.database.user.clone()), - ); - result.insert( - Self::CONNECTION_PASSWORD.to_string(), - Some(hive.spec.cluster_config.database.password.clone()), - ); - result.insert( - Self::CONNECTION_DRIVER_NAME.to_string(), - Some( - hive.spec - .cluster_config - .database - .db_type - .get_jdbc_driver_class() - .to_string(), - ), - ); - - result.insert( - Self::METASTORE_METRICS_ENABLED.to_string(), - Some("true".to_string()), - ); Ok(result) } @@ -366,8 +422,8 @@ pub struct NoNamespaceError; impl HiveCluster { /// The name of the role-level load-balanced Kubernetes `Service` - pub fn metastore_role_service_name(&self) -> Option { - self.metadata.name.clone() + pub fn metastore_role_service_name(&self) -> Option<&str> { + self.metadata.name.as_deref() } /// Metadata about a metastore rolegroup @@ -407,28 +463,31 @@ impl HiveCluster { })) } + pub fn get_role(&self, role: &HiveRole) -> Option<&Role> { + match role { + HiveRole::MetaStore => self.spec.metastore.as_ref(), + } + } + /// Retrieve and merge resource configs for role and role groups - pub fn resolve_resource_config_for_role_and_rolegroup( + pub fn merged_config( &self, role: &HiveRole, rolegroup_ref: &RoleGroupRef, - ) -> Result, Error> { + ) -> Result { // Initialize the result with all default values as baseline - let conf_defaults = MetaStoreConfig::default_resources(); + let conf_defaults = MetaStoreConfig::default_config(); - let role = match role { - HiveRole::MetaStore => self.spec.metastore.as_ref().context(NoMetaStoreRoleSnafu)?, - }; + let role = self.get_role(role).context(MissingMetaStoreRoleSnafu)?; // Retrieve role resource config - let mut conf_role: ResourcesFragment = - role.config.config.resources.clone().unwrap_or_default(); + let mut conf_role = role.config.config.to_owned(); // Retrieve rolegroup specific resource config - let mut conf_rolegroup: ResourcesFragment = role + let mut conf_rolegroup = role .role_groups .get(&rolegroup_ref.role_group) - .and_then(|rg| rg.config.config.resources.clone()) + .map(|rg| rg.config.config.clone()) .unwrap_or_default(); // Merge more specific configs into default config @@ -439,7 +498,7 @@ impl HiveCluster { conf_role.merge(&conf_defaults); conf_rolegroup.merge(&conf_role); - tracing::debug!("Merged resource config: {:?}", conf_rolegroup); + tracing::debug!("Merged config: {:?}", conf_rolegroup); fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) } } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index d15e8a8b..77c9be8a 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -19,12 +19,12 @@ serde = "1.0" serde_json = "1.0" snafu = "0.7" stackable-hive-crd = { path = "../crd" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.32.1" } strum = { version = "0.24", features = ["derive"] } tokio = { version = "1.23", features = ["full"] } tracing = "0.1" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.30.2" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.32.1" } stackable-hive-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/command.rs b/rust/operator-binary/src/command.rs index 8704708d..b499a907 100644 --- a/rust/operator-binary/src/command.rs +++ b/rust/operator-binary/src/command.rs @@ -1,6 +1,7 @@ use stackable_hive_crd::{ - HIVE_SITE_XML, STACKABLE_CONFIG_DIR, STACKABLE_RW_CONFIG_DIR, STACKABLE_TRUST_STORE, - STACKABLE_TRUST_STORE_PASSWORD, SYSTEM_TRUST_STORE, SYSTEM_TRUST_STORE_PASSWORD, + HIVE_LOG4J2_PROPERTIES, HIVE_SITE_XML, STACKABLE_CONFIG_DIR, STACKABLE_CONFIG_MOUNT_DIR, + STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + SYSTEM_TRUST_STORE, SYSTEM_TRUST_STORE_PASSWORD, }; use stackable_operator::commons::{ s3::S3ConnectionSpec, @@ -18,10 +19,14 @@ pub fn build_container_command_args( s3_connection_spec: Option<&S3ConnectionSpec>, ) -> Vec { let mut args = vec![ - // copy config files to a writeable empty folder in order to set - // s3 access and secret keys - format!("echo copying {STACKABLE_CONFIG_DIR} to {STACKABLE_RW_CONFIG_DIR}"), - format!("cp -RL {STACKABLE_CONFIG_DIR}/* {STACKABLE_RW_CONFIG_DIR}"), + // copy config files to a writeable empty folder in order to set s3 access and secret keys + format!("echo copying {STACKABLE_CONFIG_MOUNT_DIR} to {STACKABLE_CONFIG_DIR}"), + format!("cp -RL {STACKABLE_CONFIG_MOUNT_DIR}/* {STACKABLE_CONFIG_DIR}"), + + // Copy log4j2 properties + format!("echo copying {STACKABLE_LOG_CONFIG_MOUNT_DIR}/{HIVE_LOG4J2_PROPERTIES} to {STACKABLE_CONFIG_DIR}/{HIVE_LOG4J2_PROPERTIES}"), + format!("cp -RL {STACKABLE_LOG_CONFIG_MOUNT_DIR}/{HIVE_LOG4J2_PROPERTIES} {STACKABLE_CONFIG_DIR}/{HIVE_LOG4J2_PROPERTIES}"), + // Copy system truststore to stackable truststore format!("keytool -importkeystore -srckeystore {SYSTEM_TRUST_STORE} -srcstoretype jks -srcstorepass {SYSTEM_TRUST_STORE_PASSWORD} -destkeystore {STACKABLE_TRUST_STORE} -deststoretype pkcs12 -deststorepass {STACKABLE_TRUST_STORE_PASSWORD} -noprompt") ]; @@ -30,8 +35,8 @@ pub fn build_container_command_args( if s3.credentials.is_some() { args.extend([ format!("echo replacing {ACCESS_KEY_PLACEHOLDER} and {SECRET_KEY_PLACEHOLDER} with secret values."), - format!("sed -i \"s|{ACCESS_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_ACCESS_KEY})|g\" {STACKABLE_RW_CONFIG_DIR}/{HIVE_SITE_XML}"), - format!("sed -i \"s|{SECRET_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_SECRET_KEY})|g\" {STACKABLE_RW_CONFIG_DIR}/{HIVE_SITE_XML}"), + format!("sed -i \"s|{ACCESS_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_ACCESS_KEY})|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"), + format!("sed -i \"s|{SECRET_KEY_PLACEHOLDER}|$(cat {S3_SECRET_DIR}/{S3_SECRET_KEY})|g\" {STACKABLE_CONFIG_DIR}/{HIVE_SITE_XML}"), ]); } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d4fc9543..74d3e033 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,19 +1,18 @@ //! Ensures that `Pod`s are configured and running for each [`HiveCluster`] use crate::command::{self, build_container_command_args, S3_SECRET_DIR}; +use crate::product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}; use crate::{discovery, OPERATOR_NAME}; use fnv::FnvHasher; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hive_crd::{ - DbType, HiveCluster, HiveClusterStatus, HiveRole, MetaStoreConfig, MetastoreStorageConfig, - APP_NAME, CERTS_DIR, HADOOP_HEAPSIZE, HIVE_ENV_SH, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, - JVM_HEAP_FACTOR, LOG_4J_PROPERTIES, METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, - STACKABLE_RW_CONFIG_DIR, + Container, DbType, HiveCluster, HiveClusterStatus, HiveRole, MetaStoreConfig, APP_NAME, + CERTS_DIR, HADOOP_HEAPSIZE, HIVE_ENV_SH, HIVE_PORT, HIVE_PORT_NAME, HIVE_SITE_XML, + JVM_HEAP_FACTOR, METRICS_PORT, METRICS_PORT_NAME, STACKABLE_CONFIG_DIR, + STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR, STACKABLE_CONFIG_MOUNT_DIR_NAME, + STACKABLE_LOG_CONFIG_MOUNT_DIR, STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, STACKABLE_LOG_DIR, + STACKABLE_LOG_DIR_NAME, }; -use stackable_operator::commons::product_image_selection::ResolvedProductImage; -use stackable_operator::k8s_openapi::api::core::v1::VolumeMount; -use stackable_operator::kube::Resource; -use stackable_operator::labels::ObjectLabels; use stackable_operator::{ builder::{ ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder, @@ -21,7 +20,7 @@ use stackable_operator::{ }, cluster_resources::ClusterResources, commons::{ - resources::{NoRuntimeLimits, Resources}, + product_image_selection::ResolvedProductImage, s3::{S3AccessStyle, S3ConnectionSpec}, tls::{CaCert, TlsVerification}, }, @@ -29,18 +28,27 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, Probe, Service, ServicePort, ServiceSpec, - TCPSocketAction, Volume, + ConfigMap, ConfigMapVolumeSource, EmptyDirVolumeSource, Probe, Service, + ServicePort, ServiceSpec, TCPSocketAction, Volume, VolumeMount, }, }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + apimachinery::pkg::{ + api::resource::Quantity, apis::meta::v1::LabelSelector, util::intstr::IntOrString, + }, }, - kube::{runtime::controller::Action, ResourceExt}, - labels::{role_group_selector_labels, role_selector_labels}, + kube::{runtime::controller::Action, Resource, ResourceExt}, + labels::{role_group_selector_labels, role_selector_labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{to_java_heap_value, BinaryMultiple}, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, + product_logging::{ + self, + spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }, + }, role_utils::RoleGroupRef, }; use std::{ @@ -57,6 +65,12 @@ use tracing::warn; pub const HIVE_CONTROLLER_NAME: &str = "hivecluster"; const DOCKER_IMAGE_BASE_NAME: &str = "hive"; +pub const MAX_HIVE_LOG_FILES_SIZE_IN_MIB: u32 = 10; + +const OVERFLOW_BUFFER_ON_LOG_VOLUME_IN_MIB: u32 = 1; +const LOG_VOLUME_SIZE_IN_MIB: u32 = + MAX_HIVE_LOG_FILES_SIZE_IN_MIB + OVERFLOW_BUFFER_ON_LOG_VOLUME_IN_MIB; + pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, @@ -127,14 +141,10 @@ pub enum Error { source: strum::ParseError, db_type: String, }, - #[snafu(display("failed to write discovery config map"))] - InvalidDiscovery { source: discovery::Error }, #[snafu(display("failed to resolve S3 connection"))] ResolveS3Connection { source: stackable_operator::error::Error, }, - #[snafu(display("invalid S3 connection: {reason}"))] - InvalidS3Connection { reason: String }, #[snafu(display( "Hive does not support skipping the verification of the tls enabled S3 server" ))] @@ -161,6 +171,15 @@ pub enum Error { DeleteOrphanedResources { source: stackable_operator::error::Error, }, + #[snafu(display("failed to resolve the Vector aggregator address"))] + ResolveVectorAggregatorAddress { + source: crate::product_logging::Error, + }, + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, } type Result = std::result::Result; @@ -236,11 +255,15 @@ pub async fn reconcile_hive(hive: Arc, ctx: Arc) -> Result, ctx: Arc) -> Result, ctx: Arc) -> Result, - metastore_config: &HashMap>, + role_group_config: &HashMap>, s3_connection_spec: Option<&S3ConnectionSpec>, - resources: &Resources, + merged_config: &MetaStoreConfig, + vector_aggregator_address: Option<&str>, ) -> Result { let mut hive_site_data = String::new(); let mut hive_env_data = String::new(); - let log4j_data = include_str!("../../../deploy/external/log4j.properties"); - for (property_name_kind, config) in metastore_config { + for (property_name_kind, config) in role_group_config { match property_name_kind { PropertyNameKind::File(file_name) if file_name == HIVE_ENV_SH => { + let mut data = BTreeMap::new(); // heap in mebi let heap_in_mebi = to_java_heap_value( - resources + merged_config + .resources .memory .limit .as_ref() @@ -396,7 +422,20 @@ fn build_metastore_rolegroup_config_map( unit: BinaryMultiple::Mebi.to_java_memory_unit(), })?; - hive_env_data = format!("export {HADOOP_HEAPSIZE}={heap_in_mebi}"); + data.insert(HADOOP_HEAPSIZE.to_string(), Some(heap_in_mebi.to_string())); + + // other properties / overrides + for (property_name, property_value) in config { + data.insert(property_name.to_string(), Some(property_value.to_string())); + } + + hive_env_data = data + .into_iter() + .map(|(key, value)| { + format!("export {key}={val}", val = value.unwrap_or_default()) + }) + .collect::>() + .join("\n"); } PropertyNameKind::File(file_name) if file_name == HIVE_SITE_XML => { let mut data = BTreeMap::new(); @@ -454,7 +493,9 @@ fn build_metastore_rolegroup_config_map( } } - ConfigMapBuilder::new() + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder .metadata( ObjectMetaBuilder::new() .name_and_namespace(hive) @@ -470,8 +511,19 @@ fn build_metastore_rolegroup_config_map( .build(), ) .add_data(HIVE_SITE_XML, hive_site_data) - .add_data(HIVE_ENV_SH, hive_env_data) - .add_data(LOG_4J_PROPERTIES, log4j_data) + .add_data(HIVE_ENV_SH, hive_env_data); + + extend_role_group_config_map( + rolegroup, + vector_aggregator_address, + &merged_config.logging, + &mut cm_builder, + ) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + })?; + + cm_builder .build() .with_context(|_| BuildRoleGroupConfigSnafu { rolegroup: rolegroup.clone(), @@ -526,7 +578,7 @@ fn build_metastore_rolegroup_statefulset( rolegroup_ref: &RoleGroupRef, metastore_config: &HashMap>, s3_connection: Option<&S3ConnectionSpec>, - resources: &Resources, + merged_config: &MetaStoreConfig, ) -> Result { let rolegroup = hive .spec @@ -540,8 +592,6 @@ fn build_metastore_rolegroup_statefulset( ContainerBuilder::new(APP_NAME).context(FailedToCreateHiveContainerSnafu { name: APP_NAME.to_string(), })?; - let mut pod_builder = PodBuilder::new(); - pod_builder.node_selector_opt(rolegroup.and_then(|rg| rg.selector.clone())); for (property_name_kind, config) in metastore_config { match property_name_kind { @@ -570,6 +620,8 @@ fn build_metastore_rolegroup_statefulset( } } + let mut pod_builder = PodBuilder::new(); + if let Some(hdfs) = &hive.spec.cluster_config.hdfs { pod_builder.add_volume( VolumeBuilder::new("hdfs-site") @@ -630,32 +682,16 @@ fn build_metastore_rolegroup_statefulset( .join(" "), s3_connection, )) - .add_volume_mounts(vec![ - // We have to mount every config file individually, so that we can add additional config files - // such as hdfs-site.xml as well - VolumeMount { - name: "config".to_string(), - mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-env.sh"), - sub_path: Some("hive-env.sh".to_string()), - ..VolumeMount::default() - }, - VolumeMount { - name: "config".to_string(), - mount_path: format!("{STACKABLE_CONFIG_DIR}/hive-site.xml"), - sub_path: Some("hive-site.xml".to_string()), - ..VolumeMount::default() - }, - VolumeMount { - name: "config".to_string(), - mount_path: format!("{STACKABLE_CONFIG_DIR}/log4j.properties"), - sub_path: Some("log4j.properties".to_string()), - ..VolumeMount::default() - }, - ]) - .add_volume_mount("rwconfig", STACKABLE_RW_CONFIG_DIR) + .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) + .add_volume_mount(STACKABLE_CONFIG_MOUNT_DIR_NAME, STACKABLE_CONFIG_MOUNT_DIR) + .add_volume_mount(STACKABLE_LOG_DIR_NAME, STACKABLE_LOG_DIR) + .add_volume_mount( + STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME, + STACKABLE_LOG_CONFIG_MOUNT_DIR, + ) .add_container_port(HIVE_PORT_NAME, HIVE_PORT.into()) .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - .resources(resources.clone().into()) + .resources(merged_config.resources.clone().into()) .readiness_probe(Probe { initial_delay_seconds: Some(10), period_seconds: Some(10), @@ -677,6 +713,85 @@ fn build_metastore_rolegroup_statefulset( }) .build(); + pod_builder + .metadata_builder(|m| { + m.with_recommended_labels(build_recommended_labels( + hive, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + }) + .image_pull_secrets_from_product_image(resolved_product_image) + .add_container(container_hive) + .add_volume(Volume { + name: STACKABLE_CONFIG_DIR_NAME.to_string(), + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: Some(Quantity("10Mi".to_string())), + }), + ..Volume::default() + }) + .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { + name: STACKABLE_CONFIG_MOUNT_DIR_NAME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(rolegroup_ref.object_name()), + ..Default::default() + }), + ..Default::default() + }) + .add_volume(Volume { + name: STACKABLE_LOG_DIR_NAME.to_string(), + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: Some(Quantity(format!("{LOG_VOLUME_SIZE_IN_MIB}Mi"))), + }), + ..Volume::default() + }) + .node_selector_opt(rolegroup.and_then(|rg| rg.selector.clone())) + .security_context( + PodSecurityContextBuilder::new() + .run_as_user(1000) + .run_as_group(1000) + .fs_group(1000) + .build(), + ); + + if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = merged_config.logging.containers.get(&Container::Hive) + { + pod_builder.add_volume(Volume { + name: STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(config_map.into()), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } else { + pod_builder.add_volume(Volume { + name: STACKABLE_LOG_CONFIG_MOUNT_DIR_NAME.to_string(), + config_map: Some(ConfigMapVolumeSource { + name: Some(rolegroup_ref.object_name()), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }); + } + + if merged_config.logging.enable_vector_agent { + pod_builder.add_container(product_logging::framework::vector_container( + resolved_product_image, + STACKABLE_CONFIG_DIR_NAME, + STACKABLE_LOG_DIR_NAME, + merged_config.logging.containers.get(&Container::Vector), + )); + } + Ok(StatefulSet { metadata: ObjectMetaBuilder::new() .name_and_namespace(hive) @@ -707,39 +822,9 @@ fn build_metastore_rolegroup_statefulset( ..LabelSelector::default() }, service_name: rolegroup_ref.object_name(), - template: pod_builder - .metadata_builder(|m| { - m.with_recommended_labels(build_recommended_labels( - hive, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - }) - .image_pull_secrets_from_product_image(resolved_product_image) - .add_container(container_hive) - .add_volume(Volume { - name: "config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: Some(rolegroup_ref.object_name()), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .add_volume( - VolumeBuilder::new("rwconfig") - .with_empty_dir(Some(""), None) - .build(), - ) - .security_context( - PodSecurityContextBuilder::new() - .run_as_user(1000) - .run_as_group(1000) - .fs_group(1000) - .build(), - ) - .build_template(), - volume_claim_templates: Some(vec![resources + template: pod_builder.build_template(), + volume_claim_templates: Some(vec![merged_config + .resources .storage .data .build_pvc("data", Some(vec!["ReadWriteOnce"]))]), diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 176a541c..e90bba6a 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -1,6 +1,7 @@ mod command; mod controller; mod discovery; +mod product_logging; use crate::controller::HIVE_CONTROLLER_NAME; diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs new file mode 100644 index 00000000..3d3cf072 --- /dev/null +++ b/rust/operator-binary/src/product_logging.rs @@ -0,0 +1,125 @@ +use crate::controller::MAX_HIVE_LOG_FILES_SIZE_IN_MIB; + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_hive_crd::{Container, HiveCluster, HIVE_LOG4J2_PROPERTIES, STACKABLE_LOG_DIR}; +use stackable_operator::{ + builder::ConfigMapBuilder, + client::Client, + k8s_openapi::api::core::v1::ConfigMap, + kube::ResourceExt, + product_logging::{ + self, + spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, + }, + role_utils::RoleGroupRef, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object has no namespace"))] + ObjectHasNoNamespace, + #[snafu(display("failed to retrieve the ConfigMap [{cm_name}]"))] + ConfigMapNotFound { + source: stackable_operator::error::Error, + cm_name: String, + }, + #[snafu(display("failed to retrieve the entry [{entry}] for ConfigMap [{cm_name}]"))] + MissingConfigMapEntry { + entry: &'static str, + cm_name: String, + }, + #[snafu(display("crd validation failure"))] + CrdValidationFailure { source: stackable_hive_crd::Error }, + #[snafu(display("vectorAggregatorConfigMapName must be set"))] + MissingVectorAggregatorAddress, +} + +type Result = std::result::Result; + +const VECTOR_AGGREGATOR_CM_ENTRY: &str = "ADDRESS"; +const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %5p [%t] %c{2}: %m%n"; +const HIVE_LOG_FILE: &str = "hive.log4j2.xml"; + +/// Return the address of the Vector aggregator if the corresponding ConfigMap name is given in the +/// cluster spec +pub async fn resolve_vector_aggregator_address( + hbase: &HiveCluster, + client: &Client, +) -> Result> { + let vector_aggregator_address = if let Some(vector_aggregator_config_map_name) = + &hbase.spec.cluster_config.vector_aggregator_config_map_name + { + let vector_aggregator_address = client + .get::( + vector_aggregator_config_map_name, + hbase + .namespace() + .as_deref() + .context(ObjectHasNoNamespaceSnafu)?, + ) + .await + .context(ConfigMapNotFoundSnafu { + cm_name: vector_aggregator_config_map_name.to_string(), + })? + .data + .and_then(|mut data| data.remove(VECTOR_AGGREGATOR_CM_ENTRY)) + .context(MissingConfigMapEntrySnafu { + entry: VECTOR_AGGREGATOR_CM_ENTRY, + cm_name: vector_aggregator_config_map_name.to_string(), + })?; + Some(vector_aggregator_address) + } else { + None + }; + + Ok(vector_aggregator_address) +} + +/// Extend the role group ConfigMap with logging and Vector configurations +pub fn extend_role_group_config_map( + rolegroup: &RoleGroupRef, + vector_aggregator_address: Option<&str>, + logging: &Logging, + cm_builder: &mut ConfigMapBuilder, +) -> Result<()> { + if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = logging.containers.get(&Container::Hive) + { + cm_builder.add_data( + HIVE_LOG4J2_PROPERTIES, + product_logging::framework::create_log4j2_config( + &format!( + "{STACKABLE_LOG_DIR}/{container}", + container = Container::Hive + ), + HIVE_LOG_FILE, + MAX_HIVE_LOG_FILES_SIZE_IN_MIB, + CONSOLE_CONVERSION_PATTERN, + log_config, + ), + ); + } + + let vector_log_config = if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = logging.containers.get(&Container::Vector) + { + Some(log_config) + } else { + None + }; + + if logging.enable_vector_agent { + cm_builder.add_data( + product_logging::framework::VECTOR_CONFIG_FILE, + product_logging::framework::create_vector_config( + rolegroup, + vector_aggregator_address.context(MissingVectorAggregatorAddressSnafu)?, + vector_log_config, + ), + ); + } + + Ok(()) +} diff --git a/tests/templates/kuttl/logging/00-assert.yaml.j2 b/tests/templates/kuttl/logging/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/logging/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/logging/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/logging/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/logging/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/logging/01-assert.yaml b/tests/templates/kuttl/logging/01-assert.yaml new file mode 100644 index 00000000..7755c481 --- /dev/null +++ b/tests/templates/kuttl/logging/01-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-vector-aggregator +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml new file mode 100644 index 00000000..9925bf08 --- /dev/null +++ b/tests/templates/kuttl/logging/01-install-hbase-vector-aggregator.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install hive-vector-aggregator vector + --namespace $NAMESPACE + --version 0.19.0 + --repo https://helm.vector.dev + --values hive-vector-aggregator-values.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-vector-aggregator-discovery +data: + ADDRESS: hive-vector-aggregator:6123 diff --git a/tests/templates/kuttl/smoke/02-assert.yaml b/tests/templates/kuttl/logging/02-assert.yaml similarity index 100% rename from tests/templates/kuttl/smoke/02-assert.yaml rename to tests/templates/kuttl/logging/02-assert.yaml diff --git a/tests/templates/kuttl/smoke/02-install-postgres.yaml.j2 b/tests/templates/kuttl/logging/02-install-postgres.yaml.j2 similarity index 100% rename from tests/templates/kuttl/smoke/02-install-postgres.yaml.j2 rename to tests/templates/kuttl/logging/02-install-postgres.yaml.j2 diff --git a/tests/templates/kuttl/logging/03-assert.yaml b/tests/templates/kuttl/logging/03-assert.yaml new file mode 100644 index 00000000..102445d2 --- /dev/null +++ b/tests/templates/kuttl/logging/03-assert.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hive-metastore-automatic-log-config +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hive-metastore-custom-log-config +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/03-install-hive.yaml.j2 b/tests/templates/kuttl/logging/03-install-hive.yaml.j2 new file mode 100644 index 00000000..733fa189 --- /dev/null +++ b/tests/templates/kuttl/logging/03-install-hive.yaml.j2 @@ -0,0 +1,83 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-log-config +data: + hive-log4j2.properties: | + appenders = FILE, CONSOLE + + appender.CONSOLE.type = Console + appender.CONSOLE.name = CONSOLE + appender.CONSOLE.target = SYSTEM_ERR + appender.CONSOLE.layout.type = PatternLayout + appender.CONSOLE.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + appender.CONSOLE.filter.threshold.type = ThresholdFilter + appender.CONSOLE.filter.threshold.level = INFO + + appender.FILE.type = RollingFile + appender.FILE.name = FILE + appender.FILE.fileName= /stackable/log/hive/hive.log4j2.xml + appender.FILE.filePattern= /stackable/log/hive/hive.log4j2.xml.%i + appender.FILE.layout.type = XMLLayout + appender.FILE.policies.type = Policies + appender.FILE.policies.size.type = SizeBasedTriggeringPolicy + appender.FILE.policies.size.size = 5MB + appender.FILE.strategy.type = DefaultRolloverStrategy + appender.FILE.strategy.max = 1 + appender.FILE.filter.threshold.type = ThresholdFilter + appender.FILE.filter.threshold.level = INFO + + rootLogger.level=INFO + rootLogger.appenderRefs = CONSOLE, FILE + rootLogger.appenderRef.CONSOLE.ref = CONSOLE + rootLogger.appenderRef.FILE.ref = FILE +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: test-hive +spec: + image: + productVersion: "{{ test_scenario['values']['hive'].split('-stackable')[0] }}" + stackableVersion: "{{ test_scenario['values']['hive'].split('-stackable')[1] }}" + clusterConfig: + database: + connString: jdbc:postgresql://hive-postgresql:5432/hive + user: hive + password: hive + dbType: postgres + vectorAggregatorConfigMapName: hive-vector-aggregator-discovery + metastore: + roleGroups: + automatic-log-config: + replicas: 1 + config: + logging: + enableVectorAgent: true + containers: + hive: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + custom-log-config: + replicas: 1 + config: + logging: + enableVectorAgent: true + containers: + hive: + custom: + configMap: hive-log-config diff --git a/tests/templates/kuttl/logging/04-assert.yaml b/tests/templates/kuttl/logging/04-assert.yaml new file mode 100644 index 00000000..b6e7f17e --- /dev/null +++ b/tests/templates/kuttl/logging/04-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-test-runner +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/04-install-hive-test-runner.yaml b/tests/templates/kuttl/logging/04-install-hive-test-runner.yaml new file mode 100644 index 00000000..23c74abe --- /dev/null +++ b/tests/templates/kuttl/logging/04-install-hive-test-runner.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-test-runner + labels: + app: hive-test-runner +spec: + replicas: 1 + selector: + matchLabels: + app: hive-test-runner + template: + metadata: + labels: + app: hive-test-runner + spec: + containers: + - name: hive-test-runner + image: docker.stackable.tech/stackable/testing-tools:0.1.0-stackable0.1.0 + stdin: true + tty: true diff --git a/tests/templates/kuttl/logging/05-assert.yaml b/tests/templates/kuttl/logging/05-assert.yaml new file mode 100644 index 00000000..2262ea03 --- /dev/null +++ b/tests/templates/kuttl/logging/05-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +commands: + - script: >- + kubectl exec --namespace=$NAMESPACE hive-test-runner-0 -- + python /tmp/test_log_aggregation.py -n $NAMESPACE diff --git a/tests/templates/kuttl/logging/05-test-log-aggregation.yaml b/tests/templates/kuttl/logging/05-test-log-aggregation.yaml new file mode 100644 index 00000000..80ca473f --- /dev/null +++ b/tests/templates/kuttl/logging/05-test-log-aggregation.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl cp ./test_log_aggregation.py $NAMESPACE/hive-test-runner-0:/tmp diff --git a/tests/templates/kuttl/logging/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/logging/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..ac87aab7 --- /dev/null +++ b/tests/templates/kuttl/logging/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,14 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + extendedConfiguration: | + password_encryption=md5 + +auth: + username: hive + password: hive + database: hive diff --git a/tests/templates/kuttl/logging/hive-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/hive-vector-aggregator-values.yaml.j2 new file mode 100644 index 00000000..806e06bf --- /dev/null +++ b/tests/templates/kuttl/logging/hive-vector-aggregator-values.yaml.j2 @@ -0,0 +1,55 @@ +--- +role: Aggregator +service: + ports: + - name: api + port: 8686 + protocol: TCP + targetPort: 8686 + - name: vector + port: 6123 + protocol: TCP + targetPort: 6000 +customConfig: + api: + address: 0.0.0.0:8686 + enabled: true + sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + transforms: + automaticLogConfigMetastoreHive: + type: filter + inputs: [vector] + condition: >- + .pod == "test-hive-metastore-automatic-log-config-0" && + .container == "hive" + automaticLogConfigMetastoreVector: + type: filter + inputs: [vector] + condition: >- + .pod == "test-hive-metastore-automatic-log-config-0" && + .container == "vector" + customLogConfigMetastoreHive: + type: filter + inputs: [vector] + condition: >- + .pod == "test-hive-metastore-custom-log-config-0" && + .container == "hive" + customLogConfigMetastoreVector: + type: filter + inputs: [vector] + condition: >- + .pod == "test-hive-metastore-custom-log-config-0" && + .container == "vector" + sinks: + out: + inputs: [automaticLogConfig*, customLogConfig*] +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + type: vector + address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% else %} + type: blackhole +{% endif %} diff --git a/tests/templates/kuttl/logging/test_log_aggregation.py b/tests/templates/kuttl/logging/test_log_aggregation.py new file mode 100755 index 00000000..b37fa34a --- /dev/null +++ b/tests/templates/kuttl/logging/test_log_aggregation.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import requests + + +def check_processed_events(): + response = requests.post( + 'http://hive-vector-aggregator:8686/graphql', + json={ + 'query': """ + { + transforms(first:100) { + nodes { + componentId + metrics { + processedEventsTotal { + processedEventsTotal + } + } + } + } + } + """ + } + ) + + assert response.status_code == 200, \ + 'Cannot access the API of the vector aggregator.' + + result = response.json() + + transforms = result['data']['transforms']['nodes'] + for transform in transforms: + processedEvents = transform['metrics']['processedEventsTotal']['processedEventsTotal'] + componentId = transform['componentId'] + assert processedEvents > 0, \ + f'No events were processed in "{componentId}".' + + +if __name__ == '__main__': + check_processed_events() + print('Test successful!') diff --git a/tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2 b/tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/orphaned-resources/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/orphaned-resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 index 9f20a16e..3592a9ce 100644 --- a/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 +++ b/tests/templates/kuttl/orphaned-resources/01-install-hive.yaml.j2 @@ -13,7 +13,13 @@ spec: user: APP password: mine dbType: derby +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/resources/00-assert.yaml.j2 b/tests/templates/kuttl/resources/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/resources/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/resources/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/resources/10-assert.yaml b/tests/templates/kuttl/resources/10-assert.yaml.j2 similarity index 89% rename from tests/templates/kuttl/resources/10-assert.yaml rename to tests/templates/kuttl/resources/10-assert.yaml.j2 index c477da93..d9165766 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml +++ b/tests/templates/kuttl/resources/10-assert.yaml.j2 @@ -19,6 +19,9 @@ spec: limits: cpu: "4" memory: 4Gi +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - name: vector +{% endif %} status: readyReplicas: 1 replicas: 1 @@ -56,6 +59,9 @@ spec: limits: cpu: "3" memory: 3Gi +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - name: vector +{% endif %} status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 index fc5c8f28..9c62e80b 100644 --- a/tests/templates/kuttl/resources/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/resources/10-install-hive.yaml.j2 @@ -13,8 +13,13 @@ spec: user: APP password: mine dbType: derby +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} metastore: config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} resources: storage: data: diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/smoke/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-install-minio-certificates.yaml.j2 b/tests/templates/kuttl/smoke/01-install-minio-certificates.yaml.j2 similarity index 100% rename from tests/templates/kuttl/smoke/00-install-minio-certificates.yaml.j2 rename to tests/templates/kuttl/smoke/01-install-minio-certificates.yaml.j2 diff --git a/tests/templates/kuttl/smoke/01-install-minio.yaml.j2 b/tests/templates/kuttl/smoke/02-install-minio.yaml.j2 similarity index 100% rename from tests/templates/kuttl/smoke/01-install-minio.yaml.j2 rename to tests/templates/kuttl/smoke/02-install-minio.yaml.j2 diff --git a/tests/templates/kuttl/smoke/03-assert.yaml b/tests/templates/kuttl/smoke/03-assert.yaml new file mode 100644 index 00000000..4cc2b8b5 --- /dev/null +++ b/tests/templates/kuttl/smoke/03-assert.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: v1 +kind: Service +metadata: + name: hive-postgresql + labels: + app.kubernetes.io/name: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/smoke/03-install-postgres.yaml.j2 b/tests/templates/kuttl/smoke/03-install-postgres.yaml.j2 new file mode 100644 index 00000000..3bcd0134 --- /dev/null +++ b/tests/templates/kuttl/smoke/03-install-postgres.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install hive + --version={{ test_scenario['values']['postgres'] }} + --namespace $NAMESPACE + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/smoke/10-install-hive.yaml.j2 b/tests/templates/kuttl/smoke/10-install-hive.yaml.j2 index 610d1d19..8fae21ad 100644 --- a/tests/templates/kuttl/smoke/10-install-hive.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-hive.yaml.j2 @@ -15,7 +15,13 @@ spec: dbType: postgres s3: reference: minio +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 1 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c65255b5..a7b6833e 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,11 +5,11 @@ dimensions: - "12.1.5" - name: hive values: - - 2.3.9-stackable0.6.0 - - 3.1.3-stackable0.2.0 + - 2.3.9-stackable0.prerelease + - 3.1.3-stackable0.prerelease - name: hive-latest values: - - 3.1.3-stackable0.2.0 + - 3.1.3-stackable0.prerelease - name: s3-use-tls values: - "true" @@ -26,3 +26,7 @@ tests: - name: orphaned-resources dimensions: - hive-latest + - name: logging + dimensions: + - postgres + - hive