From a53faeaa90a23ad1784746962b9a2954c3da7026 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 25 Sep 2025 17:06:16 +0200 Subject: [PATCH 01/14] feat: Add product logging --- rust/operator-binary/src/controller.rs | 31 +- rust/operator-binary/src/controller/build.rs | 14 +- .../src/controller/build/node_config.rs | 356 +++++++++- .../controller/build/opensearch_server.vrl | 0 .../src/controller/build/role_builder.rs | 15 +- .../controller/build/role_group_builder.rs | 274 +++++-- .../src/controller/build/test-vector.sh | 11 + .../src/controller/build/vector-test.yaml | 463 ++++++++++++ .../src/controller/build/vector.yaml | 148 ++++ .../src/controller/validate.rs | 178 ++++- rust/operator-binary/src/crd/mod.rs | 77 +- rust/operator-binary/src/framework.rs | 8 + .../src/framework/builder/pod/container.rs | 41 +- .../src/framework/product_logging.rs | 1 + .../framework/product_logging/framework.rs | 195 +++++ .../templates/kuttl/logging/00-patch-ns.yaml | 15 + tests/templates/kuttl/logging/01-rbac.yaml | 31 + tests/templates/kuttl/logging/10-assert.yaml | 12 + ...-install-opensearch-vector-aggregator.yaml | 17 + ...pensearch-vector-aggregator-values.yaml.j2 | 126 ++++ .../templates/kuttl/logging/20-assert.yaml.j2 | 670 ++++++++++++++++++ .../logging/20-install-opensearch.yaml.j2 | 227 ++++++ tests/templates/kuttl/logging/30-assert.yaml | 11 + .../kuttl/logging/30-test-opensearch.yaml | 164 +++++ tests/templates/kuttl/smoke/10-assert.yaml.j2 | 16 + tests/test-definition.yaml | 5 + 26 files changed, 3000 insertions(+), 106 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/opensearch_server.vrl create mode 100755 rust/operator-binary/src/controller/build/test-vector.sh create mode 100644 rust/operator-binary/src/controller/build/vector-test.yaml create mode 100644 rust/operator-binary/src/controller/build/vector.yaml create mode 100644 rust/operator-binary/src/framework/product_logging.rs create mode 100644 rust/operator-binary/src/framework/product_logging/framework.rs create mode 100644 tests/templates/kuttl/logging/00-patch-ns.yaml create mode 100644 tests/templates/kuttl/logging/01-rbac.yaml create mode 100644 tests/templates/kuttl/logging/10-assert.yaml create mode 100644 tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml create mode 100644 tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 create mode 100644 tests/templates/kuttl/logging/20-assert.yaml.j2 create mode 100644 tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 create mode 100644 tests/templates/kuttl/logging/30-assert.yaml create mode 100644 tests/templates/kuttl/logging/30-test-opensearch.yaml diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 6add4d7..a7ba99f 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -35,6 +35,7 @@ use crate::{ framework::{ ClusterName, ControllerName, HasName, HasUid, NameIsValidLabelValue, NamespaceName, OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, Uid, + product_logging::framework::{ValidatedContainerLogConfigChoice, VectorContainerLogConfig}, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, }; @@ -127,10 +128,23 @@ type OpenSearchNodeResources = #[derive(Clone, Debug, PartialEq)] pub struct ValidatedOpenSearchConfig { pub affinity: StackableAffinity, + pub listener_class: String, + pub logging: ValidatedLogging, pub node_roles: NodeRoles, pub resources: OpenSearchNodeResources, pub termination_grace_period_seconds: i64, - pub listener_class: String, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedLogging { + pub opensearch_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, +} + +impl ValidatedLogging { + pub fn is_vector_agent_enabled(&self) -> bool { + self.vector_container.is_some() + } } /// The validated [`v1alpha1::OpenSearchCluster`] @@ -355,17 +369,20 @@ mod tests { commons::{affinity::StackableAffinity, product_image_selection::ResolvedProductImage}, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; - use super::{Context, OpenSearchRoleGroupConfig, ValidatedCluster}; + use super::{Context, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedLogging}; use crate::{ controller::{OpenSearchNodeResources, ValidatedOpenSearchConfig}, crd::{NodeRoles, v1alpha1}, framework::{ ClusterName, NamespaceName, OperatorName, ProductVersion, RoleGroupName, - builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, + builder::pod::container::EnvVarSet, + product_logging::framework::ValidatedContainerLogConfigChoice, + role_utils::GenericProductSpecificCommonConfig, }, }; @@ -487,10 +504,16 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: "external-stable".to_owned(), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + }, node_roles: NodeRoles(node_roles.to_vec()), resources: OpenSearchNodeResources::default(), termination_grace_period_seconds: 120, - listener_class: "external-stable".to_owned(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 8764751..dfab1ef 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -65,6 +65,7 @@ mod tests { k8s_openapi::api::core::v1::PodTemplateSpec, kube::Resource, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; @@ -73,7 +74,7 @@ mod tests { use crate::{ controller::{ ContextNames, OpenSearchNodeResources, OpenSearchRoleGroupConfig, ValidatedCluster, - ValidatedOpenSearchConfig, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ @@ -200,10 +201,19 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: "external-stable".to_owned(), + logging: ValidatedLogging { + vector_aggregator_config_map_name: None, + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + }, node_roles: NodeRoles(node_roles.to_vec()), resources: OpenSearchNodeResources::default(), termination_grace_period_seconds: 120, - listener_class: "external-stable".to_owned(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 8a18ddc..06e5b51 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -1,17 +1,20 @@ //! Configuration of an OpenSearch node -use std::str::FromStr; +use std::{cmp, str::FromStr}; use serde_json::{Value, json}; -use stackable_operator::builder::pod::container::FieldPathEnvVar; +use stackable_operator::{ + builder::pod::container::FieldPathEnvVar, product_logging::spec::AutomaticContainerLogConfig, +}; use super::ValidatedCluster; use crate::{ controller::OpenSearchRoleGroupConfig, crd::v1alpha1, framework::{ - ServiceName, + ConfigMapName, ServiceName, builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::ValidatedContainerLogConfigChoice, role_group_utils, }, }; @@ -19,6 +22,9 @@ use crate::{ /// The main configuration file of OpenSearch pub const CONFIGURATION_FILE_OPENSEARCH_YML: &str = "opensearch.yml"; +/// The log configuration file +pub const CONFIGURATION_FILE_LOG4J2_PROPERTIES: &str = "log4j2.properties"; + /// The cluster name. /// Type: string pub const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name"; @@ -58,6 +64,15 @@ pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.node pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled"; +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +const OPENSEARCH_SERVER_LOG_FILE: &str = "opensearch_server.json"; +/// File name of the Vector config file +pub const VECTOR_CONFIG_FILE: &str = "vector.json"; +// /// Key in the discovery ConfigMap that holds the vector aggregator address +// const VECTOR_AGGREGATOR_CM_KEY: &str = "ADDRESS"; +// /// Name of the env var in the vector container that holds the vector aggregator address +// const VECTOR_AGGREGATOR_ENV_NAME: &str = "VECTOR_AGGREGATOR_ADDRESS"; + /// Configuration of an OpenSearch node based on the cluster and role-group configuration pub struct NodeConfig { cluster: ValidatedCluster, @@ -81,7 +96,7 @@ impl NodeConfig { } /// Creates the main OpenSearch configuration file in YAML format - pub fn static_opensearch_config_file(&self) -> String { + pub fn static_opensearch_config_file_content(&self) -> String { Self::to_yaml(self.static_opensearch_config()) } @@ -261,6 +276,162 @@ impl NodeConfig { String::new() } } + + pub fn automatic_log_config_file_content(&self) -> Option { + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &self.role_group_config.config.logging.opensearch_container + { + Some(NodeConfig::create_log4j2_config(log_config)) + } else { + None + } + } + + fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { + let log_path = format!( + "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", + container = v1alpha1::Container::OpenSearch.to_container_name() + ); + // TODO Calculate or move to constants + let max_size_in_mib = 10; + let number_of_archived_log_files = 1; + + let loggers = config + .loggers + .iter() + .filter(|(name, _)| name.as_str() != AutomaticContainerLogConfig::ROOT_LOGGER) + .flat_map(|(name, logger_config)| { + [ + ( + format!("logger.{name}.name"), + name.escape_default().to_string(), + ), + ( + format!("logger.{name}.level"), + logger_config.level.to_log4j_literal(), + ), + ] + }) + .collect::>(); + + let root_logger = vec![ + ( + "rootLogger.level".to_owned(), + config.root_log_level().to_log4j2_literal(), + ), + ( + "rootLogger.appenderRef.CONSOLE.ref".to_owned(), + "CONSOLE".to_owned(), + ), + ( + "rootLogger.appenderRef.FILE.ref".to_owned(), + "FILE".to_owned(), + ), + ]; + + let console_appender = vec![ + ("appender.CONSOLE.type".to_owned(), "Console".to_owned()), + ("appender.CONSOLE.name".to_owned(), "CONSOLE".to_owned()), + ( + "appender.CONSOLE.target".to_owned(), + "SYSTEM_ERR".to_owned(), + ), + ( + "appender.CONSOLE.layout.type".to_owned(), + "PatternLayout".to_owned(), + ), + // Same as the default layout pattern of the console appender + // see https://github.com/opensearch-project/OpenSearch/blob/3.1.0/distribution/src/config/log4j2.properties#L17 + ( + "appender.CONSOLE.layout.pattern".to_owned(), + "[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.level".to_owned(), + config + .console + .as_ref() + .and_then(|console| console.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ]; + + let file_appender = vec![ + ("appender.FILE.type".to_owned(), "RollingFile".to_owned()), + ("appender.FILE.name".to_owned(), "FILE".to_owned()), + ("appender.FILE.fileName".to_owned(), log_path.to_owned()), + ( + "appender.FILE.filePattern".to_owned(), + format!("{log_path}.%i"), + ), + ( + "appender.FILE.layout.type".to_owned(), + "OpenSearchJsonLayout".to_owned(), + ), + ( + "appender.FILE.layout.type_name".to_owned(), + "server".to_owned(), + ), + ( + "appender.FILE.policies.type".to_owned(), + "Policies".to_owned(), + ), + ( + "appender.FILE.policies.size.type".to_owned(), + "SizeBasedTriggeringPolicy".to_owned(), + ), + ( + "appender.FILE.policies.size.size".to_owned(), + format!( + "{max_log_file_size_in_mib}MB", + max_log_file_size_in_mib = + cmp::max(1, max_size_in_mib / (1 + number_of_archived_log_files)), + ), + ), + ( + "appender.FILE.strategy.type".to_owned(), + "DefaultRolloverStrategy".to_owned(), + ), + ( + "appender.FILE.strategy.max".to_owned(), + number_of_archived_log_files.to_string(), + ), + ( + "appender.FILE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.FILE.filter.threshold.level".to_owned(), + config + .file + .as_ref() + .and_then(|file| file.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ]; + + [root_logger, loggers, console_appender, file_appender] + .iter() + .flatten() + .map(|(key, value)| format!("{key} = {value}\n")) + .collect() + } + + pub fn custom_log_config_map(&self) -> Option { + if let ValidatedContainerLogConfigChoice::Custom(config_map_name) = + &self.role_group_config.config.logging.opensearch_container + { + Some(config_map_name.clone()) + } else { + None + } + } } #[cfg(test)] @@ -275,13 +446,14 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; use super::*; use crate::{ - controller::ValidatedOpenSearchConfig, + controller::{ValidatedLogging, ValidatedOpenSearchConfig}, crd::NodeRoles, framework::{ ClusterName, NamespaceName, ProductVersion, RoleGroupName, @@ -289,18 +461,42 @@ mod tests { }, }; - pub fn node_config( + struct TestConfig { replicas: u16, - config_settings: &[(&str, &str)], - env_vars: &[(&str, &str)], - ) -> NodeConfig { + config_settings: &'static [(&'static str, &'static str)], + env_vars: &'static [(&'static str, &'static str)], + log_config: ValidatedContainerLogConfigChoice, + } + + impl Default for TestConfig { + fn default() -> Self { + Self { + replicas: 3, + config_settings: &[], + env_vars: &[], + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + } + } + } + + fn node_config(test_config: TestConfig) -> NodeConfig { let image: ProductImage = serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage"); let role_group_config = OpenSearchRoleGroupConfig { - replicas, + replicas: test_config.replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: "cluster-internal".to_string(), + logging: ValidatedLogging { + vector_aggregator_config_map_name: None, + opensearch_container: test_config.log_config, + vector_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -309,18 +505,19 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: [( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), - config_settings + test_config + .config_settings .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(), )] .into(), env_overrides: EnvVarSet::new().with_values( - env_vars + test_config + .env_vars .iter() .map(|(k, v)| (EnvVarName::from_str_unsafe(k), *v)), ), @@ -359,7 +556,10 @@ mod tests { #[test] pub fn test_static_opensearch_config_file() { - let node_config = node_config(2, &[("test", "value")], &[]); + let node_config = node_config(TestConfig { + config_settings: &[("test", "value")], + ..TestConfig::default() + }); assert_eq!( concat!( @@ -370,17 +570,23 @@ mod tests { "test: \"value\"" ) .to_owned(), - node_config.static_opensearch_config_file() + node_config.static_opensearch_config_file_content() ); } #[test] pub fn test_tls_on_http_port_enabled() { - let node_config_tls_undefined = node_config(2, &[], &[]); - let node_config_tls_enabled = - node_config(2, &[("plugins.security.ssl.http.enabled", "true")], &[]); - let node_config_tls_disabled = - node_config(2, &[("plugins.security.ssl.http.enabled", "false")], &[]); + let node_config_tls_undefined = node_config(TestConfig::default()); + + let node_config_tls_enabled = node_config(TestConfig { + config_settings: &[("plugins.security.ssl.http.enabled", "true")], + ..TestConfig::default() + }); + + let node_config_tls_disabled = node_config(TestConfig { + config_settings: &[("plugins.security.ssl.http.enabled", "false")], + ..TestConfig::default() + }); assert!(!node_config_tls_undefined.tls_on_http_port_enabled()); assert!(node_config_tls_enabled.tls_on_http_port_enabled()); @@ -426,7 +632,11 @@ mod tests { #[test] pub fn test_environment_variables() { - let node_config = node_config(2, &[], &[("TEST", "value")]); + let node_config = node_config(TestConfig { + replicas: 2, + env_vars: &[("TEST", "value")], + ..TestConfig::default() + }); assert_eq!( EnvVarSet::new() @@ -453,8 +663,20 @@ mod tests { #[test] pub fn test_discovery_type() { - let node_config_single_node = node_config(1, &[], &[]); - let node_config_multiple_nodes = node_config(2, &[], &[]); + let node_config_single_node = node_config(TestConfig { + replicas: 1, + ..TestConfig::default() + }); + + println!( + "node_config_single_node: {:?}", + node_config_single_node.role_group_config.config + ); + + let node_config_multiple_nodes = node_config(TestConfig { + replicas: 2, + ..TestConfig::default() + }); assert_eq!( "single-node".to_owned(), @@ -468,8 +690,15 @@ mod tests { #[test] pub fn test_initial_cluster_manager_nodes() { - let node_config_single_node = node_config(1, &[], &[]); - let node_config_multiple_nodes = node_config(3, &[], &[]); + let node_config_single_node = node_config(TestConfig { + replicas: 1, + ..TestConfig::default() + }); + + let node_config_multiple_nodes = node_config(TestConfig { + replicas: 3, + ..TestConfig::default() + }); assert_eq!( "".to_owned(), @@ -480,4 +709,81 @@ mod tests { node_config_multiple_nodes.initial_cluster_manager_nodes() ); } + + #[test] + pub fn test_automatic_log_config_file_content() { + let automatic_log_config_node_config = node_config(TestConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + ..TestConfig::default() + }); + + let custom_log_config_node_config = node_config(TestConfig { + log_config: ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( + "custom-log-config", + )), + ..TestConfig::default() + }); + + assert_eq!( + Some(concat!( + "appenders = FILE, CONSOLE\n\n", + "appender.CONSOLE.type = Console\n", + "appender.CONSOLE.name = CONSOLE\n", + "appender.CONSOLE.target = SYSTEM_ERR\n", + "appender.CONSOLE.layout.type = PatternLayout\n", + "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", + "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", + "appender.CONSOLE.filter.threshold.level = INFO\n\n", + "appender.FILE.type = RollingFile\n", + "appender.FILE.name = FILE\n", + "appender.FILE.fileName = /stackable/log/opensearch/opensearch.log4j2.xml\n", + "appender.FILE.filePattern = /stackable/log/opensearch/opensearch.log4j2.xml.%i\n", + "appender.FILE.layout.type = XMLLayout\n", + "appender.FILE.policies.type = Policies\n", + "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", + "appender.FILE.policies.size.size = 5MB\n", + "appender.FILE.strategy.type = DefaultRolloverStrategy\n", + "appender.FILE.strategy.max = 1\n", + "appender.FILE.filter.threshold.type = ThresholdFilter\n", + "appender.FILE.filter.threshold.level = INFO\n\n\n", + "rootLogger.level=INFO\n", + "rootLogger.appenderRefs = CONSOLE, FILE\n", + "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", + "rootLogger.appenderRef.FILE.ref = FILE" + ).to_owned()), + automatic_log_config_node_config.automatic_log_config_file_content() + ); + assert_eq!( + None, + custom_log_config_node_config.automatic_log_config_file_content() + ); + } + + #[test] + pub fn test_custom_log_config_map() { + let custom_log_config_node_config = node_config(TestConfig { + log_config: ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( + "custom-log-config", + )), + ..TestConfig::default() + }); + + let automatic_log_config_node_config = node_config(TestConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + ..TestConfig::default() + }); + + assert_eq!( + Some(ConfigMapName::from_str_unsafe("custom-log-config")), + custom_log_config_node_config.custom_log_config_map() + ); + assert_eq!( + None, + automatic_log_config_node_config.custom_log_config_map() + ); + } } diff --git a/rust/operator-binary/src/controller/build/opensearch_server.vrl b/rust/operator-binary/src/controller/build/opensearch_server.vrl new file mode 100644 index 0000000..e69de29 diff --git a/rust/operator-binary/src/controller/build/role_builder.rs b/rust/operator-binary/src/controller/build/role_builder.rs index 0ad2bcb..166a0a3 100644 --- a/rust/operator-binary/src/controller/build/role_builder.rs +++ b/rust/operator-binary/src/controller/build/role_builder.rs @@ -227,6 +227,7 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use uuid::uuid; @@ -234,7 +235,8 @@ mod tests { use super::RoleBuilder; use crate::{ controller::{ - ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedOpenSearchConfig, + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ @@ -260,6 +262,16 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: "cluster-internal".to_string(), + logging: ValidatedLogging { + vector_aggregator_config_map_name: None, + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -268,7 +280,6 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 313371a..02e09de 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -1,9 +1,9 @@ //! Builder for role-group resources -use std::str::FromStr; +use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ - builder::{meta::ObjectMetaBuilder, pod::container::ContainerBuilder}, + builder::meta::ObjectMetaBuilder, crd::listener::{self}, k8s_openapi::{ DeepMerge, @@ -11,16 +11,25 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ Affinity, ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, - PersistentVolumeClaim, PodSecurityContext, PodSpec, PodTemplateSpec, Probe, - Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, VolumeMount, + EmptyDirVolumeSource, PersistentVolumeClaim, PodSecurityContext, PodSpec, + PodTemplateSpec, Probe, Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, + VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kvp::{Annotations, Label, Labels}, + memory::MemoryQuantity, + product_logging::framework::{ + create_vector_shutdown_file_command, remove_vector_shutdown_file_command, + }, + utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use super::node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}; +use super::node_config::{ + CONFIGURATION_FILE_LOG4J2_PROPERTIES, CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig, + STACKABLE_LOG_DIR, VECTOR_CONFIG_FILE, +}; use crate::{ controller::{ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster}, crd::v1alpha1, @@ -29,11 +38,12 @@ use crate::{ builder::{ meta::ownerreference_from_resource, pod::{ - container::EnvVarName, + container::{EnvVarName, new_container_builder}, volume::{ListenerReference, listener_operator_volume_source_builder_build_pvc}, }, }, kvp::label::{recommended_labels, role_group_selector, role_selector}, + product_logging::framework::vector_container, role_group_utils::ResourceNames, }, }; @@ -44,17 +54,25 @@ pub const TRANSPORT_PORT_NAME: &str = "transport"; pub const TRANSPORT_PORT: u16 = 9300; const CONFIG_VOLUME_NAME: &str = "config"; +const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; const DATA_VOLUME_NAME: &str = "data"; const LISTENER_VOLUME_NAME: &str = "listener"; const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; +const LOG_VOLUME_NAME: &str = "log"; +const LOG_VOLUME_DIR: &str = "/stackable/log"; + const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; fn config_volume_name() -> VolumeName { VolumeName::from_str(CONFIG_VOLUME_NAME).expect("should be a valid Volume name") } +fn log_config_volume_name() -> VolumeName { + VolumeName::from_str(LOG_CONFIG_VOLUME_NAME).expect("should be a valid Volume name") +} + fn data_volume_name() -> VolumeName { VolumeName::from_str(DATA_VOLUME_NAME).expect("should be a valid Volume name") } @@ -64,6 +82,10 @@ fn listener_volume_name() -> PersistentVolumeClaimName { .expect("should be a valid PersistentVolumeClaim name") } +fn log_volume_name() -> VolumeName { + VolumeName::from_str(LOG_VOLUME_NAME).expect("should be a valid Volume name") +} + /// Builder for role-group resources pub struct RoleGroupBuilder<'a> { service_account_name: ServiceAccountName, @@ -110,11 +132,32 @@ impl<'a> RoleGroupBuilder<'a> { .common_metadata(self.resource_names.role_group_config_map()) .build(); - let data = [( + let mut data = BTreeMap::new(); + + data.insert( CONFIGURATION_FILE_OPENSEARCH_YML.to_owned(), - self.node_config.static_opensearch_config_file(), - )] - .into(); + self.node_config.static_opensearch_config_file_content(), + ); + + if let Some(log_config_file_content) = self.node_config.automatic_log_config_file_content() + { + data.insert( + CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned(), + log_config_file_content, + ); + } + + if self + .role_group_config + .config + .logging + .is_vector_agent_enabled() + { + data.insert( + VECTOR_CONFIG_FILE.to_owned(), + include_str!("vector.yaml").to_owned(), + ); + } ConfigMap { metadata, @@ -190,7 +233,54 @@ impl<'a> RoleGroupBuilder<'a> { .with_labels(node_role_labels) .build(); - let container = self.build_container(&self.role_group_config); + let opensearch_container = self.build_opensearch_container(); + let vector_container = if let Some(vector_container_log_config) = + &self.role_group_config.config.logging.vector_container + { + vector_container( + &v1alpha1::Container::Vector.to_container_name(), + vector_container_log_config, + &self.resource_names.cluster_name, + &self.resource_names.role_name, + &self.resource_names.role_group_name, + &self.cluster.image, + &log_config_volume_name(), + &log_volume_name(), + ) + } else { + None + }; + + let volumes = vec![ + Volume { + name: config_volume_name().to_string(), + config_map: Some(ConfigMapVolumeSource { + name: self.resource_names.role_group_config_map().to_string(), + ..Default::default() + }), + ..Volume::default() + }, + Volume { + name: log_config_volume_name().to_string(), + config_map: Some(ConfigMapVolumeSource { + name: self + .node_config + .custom_log_config_map() + .unwrap_or_else(|| self.resource_names.role_group_config_map()) + .to_string(), + ..Default::default() + }), + ..Volume::default() + }, + Volume { + name: log_volume_name().to_string(), + empty_dir: Some(EmptyDirVolumeSource { + size_limit: Some(MemoryQuantity::from_mebi(100.0).into()), + ..EmptyDirVolumeSource::default() + }), + ..Volume::default() + }, + ]; // The PodBuilder is not used because it re-validates the values which are already // validated. For instance, it would be necessary to convert the @@ -209,7 +299,11 @@ impl<'a> RoleGroupBuilder<'a> { .pod_anti_affinity .clone(), }), - containers: vec![container], + // TODO Add annotation that the opensearch container is the main one + containers: [Some(opensearch_container), vector_container] + .into_iter() + .flatten() + .collect(), node_selector: self .role_group_config .config @@ -227,14 +321,7 @@ impl<'a> RoleGroupBuilder<'a> { .config .termination_grace_period_seconds, ), - volumes: Some(vec![Volume { - name: config_volume_name().to_string(), - config_map: Some(ConfigMapVolumeSource { - name: self.resource_names.role_group_config_map().to_string(), - ..Default::default() - }), - ..Volume::default() - }]), + volumes: Some(volumes), ..PodSpec::default() }), }; @@ -277,7 +364,7 @@ impl<'a> RoleGroupBuilder<'a> { } /// Builds the container for the [`PodTemplateSpec`] - fn build_container(&self, role_group_config: &OpenSearchRoleGroupConfig) -> Container { + fn build_opensearch_container(&self) -> Container { // Probe values taken from the official Helm chart let startup_probe = Probe { failure_threshold: Some(30), @@ -315,35 +402,69 @@ impl<'a> RoleGroupBuilder<'a> { .and_then(|env_var| env_var.value.clone()) .unwrap_or(format!("{opensearch_home}/config")); - ContainerBuilder::new("opensearch") - .expect("should be a valid container name") + let volume_mounts = [ + VolumeMount { + mount_path: format!("{opensearch_path_conf}/{CONFIGURATION_FILE_OPENSEARCH_YML}"), + name: config_volume_name().to_string(), + read_only: Some(true), + sub_path: Some(CONFIGURATION_FILE_OPENSEARCH_YML.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: format!( + "{opensearch_path_conf}/{CONFIGURATION_FILE_LOG4J2_PROPERTIES}" + ), + name: log_config_volume_name().to_string(), + read_only: Some(true), + sub_path: Some(CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: format!("{opensearch_home}/data"), + name: data_volume_name().to_string(), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: LISTENER_VOLUME_DIR.to_owned(), + name: LISTENER_VOLUME_NAME.to_owned(), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: LOG_VOLUME_DIR.to_owned(), + name: log_volume_name().to_string(), + ..VolumeMount::default() + }, + ]; + + new_container_builder(&v1alpha1::Container::OpenSearch.to_container_name()) .image_from_product_image(&self.cluster.image) .command(vec![format!( "{opensearch_home}/opensearch-docker-entrypoint.sh" )]) - .args(role_group_config.cli_overrides_to_vec()) - .add_env_vars(env_vars.into()) - .add_volume_mounts([ - VolumeMount { - mount_path: format!( - "{opensearch_path_conf}/{CONFIGURATION_FILE_OPENSEARCH_YML}" - ), - name: config_volume_name().to_string(), - read_only: Some(true), - sub_path: Some(CONFIGURATION_FILE_OPENSEARCH_YML.to_owned()), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: format!("{opensearch_home}/data"), - name: data_volume_name().to_string(), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: LISTENER_VOLUME_DIR.to_owned(), - name: LISTENER_VOLUME_NAME.to_owned(), - ..VolumeMount::default() - }, + .args(self.role_group_config.cli_overrides_to_vec()) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), ]) + .args(vec![format!( + "{COMMON_BASH_TRAP_FUNCTIONS}\n\ + {remove_vector_shutdown_file_command}\n\ + prepare_signal_handlers\n\ + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ + {opensearch_home}/opensearch-docker-entrypoint.sh {extra_args} &\n\ + wait_for_termination $!\n\ + {create_vector_shutdown_file_command}", + extra_args = self.role_group_config.cli_overrides_to_vec().join(" "), + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + )]) + .add_env_vars(env_vars.into()) + .add_volume_mounts(volume_mounts) .expect("The mount paths are statically defined and there should be no duplicates.") .add_container_ports(vec![ ContainerPort { @@ -519,20 +640,26 @@ mod tests { }, k8s_openapi::api::core::v1::PodTemplateSpec, kvp::LabelValue, + product_logging::spec::AutomaticContainerLogConfig, role_utils::GenericRoleConfig, }; use strum::IntoEnumIterator; use uuid::uuid; - use super::{RoleGroupBuilder, config_volume_name, data_volume_name, listener_volume_name}; + use super::{ + RoleGroupBuilder, config_volume_name, data_volume_name, listener_volume_name, + log_config_volume_name, + }; use crate::{ controller::{ - ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, ValidatedOpenSearchConfig, + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + ValidatedContainerLogConfigChoice, ValidatedLogging, ValidatedOpenSearchConfig, }, crd::{NodeRoles, v1alpha1}, framework::{ ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, RoleGroupName, ServiceAccountName, ServiceName, builder::pod::container::EnvVarSet, + product_logging::framework::VectorContainerLogConfig, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -541,6 +668,7 @@ mod tests { fn test_volume_names() { // Test that the functions do not panic config_volume_name(); + log_config_volume_name(); data_volume_name(); listener_volume_name(); } @@ -567,6 +695,19 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), + listener_class: "cluster-internal".to_string(), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_container: None, + // Some(VectorContainerLogConfig { + // log_config: ValidatedContainerLogConfigChoice::Automatic( + // AutomaticContainerLogConfig::default(), + // ), + // vector_aggregator_config_map_name: None, + // }), + }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Data, @@ -575,7 +716,6 @@ mod tests { ]), resources: Resources::default(), termination_grace_period_seconds: 30, - listener_class: "cluster-internal".to_string(), }, config_overrides: HashMap::default(), env_overrides: EnvVarSet::default(), @@ -655,6 +795,32 @@ mod tests { ] }, "data": { + "log4j2.properties": concat!( + "appenders = FILE, CONSOLE\n\n", + "appender.CONSOLE.type = Console\n", + "appender.CONSOLE.name = CONSOLE\n", + "appender.CONSOLE.target = SYSTEM_ERR\n", + "appender.CONSOLE.layout.type = PatternLayout\n", + "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", + "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", + "appender.CONSOLE.filter.threshold.level = INFO\n\n", + "appender.FILE.type = RollingFile\n", + "appender.FILE.name = FILE\n", + "appender.FILE.fileName = /stackable/log/opensearch/opensearch.log4j2.xml\n", + "appender.FILE.filePattern = /stackable/log/opensearch/opensearch.log4j2.xml.%i\n", + "appender.FILE.layout.type = XMLLayout\n", + "appender.FILE.policies.type = Policies\n", + "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", + "appender.FILE.policies.size.size = 5MB\n", + "appender.FILE.strategy.type = DefaultRolloverStrategy\n", + "appender.FILE.strategy.max = 1\n", + "appender.FILE.filter.threshold.type = ThresholdFilter\n", + "appender.FILE.filter.threshold.level = INFO\n\n\n", + "rootLogger.level=INFO\n", + "rootLogger.appenderRefs = CONSOLE, FILE\n", + "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", + "rootLogger.appenderRef.FILE.ref = FILE" + ), "opensearch.yml": concat!( "cluster.name: \"my-opensearch-cluster\"\n", "discovery.type: \"single-node\"\n", @@ -797,6 +963,12 @@ mod tests { "readOnly": true, "subPath": "opensearch.yml" }, + { + "mountPath": "/stackable/opensearch/config/log4j2.properties", + "name": "log-config", + "readOnly": true, + "subPath": "log4j2.properties" + }, { "mountPath": "/stackable/opensearch/data", "name": "data" @@ -819,6 +991,12 @@ mod tests { "name": "my-opensearch-cluster-nodes-default" }, "name": "config" + }, + { + "configMap": { + "name": "my-opensearch-cluster-nodes-default" + }, + "name": "log-config" } ] } diff --git a/rust/operator-binary/src/controller/build/test-vector.sh b/rust/operator-binary/src/controller/build/test-vector.sh new file mode 100755 index 0000000..19be25c --- /dev/null +++ b/rust/operator-binary/src/controller/build/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +LOG_DIR=/stackable/log \ +OPENSEARCH_SERVER_LOG_FILE=opensearch_server.json \ +NAMESPACE=default \ +CLUSTER_NAME=opensearch \ +ROLE_NAME=nodes \ +ROLE_GROUP_NAME=cluster-manager \ +VECTOR_AGGREGATOR=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/vector-test.yaml b/rust/operator-binary/src/controller/build/vector-test.yaml new file mode 100644 index 0000000..162868b --- /dev/null +++ b/rust/operator-binary/src/controller/build/vector-test.yaml @@ -0,0 +1,463 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_timestamp\" at (584:643)". Please adapt the tests if you change VRL code in vector.yaml. + +tests: + - name: Test opensearch_server log entry without stacktrace + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with stacktrace + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,363Z", "level": "INFO", "component": "o.o.c.c.JoinHelper", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}", + "stacktrace": ["org.opensearch.transport.RemoteTransportException: [opensearch-nodes-cluster-manager-0][10.244.0.20:9300][internal:cluster/coordination/join]", + "Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: became follower", + "at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.lambda$$close$$3(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]", + "at java.base/java.util.HashMap$$Values.forEach(HashMap.java:1073) ~[?:?]", + "at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.close(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:829) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:405) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.cluster.coordination.FollowersChecker$$2.doRun(FollowersChecker.java:250) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.common.util.concurrent.ThreadContext$$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.1.0.jar:3.1.0]", + "at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.1.0.jar:3.1.0]", + "at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]", + "at java.base/java.util.concurrent.ThreadPoolExecutor$$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]", + "at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]"] } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473451193Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.c.c.JoinHelper", + "message": "\ + failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}\n\ + \n\ + org.opensearch.transport.RemoteTransportException: [opensearch-nodes-cluster-manager-0][10.244.0.20:9300][internal:cluster/coordination/join]\n\ + Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: became follower\n\ + at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.lambda$$close$$3(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at java.base/java.util.HashMap$$Values.forEach(HashMap.java:1073) ~[?:?]\n\ + at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.close(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:829) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:405) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.cluster.coordination.FollowersChecker$$2.doRun(FollowersChecker.java:250) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.common.util.concurrent.ThreadContext$$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:975) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at org.opensearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:52) ~[opensearch-3.1.0.jar:3.1.0]\n\ + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) ~[?:?]\n\ + at java.base/java.util.concurrent.ThreadPoolExecutor$$Worker.run(ThreadPoolExecutor.java:642) ~[?:?]\n\ + at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.363Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unparsable message + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "JSON not parsable: function call error for \"parse_json\" at (110:133): unable to parse json: EOF while parsing an object at line 2 column 0" + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "{\"type\": \"server\", \"timestamp\": \"2025-10-01T12:47:28,582Z\", \"level\": \"INFO\", \"component\": \"o.o.n.Node\", \"cluster.name\": \"opensearch\", \"node.name\": \"opensearch-nodes-cluster-manager-0\", \"message\": \"started\", \"cluster.uuid\": \"Jh1D6YAhTmyzkHI7vM1WWw\", \"node.id\": \"sk-r0P_TTYuPqaamTFbjKg\"\n", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without a JSON object in the message field + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + false + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "false\n", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unparsable timestamp + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "unparsable", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Timestamp not parsable, using current time instead: function call error for \"parse_timestamp\" at (584:643): Invalid timestamp \"unparsable\": input contains invalid characters" + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without timestamp + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-01T12:47:29.473487331Z" + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without logger + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Logger not found." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without level + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "file": "opensearch_server.json", + "errors": [ + "Level not found, using \"INFO\" instead." + ], + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry with unknown level + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "CRITICAL", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "started", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Level \"CRITICAL\" unknown, using \"INFO\" instead." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "started", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test opensearch_server log entry without message + inputs: + - type: log + insert_at: processed_files_opensearch_server + log_fields: + file: /stackable/log/opensearch/opensearch_server.json + message: | + {"type": "server", "timestamp": "2025-10-01T12:47:28,582Z", "level": "INFO", "component": "o.o.n.Node", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "cluster.uuid": "Jh1D6YAhTmyzkHI7vM1WWw", "node.id": "sk-r0P_TTYuPqaamTFbjKg" } + pod: opensearch-nodes-cluster-manager-0 + source_type: file + timestamp: 2025-10-01T12:47:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opensearch", + "container": "opensearch", + "errors": [ + "Message not found." + ], + "file": "opensearch_server.json", + "level": "INFO", + "logger": "o.o.n.Node", + "message": "", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": t'2025-10-01T10:47:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + debug: false, + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: opensearch-nodes-cluster-manager-0 + revision: dc7e792 2025-08-12 13:47:08.632326804 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "opensearch", + "container": "vector", + "debug": "false,", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "opensearch-nodes-cluster-manager-0", + "revision": "dc7e792 2025-08-12 13:47:08.632326804", + "role": "nodes", + "roleGroup": "cluster-manager", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - TRACE/DEBUG + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: TRACE + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector + - name: Test Vector internal log level filtering - INFO + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - WARN + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: WARN + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("WARN", .metadata.level) + - name: Test Vector internal log level filtering - ERROR + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: ERROR + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("ERROR", .metadata.level) diff --git a/rust/operator-binary/src/controller/build/vector.yaml b/rust/operator-binary/src/controller/build/vector.yaml new file mode 100644 index 0000000..8233111 --- /dev/null +++ b/rust/operator-binary/src/controller/build/vector.yaml @@ -0,0 +1,148 @@ +data_dir: /stackable/vector/var + +log_schema: + host_key: pod + +sources: + # Reads logs created with the OpenSearchJsonLayout and the type server + files_opensearch_server: + type: file + include: + - ${LOG_DIR}/*/${OPENSEARCH_SERVER_LOG_FILE} + multiline: + condition_pattern: ^[^{] + mode: continue_through + start_pattern: ^\{ + timeout_ms: 100 + + # Reads the internal Vector logs + vector: + type: internal_logs + +transforms: + # Transforms logs created with the OpenSearchJsonLayout and the type server + processed_files_opensearch_server: + inputs: + - files_opensearch_server + type: remap + source: | + raw_message = string!(.message) + + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S,%3fZ") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.component) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + stacktrace = join(event.stacktrace, "\n") ?? "" + .message = join!(compact([.message, stacktrace]), "\n\n") + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { + .processed_file = del(.file) + } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + # TODO + address: ${VECTOR_AGGREGATOR} + console: + inputs: + - vector + type: console + encoding: + codec: json diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index bbf763a..24dd20e 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -5,6 +5,7 @@ use std::{collections::BTreeMap, num::TryFromIntError, str::FromStr}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ kube::{Resource, ResourceExt}, + product_logging::spec::Logging, role_utils::RoleGroup, shared::time::Duration, }; @@ -12,13 +13,16 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use super::{ ContextNames, OpenSearchRoleGroupConfig, ProductVersion, RoleGroupName, ValidatedCluster, - ValidatedOpenSearchConfig, + ValidatedLogging, ValidatedOpenSearchConfig, }; use crate::{ - crd::v1alpha1::{self, OpenSearchConfig, OpenSearchConfigFragment}, + crd::v1alpha1::{self}, framework::{ - ClusterName, NamespaceName, Uid, + ClusterName, ConfigMapName, NamespaceName, Uid, builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::{ + VectorContainerLogConfig, validate_logging_configuration_for_container, + }, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig, with_validated_config}, }, }; @@ -35,6 +39,11 @@ pub enum Error { #[snafu(display("failed to get the cluster UID"))] GetClusterUid {}, + #[snafu(display( + "failed to get vectorAggregatorConfigMapName; It must be set if enableVectorAgent is true." + ))] + GetVectorAggregatorConfigMapName {}, + #[snafu(display("failed to set cluster name"))] ParseClusterName { source: crate::framework::Error }, @@ -55,11 +64,19 @@ pub enum Error { #[snafu(display("failed to set role-group name"))] ParseRoleGroupName { source: crate::framework::Error }, + #[snafu(display("failed to set vectorAggregatorConfigMapName"))] + ParseVectorAggregatorConfigMapName { source: crate::framework::Error }, + #[snafu(display("failed to resolve product image"))] ResolveProductImage { source: stackable_operator::commons::product_image_selection::Error, }, + #[snafu(display("failed to validate the logging configuration"))] + ValidateLoggingConfig { + source: crate::framework::product_logging::framework::Error, + }, + #[snafu(display("fragment validation failure"))] ValidateOpenSearchConfig { source: stackable_operator::config::fragment::ValidationError, @@ -133,9 +150,12 @@ fn validate_role_group_config( context_names: &ContextNames, cluster_name: &ClusterName, cluster: &v1alpha1::OpenSearchCluster, - role_group_config: &RoleGroup, + role_group_config: &RoleGroup< + v1alpha1::OpenSearchConfigFragment, + GenericProductSpecificCommonConfig, + >, ) -> Result { - let merged_role_group: RoleGroup = with_validated_config( + let merged_role_group: RoleGroup = with_validated_config( role_group_config, &cluster.spec.nodes, &v1alpha1::OpenSearchConfig::default_config( @@ -146,6 +166,24 @@ fn validate_role_group_config( ) .context(ValidateOpenSearchConfigSnafu)?; + let vector_aggregator_config_map_name = if let Some(config_map_name) = &cluster + .spec + .cluster_config + .vector_aggregator_config_map_name + { + Some( + ConfigMapName::from_str(config_map_name) + .context(ParseVectorAggregatorConfigMapNameSnafu)?, + ) + } else { + None + }; + + let logging = validate_logging_configuration( + &merged_role_group.config.config.logging, + vector_aggregator_config_map_name, + )?; + let graceful_shutdown_timeout = merged_role_group.config.config.graceful_shutdown_timeout; let termination_grace_period_seconds = graceful_shutdown_timeout.as_secs().try_into().context( TerminationGracePeriodTooLongSnafu { @@ -155,10 +193,11 @@ fn validate_role_group_config( let validated_config = ValidatedOpenSearchConfig { affinity: merged_role_group.config.config.affinity, + listener_class: merged_role_group.config.config.listener_class, + logging, node_roles: merged_role_group.config.config.node_roles, resources: merged_role_group.config.config.resources, termination_grace_period_seconds, - listener_class: merged_role_group.config.config.listener_class, }; let mut env_overrides = EnvVarSet::new(); @@ -182,6 +221,36 @@ fn validate_role_group_config( }) } +fn validate_logging_configuration( + logging: &Logging, + vector_aggregator_config_map_name: Option, +) -> Result { + let opensearch_container = + validate_logging_configuration_for_container(logging, v1alpha1::Container::OpenSearch) + .context(ValidateLoggingConfigSnafu)?; + + // TODO Move to framework? + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = + vector_aggregator_config_map_name.context(GetVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container( + logging, + v1alpha1::Container::Vector, + ) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + + Ok(ValidatedLogging { + opensearch_container, + vector_container, + }) +} + #[cfg(test)] mod tests { use std::str::FromStr; @@ -201,6 +270,11 @@ mod tests { }, kube::api::ObjectMeta, kvp::LabelValue, + product_logging::spec::{ + AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfigFragment, + ContainerLogConfigChoiceFragment, ContainerLogConfigFragment, + CustomContainerLogConfigFragment, LogLevel, LoggerConfig, LoggingFragment, + }, role_utils::{CommonConfiguration, GenericRoleConfig, Role, RoleGroup}, shared::time::Duration, }; @@ -208,15 +282,16 @@ mod tests { use super::{ErrorDiscriminants, validate}; use crate::{ - controller::{ContextNames, ValidatedCluster, ValidatedOpenSearchConfig}, + controller::{ContextNames, ValidatedCluster, ValidatedLogging, ValidatedOpenSearchConfig}, crd::{ NodeRoles, - v1alpha1::{self, OpenSearchClusterSpec, OpenSearchConfigFragment, StorageConfig}, + v1alpha1::{self}, }, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, + ClusterName, ConfigMapName, ControllerName, NamespaceName, OperatorName, ProductName, + ProductVersion, RoleGroupName, builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::ValidatedContainerLogConfigChoice, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, }; @@ -283,6 +358,30 @@ mod tests { }), ..StackableAffinity::default() }, + listener_class: "listener-class-from-role-group-level".to_owned(), + logging: ValidatedLogging { + opensearch_container: ValidatedContainerLogConfigChoice::Custom( + ConfigMapName::from_str_unsafe("custom-log-config-map") + ), + vector_container: None, + // ValidatedContainerLogConfigChoice::Automatic( + // AutomaticContainerLogConfig { + // loggers: [( + // "ROOT".to_owned(), + // LoggerConfig { + // level: LogLevel::INFO + // } + // )] + // .into(), + // console: Some(AppenderConfig { + // level: Some(LogLevel::INFO) + // }), + // file: Some(AppenderConfig { + // level: Some(LogLevel::INFO) + // }), + // }, + // ), + }, node_roles: NodeRoles( [ v1alpha1::NodeRole::ClusterManager, @@ -301,7 +400,7 @@ mod tests { min: Some(Quantity("1".to_owned())), max: Some(Quantity("4".to_owned())) }, - storage: StorageConfig { + storage: v1alpha1::StorageConfig { data: PvcConfig { capacity: Some(Quantity("8Gi".to_owned())), ..PvcConfig::default() @@ -309,7 +408,6 @@ mod tests { } }, termination_grace_period_seconds: 300, - listener_class: "listener-class-from-role-group-level".to_owned(), }, config_overrides: [( "opensearch.yml".to_owned(), @@ -463,6 +561,31 @@ mod tests { ); } + // #[test] + // fn test_validate_err_parse_container_name() { + // test_validate_err( + // |cluster| { + // cluster.spec.nodes.config.config.logging = LoggingFragment { + // enable_vector_agent: Some(true), + // containers: [( + // v1alpha1::Container::OpenSearch, + // ContainerLogConfigFragment { + // choice: Some(ContainerLogConfigChoiceFragment::Custom( + // CustomContainerLogConfigFragment { + // custom: ConfigMapLogConfigFragment { + // config_map: Some("invalid ConfigMap name".to_owned()), + // }, + // }, + // )), + // }, + // )] + // .into(), + // } + // }, + // ErrorDiscriminants::ParseContainerName, + // ); + // } + #[test] fn test_validate_err_termination_grace_period_too_long() { test_validate_err( @@ -516,16 +639,37 @@ mod tests { uid: Some("e6ac237d-a6d4-43a1-8135-f36506110912".to_owned()), ..ObjectMeta::default() }, - spec: OpenSearchClusterSpec { + spec: v1alpha1::OpenSearchClusterSpec { image: serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage structure"), + cluster_config: v1alpha1::OpenSearchClusterConfig { + vector_aggregator_config_map_name: None, + }, cluster_operation: ClusterOperation::default(), nodes: Role { config: CommonConfiguration { - config: OpenSearchConfigFragment { + config: v1alpha1::OpenSearchConfigFragment { graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(5)), listener_class: Some("listener-class-from-role-level".to_owned()), - ..OpenSearchConfigFragment::default() + logging: LoggingFragment { + enable_vector_agent: Some(true), + containers: [( + v1alpha1::Container::OpenSearch, + ContainerLogConfigFragment { + choice: Some(ContainerLogConfigChoiceFragment::Custom( + CustomContainerLogConfigFragment { + custom: ConfigMapLogConfigFragment { + config_map: Some( + "custom-log-config-map".to_owned(), + ), + }, + }, + )), + }, + )] + .into(), + }, + ..v1alpha1::OpenSearchConfigFragment::default() }, config_overrides: [( "opensearch.yml".to_owned(), @@ -567,11 +711,11 @@ mod tests { "default".to_owned(), RoleGroup { config: CommonConfiguration { - config: OpenSearchConfigFragment { + config: v1alpha1::OpenSearchConfigFragment { listener_class: Some( "listener-class-from-role-group-level".to_owned(), ), - ..OpenSearchConfigFragment::default() + ..v1alpha1::OpenSearchConfigFragment::default() }, config_overrides: [( "opensearch.yml".to_owned(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 199c51d..af72612 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,6 +17,7 @@ use stackable_operator::{ }, k8s_openapi::{api::core::v1::PodAntiAffinity, apimachinery::pkg::api::resource::Quantity}, kube::CustomResource, + product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, @@ -26,7 +27,7 @@ use stackable_operator::{ use strum::{Display, EnumIter}; use crate::framework::{ - ClusterName, NameIsValidLabelValue, ProductName, RoleName, + ClusterName, ContainerName, NameIsValidLabelValue, ProductName, RoleName, role_utils::GenericProductSpecificCommonConfig, }; @@ -43,7 +44,6 @@ const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; ) )] pub mod versioned { - /// An OpenSearch cluster stacklet. This resource is managed by the Stackable operator for /// OpenSearch. Find more information on how to use it and the resources that the operator /// generates in the [operator documentation](DOCS_BASE_URL_PLACEHOLDER/opensearch/). @@ -61,6 +61,10 @@ pub mod versioned { // no doc - docs in ProductImage struct. pub image: ProductImage, + /// Configuration that applies to all roles and role groups + #[serde(default)] + pub cluster_config: v1alpha1::OpenSearchClusterConfig, + // no doc - docs in ClusterOperation struct. #[serde(default)] pub cluster_operation: ClusterOperation, @@ -70,6 +74,17 @@ pub mod versioned { Role, } + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[serde(rename_all = "camelCase")] + pub struct OpenSearchClusterConfig { + /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). + /// It must contain the key `ADDRESS` with the address of the Vector aggregator. + /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) + /// to learn how to configure log aggregation with Vector. + #[serde(skip_serializing_if = "Option::is_none")] + pub vector_aggregator_config_map_name: Option, + } + // The possible node roles are by default the built-in roles and the search role, see // https://github.com/opensearch-project/OpenSearch/blob/3.0.0/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java#L609-L614. // @@ -134,6 +149,13 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Duration, + /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + #[fragment_attrs(serde(default))] + pub listener_class: String, + + #[fragment_attrs(serde(default))] + pub logging: Logging, + /// Roles of the OpenSearch node. /// /// Consult the [node roles @@ -142,10 +164,28 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub resources: Resources, + } - /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. - #[fragment_attrs(serde(default))] - pub listener_class: String, + // TODO All derives required? + #[derive( + Clone, + Debug, + Deserialize, + Display, + Eq, + EnumIter, + JsonSchema, + Ord, + PartialEq, + PartialOrd, + Serialize, + )] + pub enum Container { + #[serde(rename = "opensearch")] + OpenSearch, + + #[serde(rename = "vector")] + Vector, } #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] @@ -217,6 +257,8 @@ impl v1alpha1::OpenSearchConfig { ), // Defaults taken from the Helm chart, see // https://github.com/opensearch-project/helm-charts/blob/opensearch-3.0.0/charts/opensearch/values.yaml#L16-L20 + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), + logging: product_logging::spec::default_logging(), node_roles: Some(NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Ingest, @@ -248,7 +290,6 @@ impl v1alpha1::OpenSearchConfig { }, }, }, - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } } @@ -268,8 +309,24 @@ impl NodeRoles { impl Atomic for NodeRoles {} +impl v1alpha1::Container { + /// Returns the validated container name + /// + /// This name should match the one defined by the user (see the serde annotation at + /// [`v1alpha1::Container`], but it could differ if it was renamed. + pub fn to_container_name(&self) -> ContainerName { + ContainerName::from_str(match self { + v1alpha1::Container::OpenSearch => "opensearch", + v1alpha1::Container::Vector => "vector", + }) + .expect("should be a valid container name") + } +} + #[cfg(test)] mod tests { + use strum::IntoEnumIterator; + use crate::crd::v1alpha1; #[test] @@ -292,4 +349,12 @@ mod tests { serde_json::from_str("\"cluster_manager\"").expect("should be deserializable") ); } + + #[test] + fn test_to_container_name() { + for container in v1alpha1::Container::iter() { + // Test that the function does not panic + container.to_container_name(); + } + } } diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 46c1d5a..b338d4b 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -29,6 +29,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; pub mod builder; pub mod cluster_resources; pub mod kvp; +pub mod product_logging; pub mod role_group_utils; pub mod role_utils; @@ -256,6 +257,13 @@ attributed_string_type! { (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), is_rfc_1123_dns_subdomain_name } +attributed_string_type! { + ContainerName, + "The name of a container in a Pod", + "opensearch", + (max_length = RFC_1123_LABEL_MAX_LENGTH), + is_rfc_1123_label_name +} attributed_string_type! { ClusterRoleName, "The name of a ClusterRole", diff --git a/rust/operator-binary/src/framework/builder/pod/container.rs b/rust/operator-binary/src/framework/builder/pod/container.rs index 589922f..de71f2b 100644 --- a/rust/operator-binary/src/framework/builder/pod/container.rs +++ b/rust/operator-binary/src/framework/builder/pod/container.rs @@ -2,11 +2,13 @@ use std::{collections::BTreeMap, fmt::Display, str::FromStr}; use snafu::Snafu; use stackable_operator::{ - builder::pod::container::FieldPathEnvVar, - k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector}, + builder::pod::container::{ContainerBuilder, FieldPathEnvVar}, + k8s_openapi::api::core::v1::{ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector}, }; use strum::{EnumDiscriminants, IntoStaticStr}; +use crate::framework::{ConfigMapName, ContainerName}; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { @@ -17,6 +19,11 @@ pub enum Error { ParseEnvVarName { env_var_name: String }, } +/// Infallible variant of [`stackable_operator::builder::pod::container::ContainerBuilder::new`] +pub fn new_container_builder(container_name: &ContainerName) -> ContainerBuilder { + ContainerBuilder::new(container_name.as_ref()).expect("should be a valid container name") +} + /// Validated environment variable name #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct EnvVarName(String); @@ -139,6 +146,36 @@ impl EnvVarSet { self } + + /// Adds an environment variable with the given name and field path to this set + /// + /// An [`EnvVar`] with the same name is overridden. + pub fn with_config_map_key_ref( + mut self, + name: impl Into, + config_map_name: &ConfigMapName, + config_map_key: impl Into, + ) -> Self { + let name: EnvVarName = name.into(); + + self.0.insert( + name.clone(), + EnvVar { + name: name.to_string(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: config_map_key.into(), + name: config_map_name.to_string(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }, + ); + + self + } } impl From for Vec { diff --git a/rust/operator-binary/src/framework/product_logging.rs b/rust/operator-binary/src/framework/product_logging.rs new file mode 100644 index 0000000..0c71749 --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging.rs @@ -0,0 +1 @@ +pub mod framework; diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs new file mode 100644 index 0000000..2c630c4 --- /dev/null +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -0,0 +1,195 @@ +use std::{fmt::Display, str::FromStr}; + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::pod::{container::FieldPathEnvVar, resources::ResourceRequirementsBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::{Container, VolumeMount}, + product_logging::spec::{ + AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, LogLevel, Logging, + }, +}; +use strum::{EnumDiscriminants, IntoStaticStr}; + +use crate::framework::{ + ClusterName, ConfigMapName, ContainerName, RoleGroupName, RoleName, VolumeName, + builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, +}; + +const STACKABLE_LOG_DIR: &str = "/stackable/log"; +const VECTOR_CONFIG_FILE: &str = "vector.yaml"; + +#[derive(Snafu, Debug, EnumDiscriminants)] +#[strum_discriminants(derive(IntoStaticStr))] +pub enum Error { + #[snafu(display("failed to get the container log configuration"))] + GetContainerLogConfiguration { container: String }, + + #[snafu(display("failed to parse the container name"))] + ParseContainerName { source: crate::framework::Error }, +} + +type Result = std::result::Result; + +#[derive(Clone, Debug, PartialEq)] +pub enum ValidatedContainerLogConfigChoice { + Automatic(AutomaticContainerLogConfig), + Custom(ConfigMapName), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct VectorContainerLogConfig { + pub log_config: ValidatedContainerLogConfigChoice, + pub vector_aggregator_config_map_name: ConfigMapName, +} + +pub fn validate_logging_configuration_for_container( + logging: &Logging, + container: T, +) -> Result +where + T: Clone + Display + Ord, +{ + let container_log_config_choice = logging + .containers + .get(&container) + .and_then(|container_log_config| container_log_config.choice.as_ref()) + // This should never happen because a default configuration should have been set in + // `v1alpha1::OpenSearchConfig` for all containers. + .context(GetContainerLogConfigurationSnafu { + container: container.to_string(), + })?; + + let validated_container_log_config_choice = match container_log_config_choice { + ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + }) => ValidatedContainerLogConfigChoice::Custom( + ConfigMapName::from_str(config_map).context(ParseContainerNameSnafu)?, + ), + ContainerLogConfigChoice::Automatic(automatic_log_config) => { + ValidatedContainerLogConfigChoice::Automatic(automatic_log_config.clone()) + } + }; + + Ok(validated_container_log_config_choice) +} + +/// Builds the container for the [`PodTemplateSpec`] +pub fn vector_container( + container_name: &ContainerName, + vector_container_log_config: &VectorContainerLogConfig, + cluster_name: &ClusterName, + role_name: &RoleName, + role_group_name: &RoleGroupName, + image: &ResolvedProductImage, + log_config_volume_name: &VolumeName, + log_volume_name: &VolumeName, +) -> Option { + let log_level = if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &vector_container_log_config.log_config + { + log_config.root_log_level() + } else { + LogLevel::default() + }; + let vector_file_log_level = + if let ValidatedContainerLogConfigChoice::Automatic(AutomaticContainerLogConfig { + file: Some(AppenderConfig { + level: Some(log_level), + }), + .. + }) = vector_container_log_config.log_config + { + log_level + } else { + LogLevel::default() + }; + + let env_vars = EnvVarSet::new() + .with_value(EnvVarName::from_str_unsafe("CLUSTER_NAME"), cluster_name) + .with_value(EnvVarName::from_str_unsafe("LOG_DIR"), "/stackable/log") + .with_field_path( + EnvVarName::from_str_unsafe("NAMESPACE"), + FieldPathEnvVar::Namespace, + ) + .with_value( + EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), + "opensearch_server.json", + ) + .with_value( + EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), + role_group_name, + ) + .with_value(EnvVarName::from_str_unsafe("ROLE_NAME"), role_name) + .with_config_map_key_ref( + EnvVarName::from_str_unsafe("VECTOR_AGGREGATOR"), + &vector_container_log_config.vector_aggregator_config_map_name, + // TODO type-safe? + "ADDRESS", + ) + .with_value( + EnvVarName::from_str_unsafe("VECTOR_CONFIG_YAML"), + format!("/stackable/config/{VECTOR_CONFIG_FILE}"), + ) + .with_value( + EnvVarName::from_str_unsafe("VECTOR_FILE_LOG_LEVEL"), + vector_file_log_level.to_vector_literal(), + ) + .with_value( + EnvVarName::from_str_unsafe("VECTOR_LOG"), + log_level.to_vector_literal(), + ); + + let resources = ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(); + + let container = new_container_builder(container_name) + .image_from_product_image(image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![format!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n\ + vector & vector_pid=$!\n\ + if [ ! -f \"{vector_control_directory}/{SHUTDOWN_FILE}\" ]; then\n\ + mkdir -p {vector_control_directory}\n\ + inotifywait -qq --event create {vector_control_directory};\n\ + fi\n\ + sleep 1\n\ + kill $vector_pid", + vector_control_directory = format!("{STACKABLE_LOG_DIR}/_vector"), + // TODO + SHUTDOWN_FILE = "shutdown" + )]) + .add_env_vars(env_vars.into()) + .add_volume_mounts([ + VolumeMount { + mount_path: format!( + "/stackable/config/{VECTOR_CONFIG_FILE}" + ), + name: log_config_volume_name.to_string(), + read_only: Some(true), + sub_path: Some(VECTOR_CONFIG_FILE.to_owned()), + ..VolumeMount::default() + }, + VolumeMount { + mount_path: STACKABLE_LOG_DIR.to_owned(), + name: log_volume_name.to_string(), + ..VolumeMount::default() + }, + ]) + .expect("The mount paths are statically defined and there should be no duplicates.") + .resources(resources) + .build(); + + Some(container) +} diff --git a/tests/templates/kuttl/logging/00-patch-ns.yaml b/tests/templates/kuttl/logging/00-patch-ns.yaml new file mode 100644 index 0000000..d4f91fa --- /dev/null +++ b/tests/templates/kuttl/logging/00-patch-ns.yaml @@ -0,0 +1,15 @@ +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl patch namespace $NAMESPACE --patch=' + { + "metadata": { + "labels": { + "pod-security.kubernetes.io/enforce": "privileged" + } + } + }' + timeout: 120 diff --git a/tests/templates/kuttl/logging/01-rbac.yaml b/tests/templates/kuttl/logging/01-rbac.yaml new file mode 100644 index 0000000..64eced8 --- /dev/null +++ b/tests/templates/kuttl/logging/01-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/logging/10-assert.yaml b/tests/templates/kuttl/logging/10-assert.yaml new file mode 100644 index 0000000..88b268d --- /dev/null +++ b/tests/templates/kuttl/logging/10-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-vector-aggregator +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml b/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml new file mode 100644 index 0000000..dfef758 --- /dev/null +++ b/tests/templates/kuttl/logging/10-install-opensearch-vector-aggregator.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install opensearch-vector-aggregator vector + --namespace $NAMESPACE + --version 0.45.0 + --repo https://helm.vector.dev + --values 10_opensearch-vector-aggregator-values.yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: opensearch-vector-aggregator-discovery +data: + ADDRESS: opensearch-vector-aggregator:6123 diff --git a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 new file mode 100644 index 0000000..9bc5fed --- /dev/null +++ b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 @@ -0,0 +1,126 @@ +--- +role: Aggregator +service: + ports: + - name: api + port: 8686 + protocol: TCP + targetPort: 8686 + - name: vector + port: 6123 + protocol: TCP + targetPort: 6000 +customConfig: + api: + address: 0.0.0.0:8686 + enabled: true + sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" + transforms: + validEvents: + type: filter + inputs: [vector] + condition: is_null(.errors) + filteredAutomaticLogConfigMasterHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-master-automatic-log-config-0" && + .container == "opensearch" + filteredAutomaticLogConfigMasterVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-master-automatic-log-config-0" && + .container == "vector" + filteredCustomLogConfigMasterHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-master-custom-log-config-0" && + .container == "opensearch" + filteredCustomLogConfigMasterVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-master-custom-log-config-0" && + .container == "vector" + filteredAutomaticLogConfigRegionserverHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-regionserver-automatic-log-config-0" && + .container == "opensearch" + filteredAutomaticLogConfigRegionserverVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-regionserver-automatic-log-config-0" && + .container == "vector" + filteredCustomLogConfigRegionserverHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-regionserver-custom-log-config-0" && + .container == "opensearch" + filteredCustomLogConfigRegionserverVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-regionserver-custom-log-config-0" && + .container == "vector" + filteredAutomaticLogConfigRestserverHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-restserver-automatic-log-config-0" && + .container == "opensearch" + filteredAutomaticLogConfigRestserverVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-restserver-automatic-log-config-0" && + .container == "vector" + filteredCustomLogConfigRestserverHbase: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-restserver-custom-log-config-0" && + .container == "opensearch" + filteredCustomLogConfigRestserverVector: + type: filter + inputs: [validEvents] + condition: >- + .pod == "test-opensearch-restserver-custom-log-config-0" && + .container == "vector" + filteredInvalidEvents: + type: filter + inputs: [vector] + condition: |- + .timestamp == from_unix_timestamp!(0) || + is_null(.level) || + is_null(.logger) || + is_null(.message) + sinks: + test: + inputs: [filtered*] + type: blackhole + stdout: + inputs: + - vector + type: console + encoding: + codec: json +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + aggregator: + inputs: [vector] + type: vector + address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} + buffer: + # Avoid back pressure from VECTOR_AGGREGATOR. The test should + # not fail if the aggregator is not available. + when_full: drop_newest +{% endif %} diff --git a/tests/templates/kuttl/logging/20-assert.yaml.j2 b/tests/templates/kuttl/logging/20-assert.yaml.j2 new file mode 100644 index 0000000..ca660d1 --- /dev/null +++ b/tests/templates/kuttl/logging/20-assert.yaml.j2 @@ -0,0 +1,670 @@ +# All fields are checked that are set by the operator. +# This helps to detect unintentional changes. It is also a good reference for the output of the +# operator. The maintenance effort should be okay as long as it is only done in the smoke test. +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-cluster-manager + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + serviceName: opensearch-nodes-cluster-manager-headless + template: + metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/opensearch-role.cluster_manager: "true" + stackable.tech/vendor: Stackable + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - command: + - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + env: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + - name: OPENSEARCH_HOME + value: {{ test_scenario['values']['opensearch_home'] }} + - name: cluster.initial_cluster_manager_nodes + value: opensearch-nodes-cluster-manager-0,opensearch-nodes-cluster-manager-1,opensearch-nodes-cluster-manager-2 + - name: discovery.seed_hosts + value: opensearch + - name: node.name + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: node.roles + value: cluster_manager + imagePullPolicy: IfNotPresent + name: opensearch + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + resources: + limits: + cpu: "4" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + startupProbe: + failureThreshold: 30 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + volumeMounts: + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch.yml + name: config + readOnly: true + subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data + name: data + - mountPath: /stackable/listener + name: listener + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + name: security-config + readOnly: true + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + name: tls + readOnly: true + securityContext: + fsGroup: 1000 + serviceAccount: opensearch-serviceaccount + serviceAccountName: opensearch-serviceaccount + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + defaultMode: 420 + name: opensearch-nodes-cluster-manager + name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-cluster-manager + name: log-config + - name: security-config + secret: + defaultMode: 420 + secretName: opensearch-security-config + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + annotations: + listeners.stackable.tech/listener-name: opensearch-nodes-cluster-manager + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: listener + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1" + storageClassName: listeners.stackable.tech + volumeMode: Filesystem + status: + phase: Pending +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-data + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +spec: + podManagementPolicy: Parallel + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + serviceName: opensearch-nodes-data-headless + template: + metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/opensearch-role.data: "true" + stackable.tech/opensearch-role.ingest: "true" + stackable.tech/opensearch-role.remote_cluster_client: "true" + stackable.tech/vendor: Stackable + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + topologyKey: kubernetes.io/hostname + weight: 1 + containers: + - command: + - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + env: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + - name: OPENSEARCH_HOME + value: {{ test_scenario['values']['opensearch_home'] }} + - name: cluster.initial_cluster_manager_nodes + - name: discovery.seed_hosts + value: opensearch + - name: node.name + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: node.roles + value: ingest,data,remote_cluster_client + imagePullPolicy: IfNotPresent + name: opensearch + ports: + - containerPort: 9200 + name: http + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + readinessProbe: + failureThreshold: 3 + periodSeconds: 5 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + resources: + limits: + cpu: "4" + memory: 2Gi + requests: + cpu: "1" + memory: 2Gi + startupProbe: + failureThreshold: 30 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 3 + volumeMounts: + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch.yml + name: config + readOnly: true + subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data + name: data + - mountPath: /stackable/listener + name: listener + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + name: security-config + readOnly: true + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + name: tls + readOnly: true + securityContext: + fsGroup: 1000 + serviceAccount: opensearch-serviceaccount + serviceAccountName: opensearch-serviceaccount + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + defaultMode: 420 + name: opensearch-nodes-data + name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-data + name: log-config + - name: security-config + secret: + defaultMode: 420 + secretName: opensearch-security-config + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + volumeMode: Filesystem + name: tls + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + volumeMode: Filesystem + status: + phase: Pending + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + annotations: + listeners.stackable.tech/listener-name: opensearch-nodes-data + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: listener + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1" + storageClassName: listeners.stackable.tech + volumeMode: Filesystem + status: + phase: Pending +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-cluster-manager + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +data: + opensearch.yml: |- + cluster.name: "opensearch" + cluster.routing.allocation.disk.threshold_enabled: "false" + discovery.type: "zen" + network.host: "0.0.0.0" + node.store.allow_mmap: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.nodes_dn: ["CN=generated certificate for pod"] + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" + plugins.security.ssl.http.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" + plugins.security.ssl.http.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" + plugins.security.ssl.transport.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" + plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-data + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +data: + opensearch.yml: |- + cluster.name: "opensearch" + cluster.routing.allocation.disk.threshold_enabled: "false" + discovery.type: "zen" + network.host: "0.0.0.0" + node.store.allow_mmap: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.nodes_dn: ["CN=generated certificate for pod"] + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" + plugins.security.ssl.http.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" + plugins.security.ssl.http.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" + plugins.security.ssl.transport.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" + plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-cluster-manager-headless +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-nodes-data-headless +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +spec: + ports: + - name: http + port: 9200 + protocol: TCP + targetPort: 9200 + - name: transport + port: 9300 + protocol: TCP + targetPort: 9300 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch + stackable.tech/opensearch-role.cluster_manager: "true" + type: ClusterIP +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-serviceaccount + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} + stackable.tech/vendor: Stackable + name: opensearch-rolebinding + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: opensearch-clusterrole +subjects: +- kind: ServiceAccount + name: opensearch-serviceaccount +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + name: opensearch-nodes +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/name: opensearch +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: Listener +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: cluster-manager + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-nodes-cluster-manager + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +spec: + className: external-stable + extraPodSelectorLabels: {} + ports: + - name: http + port: 9200 + protocol: TCP + publishNotReadyAddresses: null +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: Listener +metadata: + labels: + app.kubernetes.io/component: nodes + app.kubernetes.io/instance: opensearch + app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster + app.kubernetes.io/name: opensearch + app.kubernetes.io/role-group: data + app.kubernetes.io/version: 3.1.0 + stackable.tech/vendor: Stackable + name: opensearch-nodes-data + ownerReferences: + - apiVersion: opensearch.stackable.tech/v1alpha1 + controller: true + kind: OpenSearchCluster + name: opensearch +spec: + className: cluster-internal + extraPodSelectorLabels: {} + ports: + - name: http + port: 9200 + protocol: TCP + publishNotReadyAddresses: null diff --git a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 new file mode 100644 index 0000000..025da48 --- /dev/null +++ b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 @@ -0,0 +1,227 @@ +--- +apiVersion: opensearch.stackable.tech/v1alpha1 +kind: OpenSearchCluster +metadata: + name: opensearch +spec: + image: +{% if test_scenario['values']['opensearch'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opensearch'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opensearch'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opensearch'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + vectorAggregatorConfigMapName: opensearch-vector-aggregator-discovery + nodes: + config: + logging: + enableVectorAgent: true + containers: + opensearch: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + roleGroups: + cluster-manager: + config: + nodeRoles: + - cluster_manager + resources: + storage: + data: + capacity: 100Mi + listenerClass: external-stable + replicas: 3 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless + data: + config: + nodeRoles: + - ingest + - data + - remote_cluster_client + resources: + storage: + data: + capacity: 2Gi + listenerClass: cluster-internal + replicas: 2 + podOverrides: + spec: + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless + envOverrides: + # Only required for the official image + # The official image (built with https://github.com/opensearch-project/opensearch-build) + # installs a demo configuration if not disabled explicitly. + DISABLE_INSTALL_DEMO_CONFIG: "true" + OPENSEARCH_HOME: {{ test_scenario['values']['opensearch_home'] }} + configOverrides: + opensearch.yml: + # Disable memory mapping in this test; If memory mapping were activated, the kernel setting + # vm.max_map_count would have to be increased to 262144 on the node. + node.store.allow_mmap: "false" + # Disable the disk allocation decider in this test; Otherwise the test depends on the disk + # usage of the node and if the relative watermark set in + # `cluster.routing.allocation.disk.watermark.high` is reached then the security index could + # not be created even if enough disk space would be available. + cluster.routing.allocation.disk.threshold_enabled: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + podOverrides: + spec: + containers: + - name: opensearch + volumeMounts: + - name: security-config + mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + readOnly: true + - name: tls + mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + readOnly: true + volumes: + - name: security-config + secret: + secretName: opensearch-security-config + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" +--- +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-security-config +stringData: + action_groups.yml: | + --- + _meta: + type: actiongroups + config_version: 2 + allowlist.yml: | + --- + _meta: + type: allowlist + config_version: 2 + + config: + enabled: false + audit.yml: | + --- + _meta: + type: audit + config_version: 2 + + config: + enabled: false + config.yml: | + --- + _meta: + type: config + config_version: 2 + + config: + dynamic: + authc: + basic_internal_auth_domain: + description: Authenticate via HTTP Basic against internal users database + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + authz: {} + internal_users.yml: | + --- + # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + + _meta: + type: internalusers + config_version: 2 + + admin: + hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e + reserved: true + backend_roles: + - admin + description: OpenSearch admin user + + kibanaserver: + hash: $2y$10$vPgQ/6ilKDM5utawBqxoR.7euhVQ0qeGl8mPTeKhmFT475WUDrfQS + reserved: true + description: OpenSearch Dashboards user + nodes_dn.yml: | + --- + _meta: + type: nodesdn + config_version: 2 + roles.yml: | + --- + _meta: + type: roles + config_version: 2 + roles_mapping.yml: | + --- + _meta: + type: rolesmapping + config_version: 2 + + all_access: + reserved: false + backend_roles: + - admin + + kibana_server: + reserved: true + users: + - kibanaserver + tenants.yml: | + --- + _meta: + type: tenants + config_version: 2 diff --git a/tests/templates/kuttl/logging/30-assert.yaml b/tests/templates/kuttl/logging/30-assert.yaml new file mode 100644 index 0000000..bebbaa9 --- /dev/null +++ b/tests/templates/kuttl/logging/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-opensearch +status: + succeeded: 1 diff --git a/tests/templates/kuttl/logging/30-test-opensearch.yaml b/tests/templates/kuttl/logging/30-test-opensearch.yaml new file mode 100644 index 0000000..b7cfe4c --- /dev/null +++ b/tests/templates/kuttl/logging/30-test-opensearch.yaml @@ -0,0 +1,164 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-opensearch +spec: + template: + spec: + containers: + - name: test-opensearch + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - /bin/bash + - -euxo + - pipefail + - -c + args: + - | + pip install opensearch-py==3.0.0 + python scripts/test.py + env: + # required for pip install + - name: HOME + value: /stackable + volumeMounts: + - name: script + mountPath: /stackable/scripts + - name: tls + mountPath: /stackable/tls + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m + volumes: + - name: script + configMap: + name: test-opensearch + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-opensearch +data: + test.py: | + # https://docs.opensearch.org/docs/latest/clients/python-low-level/#sample-program + + from opensearchpy import OpenSearch + + host = 'opensearch' + port = 9200 + auth = ('admin', 'AJVFsGJBbpT6mChn') # For testing only. Don't store credentials in code. + ca_certs_path = '/stackable/tls/ca.crt' + + # Create the client with SSL/TLS enabled, but hostname verification disabled. + client = OpenSearch( + hosts = [{'host': host, 'port': port}], + http_compress = True, # enables gzip compression for request bodies + http_auth = auth, + use_ssl = True, + verify_certs = True, + ssl_assert_hostname = False, + ssl_show_warn = False, + ca_certs = ca_certs_path + ) + + # Create an index with non-default settings. + index_name = 'python-test-index' + index_body = { + 'settings': { + 'index': { + 'number_of_shards': 4 + } + } + } + + response = client.indices.create(index=index_name, body=index_body) + print('\nCreating index:') + print(response) + + # Add a document to the index. + document = { + 'title': 'Moneyball', + 'director': 'Bennett Miller', + 'year': '2011' + } + id = '1' + + response = client.index( + index = index_name, + body = document, + id = id, + refresh = True + ) + + print('\nAdding document:') + print(response) + + # Perform bulk operations + + movies = '{ "index" : { "_index" : "my-dsl-index", "_id" : "2" } } \n { "title" : "Interstellar", "director" : "Christopher Nolan", "year" : "2014"} \n { "create" : { "_index" : "my-dsl-index", "_id" : "3" } } \n { "title" : "Star Trek Beyond", "director" : "Justin Lin", "year" : "2015"} \n { "update" : {"_id" : "3", "_index" : "my-dsl-index" } } \n { "doc" : {"year" : "2016"} }' + + client.bulk(body=movies) + + # Search for the document. + q = 'miller' + query = { + 'size': 5, + 'query': { + 'multi_match': { + 'query': q, + 'fields': ['title^2', 'director'] + } + } + } + + response = client.search( + body = query, + index = index_name + ) + print('\nSearch results:') + print(response) + + # Delete the document. + response = client.delete( + index = index_name, + id = id + ) + + print('\nDeleting document:') + print(response) + + # Delete the index. + response = client.indices.delete( + index = index_name + ) + + print('\nDeleting index:') + print(response) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 729b588..ca660d1 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -111,6 +111,10 @@ spec: name: config readOnly: true subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data name: data - mountPath: /stackable/listener @@ -131,6 +135,10 @@ spec: defaultMode: 420 name: opensearch-nodes-cluster-manager name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-cluster-manager + name: log-config - name: security-config secret: defaultMode: 420 @@ -298,6 +306,10 @@ spec: name: config readOnly: true subPath: opensearch.yml + - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties + name: log-config + readOnly: true + subPath: log4j2.properties - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data name: data - mountPath: /stackable/listener @@ -318,6 +330,10 @@ spec: defaultMode: 420 name: opensearch-nodes-data name: config + - configMap: + defaultMode: 420 + name: opensearch-nodes-data + name: log-config - name: security-config secret: defaultMode: 420 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index efad126..f2dea4a 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -35,6 +35,11 @@ tests: - opensearch - openshift - opensearch_home + - name: logging + dimensions: + - opensearch + - openshift + - opensearch_home suites: - name: nightly patch: From 752136ed3330a7af90c3630794b4b33b8f2af815 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 8 Oct 2025 10:30:51 +0200 Subject: [PATCH 02/14] chore: Move product logging code into a separate module --- Cargo.lock | 32 +- Cargo.nix | 69 +++- Cargo.toml | 2 + .../helm/opensearch-operator/crds/crds.yaml | 199 +++++++++ rust/operator-binary/Cargo.toml | 2 + rust/operator-binary/src/controller/build.rs | 6 +- .../src/controller/build/node_config.rs | 282 +------------ .../controller/build/opensearch_server.vrl | 0 .../src/controller/build/product_logging.rs | 1 + .../build/product_logging/config.rs | 230 +++++++++++ .../{ => product_logging}/test-vector.sh | 2 +- .../{ => product_logging}/vector-test.yaml | 0 .../build/{ => product_logging}/vector.yaml | 3 +- .../src/controller/build/role_builder.rs | 6 +- .../controller/build/role_group_builder.rs | 383 ++++++++++++------ .../src/controller/validate.rs | 156 ++++--- rust/operator-binary/src/framework.rs | 36 +- .../src/framework/builder/pod/container.rs | 48 +-- .../framework/product_logging/framework.rs | 103 +++-- .../src/framework/validation.rs | 56 +++ 20 files changed, 1076 insertions(+), 540 deletions(-) delete mode 100644 rust/operator-binary/src/controller/build/opensearch_server.vrl create mode 100644 rust/operator-binary/src/controller/build/product_logging.rs create mode 100644 rust/operator-binary/src/controller/build/product_logging/config.rs rename rust/operator-binary/src/controller/build/{ => product_logging}/test-vector.sh (85%) rename rust/operator-binary/src/controller/build/{ => product_logging}/vector-test.yaml (100%) rename rust/operator-binary/src/controller/build/{ => product_logging}/vector.yaml (99%) create mode 100644 rust/operator-binary/src/framework/validation.rs diff --git a/Cargo.lock b/Cargo.lock index b5523af..2542309 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -553,6 +553,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1904,6 +1910,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -2036,9 +2052,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -2048,9 +2064,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -2535,6 +2551,8 @@ dependencies = [ "built", "clap", "futures 0.3.31", + "pretty_assertions", + "regex", "rstest", "schemars 1.0.4", "serde", @@ -3593,6 +3611,12 @@ version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fd8403733700263c6eb89f192880191f1b83e332f7a20371ddcf421c4a337c7" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.0" diff --git a/Cargo.nix b/Cargo.nix index 1088959..72263d2 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1736,6 +1736,16 @@ rec { }; resolvedDefaultFeatures = [ "default" "from" ]; }; + "diff" = rec { + crateName = "diff"; + version = "0.1.13"; + edition = "2015"; + sha256 = "1j0nzjxci2zqx63hdcihkp0a4dkdmzxd7my4m7zk6cjyfy34j9an"; + authors = [ + "Utkarsh Kukreti " + ]; + + }; "digest" = rec { crateName = "digest"; version = "0.10.7"; @@ -6180,6 +6190,31 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "pretty_assertions" = rec { + crateName = "pretty_assertions"; + version = "1.4.1"; + edition = "2018"; + sha256 = "0v8iq35ca4rw3rza5is3wjxwsf88303ivys07anc5yviybi31q9s"; + authors = [ + "Colin Kiegel " + "Florent Fayolle " + "Tom Milligan " + ]; + dependencies = [ + { + name = "diff"; + packageId = "diff"; + } + { + name = "yansi"; + packageId = "yansi"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "proc-macro-crate" = rec { crateName = "proc-macro-crate"; version = "3.4.0"; @@ -6536,9 +6571,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.11.2"; + version = "1.11.3"; edition = "2021"; - sha256 = "04k9rzxd11hcahpyihlswy6f1zqw7lspirv4imm4h0lcdl8gvmr3"; + sha256 = "0b58ya98c4i5cjjiwhpcnjr61cv9g143qhdwhsryggj09098hllb"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -6594,9 +6629,9 @@ rec { }; "regex-automata" = rec { crateName = "regex-automata"; - version = "0.4.10"; + version = "0.4.11"; edition = "2021"; - sha256 = "1mllcfmgjcl6d52d5k09lwwq9wj5mwxccix4bhmw5spy1gx5i53b"; + sha256 = "1bawj908pxixpggcnma3xazw53mwyz68lv9hn4yg63nlhv7bjgl3"; libName = "regex_automata"; authors = [ "The Rust Project Developers" @@ -8239,6 +8274,10 @@ rec { packageId = "futures 0.3.31"; features = [ "compat" ]; } + { + name = "regex"; + packageId = "regex"; + } { name = "schemars"; packageId = "schemars 1.0.4"; @@ -8289,6 +8328,10 @@ rec { } ]; devDependencies = [ + { + name = "pretty_assertions"; + packageId = "pretty_assertions"; + } { name = "rstest"; packageId = "rstest"; @@ -13023,6 +13066,24 @@ rec { ]; }; + "yansi" = rec { + crateName = "yansi"; + version = "1.0.1"; + edition = "2021"; + sha256 = "0jdh55jyv0dpd38ij4qh60zglbw9aa8wafqai6m0wa7xaxk3mrfg"; + authors = [ + "Sergio Benitez " + ]; + features = { + "default" = [ "std" ]; + "detect-env" = [ "std" ]; + "detect-tty" = [ "is-terminal" "std" ]; + "hyperlink" = [ "std" ]; + "is-terminal" = [ "dep:is-terminal" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "yoke" = rec { crateName = "yoke"; version = "0.8.0"; diff --git a/Cargo.toml b/Cargo.toml index 3075a44..9e8641f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", built = { version = "0.8.0", features = ["chrono", "git2"] } clap = "4.5" futures = { version = "0.3", features = ["compat"] } +pretty_assertions = "1.4" +regex = "1.11.3" rstest = "0.26" schemars = { version = "1.0.0", features = ["url2"] } # same as in operator-rs serde = { version = "1.0", features = ["derive"] } diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index 3443dda..e48aac2 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -28,6 +28,19 @@ spec: OpenSearch. Find more information on how to use it and the resources that the operator generates in the [operator documentation](https://docs.stackable.tech/home/nightly/opensearch/). properties: + clusterConfig: + default: {} + description: Configuration that applies to all roles and role groups + properties: + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + nullable: true + type: string + type: object clusterOperation: default: reconciliationPaused: false @@ -172,6 +185,99 @@ spec: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object nodeRoles: description: |- Roles of the OpenSearch node. @@ -422,6 +528,99 @@ spec: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object nodeRoles: description: |- Roles of the OpenSearch node. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index c5a9213..e93fbdc 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -14,6 +14,7 @@ stackable-operator.workspace = true clap.workspace = true futures.workspace = true +regex.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true @@ -27,4 +28,5 @@ uuid.workspace = true built.workspace = true [dev-dependencies] +pretty_assertions.workspace = true rstest.workspace = true diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index dfab1ef..768fdcf 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -7,6 +7,7 @@ use role_builder::RoleBuilder; use super::{ContextNames, KubernetesResources, Prepared, ValidatedCluster}; pub mod node_config; +pub mod product_logging; pub mod role_builder; pub mod role_group_builder; @@ -203,13 +204,10 @@ mod tests { affinity: StackableAffinity::default(), listener_class: "external-stable".to_owned(), logging: ValidatedLogging { - vector_aggregator_config_map_name: None, opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), ), - vector_container: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), + vector_container: None, }, node_roles: NodeRoles(node_roles.to_vec()), resources: OpenSearchNodeResources::default(), diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 06e5b51..56d5800 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -1,20 +1,17 @@ //! Configuration of an OpenSearch node -use std::{cmp, str::FromStr}; +use std::str::FromStr; use serde_json::{Value, json}; -use stackable_operator::{ - builder::pod::container::FieldPathEnvVar, product_logging::spec::AutomaticContainerLogConfig, -}; +use stackable_operator::builder::pod::container::FieldPathEnvVar; use super::ValidatedCluster; use crate::{ controller::OpenSearchRoleGroupConfig, crd::v1alpha1, framework::{ - ConfigMapName, ServiceName, + ServiceName, builder::pod::container::{EnvVarName, EnvVarSet}, - product_logging::framework::ValidatedContainerLogConfigChoice, role_group_utils, }, }; @@ -22,9 +19,6 @@ use crate::{ /// The main configuration file of OpenSearch pub const CONFIGURATION_FILE_OPENSEARCH_YML: &str = "opensearch.yml"; -/// The log configuration file -pub const CONFIGURATION_FILE_LOG4J2_PROPERTIES: &str = "log4j2.properties"; - /// The cluster name. /// Type: string pub const CONFIG_OPTION_CLUSTER_NAME: &str = "cluster.name"; @@ -64,15 +58,6 @@ pub const CONFIG_OPTION_PLUGINS_SECURITY_NODES_DN: &str = "plugins.security.node pub const CONFIG_OPTION_PLUGINS_SECURITY_SSL_HTTP_ENABLED: &str = "plugins.security.ssl.http.enabled"; -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -const OPENSEARCH_SERVER_LOG_FILE: &str = "opensearch_server.json"; -/// File name of the Vector config file -pub const VECTOR_CONFIG_FILE: &str = "vector.json"; -// /// Key in the discovery ConfigMap that holds the vector aggregator address -// const VECTOR_AGGREGATOR_CM_KEY: &str = "ADDRESS"; -// /// Name of the env var in the vector container that holds the vector aggregator address -// const VECTOR_AGGREGATOR_ENV_NAME: &str = "VECTOR_AGGREGATOR_ADDRESS"; - /// Configuration of an OpenSearch node based on the cluster and role-group configuration pub struct NodeConfig { cluster: ValidatedCluster, @@ -171,19 +156,19 @@ impl NodeConfig { // Set the OpenSearch node name to the Pod name. // The node name is used e.g. for INITIAL_CLUSTER_MANAGER_NODES. .with_field_path( - EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_NAME), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_NAME), FieldPathEnvVar::Name, ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_DISCOVERY_SEED_HOSTS), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_DISCOVERY_SEED_HOSTS), &self.discovery_service_name, ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_INITIAL_CLUSTER_MANAGER_NODES), self.initial_cluster_manager_nodes(), ) .with_value( - EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_ROLES), + &EnvVarName::from_str_unsafe(CONFIG_OPTION_NODE_ROLES), self.role_group_config .config .node_roles @@ -276,162 +261,6 @@ impl NodeConfig { String::new() } } - - pub fn automatic_log_config_file_content(&self) -> Option { - if let ValidatedContainerLogConfigChoice::Automatic(log_config) = - &self.role_group_config.config.logging.opensearch_container - { - Some(NodeConfig::create_log4j2_config(log_config)) - } else { - None - } - } - - fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { - let log_path = format!( - "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", - container = v1alpha1::Container::OpenSearch.to_container_name() - ); - // TODO Calculate or move to constants - let max_size_in_mib = 10; - let number_of_archived_log_files = 1; - - let loggers = config - .loggers - .iter() - .filter(|(name, _)| name.as_str() != AutomaticContainerLogConfig::ROOT_LOGGER) - .flat_map(|(name, logger_config)| { - [ - ( - format!("logger.{name}.name"), - name.escape_default().to_string(), - ), - ( - format!("logger.{name}.level"), - logger_config.level.to_log4j_literal(), - ), - ] - }) - .collect::>(); - - let root_logger = vec![ - ( - "rootLogger.level".to_owned(), - config.root_log_level().to_log4j2_literal(), - ), - ( - "rootLogger.appenderRef.CONSOLE.ref".to_owned(), - "CONSOLE".to_owned(), - ), - ( - "rootLogger.appenderRef.FILE.ref".to_owned(), - "FILE".to_owned(), - ), - ]; - - let console_appender = vec![ - ("appender.CONSOLE.type".to_owned(), "Console".to_owned()), - ("appender.CONSOLE.name".to_owned(), "CONSOLE".to_owned()), - ( - "appender.CONSOLE.target".to_owned(), - "SYSTEM_ERR".to_owned(), - ), - ( - "appender.CONSOLE.layout.type".to_owned(), - "PatternLayout".to_owned(), - ), - // Same as the default layout pattern of the console appender - // see https://github.com/opensearch-project/OpenSearch/blob/3.1.0/distribution/src/config/log4j2.properties#L17 - ( - "appender.CONSOLE.layout.pattern".to_owned(), - "[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n".to_owned(), - ), - ( - "appender.CONSOLE.filter.threshold.type".to_owned(), - "ThresholdFilter".to_owned(), - ), - ( - "appender.CONSOLE.filter.threshold.level".to_owned(), - config - .console - .as_ref() - .and_then(|console| console.level) - .unwrap_or_default() - .to_log4j2_literal(), - ), - ]; - - let file_appender = vec![ - ("appender.FILE.type".to_owned(), "RollingFile".to_owned()), - ("appender.FILE.name".to_owned(), "FILE".to_owned()), - ("appender.FILE.fileName".to_owned(), log_path.to_owned()), - ( - "appender.FILE.filePattern".to_owned(), - format!("{log_path}.%i"), - ), - ( - "appender.FILE.layout.type".to_owned(), - "OpenSearchJsonLayout".to_owned(), - ), - ( - "appender.FILE.layout.type_name".to_owned(), - "server".to_owned(), - ), - ( - "appender.FILE.policies.type".to_owned(), - "Policies".to_owned(), - ), - ( - "appender.FILE.policies.size.type".to_owned(), - "SizeBasedTriggeringPolicy".to_owned(), - ), - ( - "appender.FILE.policies.size.size".to_owned(), - format!( - "{max_log_file_size_in_mib}MB", - max_log_file_size_in_mib = - cmp::max(1, max_size_in_mib / (1 + number_of_archived_log_files)), - ), - ), - ( - "appender.FILE.strategy.type".to_owned(), - "DefaultRolloverStrategy".to_owned(), - ), - ( - "appender.FILE.strategy.max".to_owned(), - number_of_archived_log_files.to_string(), - ), - ( - "appender.FILE.filter.threshold.type".to_owned(), - "ThresholdFilter".to_owned(), - ), - ( - "appender.FILE.filter.threshold.level".to_owned(), - config - .file - .as_ref() - .and_then(|file| file.level) - .unwrap_or_default() - .to_log4j2_literal(), - ), - ]; - - [root_logger, loggers, console_appender, file_appender] - .iter() - .flatten() - .map(|(key, value)| format!("{key} = {value}\n")) - .collect() - } - - pub fn custom_log_config_map(&self) -> Option { - if let ValidatedContainerLogConfigChoice::Custom(config_map_name) = - &self.role_group_config.config.logging.opensearch_container - { - Some(config_map_name.clone()) - } else { - None - } - } } #[cfg(test)] @@ -457,6 +286,7 @@ mod tests { crd::NodeRoles, framework::{ ClusterName, NamespaceName, ProductVersion, RoleGroupName, + product_logging::framework::ValidatedContainerLogConfigChoice, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -465,7 +295,6 @@ mod tests { replicas: u16, config_settings: &'static [(&'static str, &'static str)], env_vars: &'static [(&'static str, &'static str)], - log_config: ValidatedContainerLogConfigChoice, } impl Default for TestConfig { @@ -474,9 +303,6 @@ mod tests { replicas: 3, config_settings: &[], env_vars: &[], - log_config: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), } } } @@ -491,11 +317,10 @@ mod tests { affinity: StackableAffinity::default(), listener_class: "cluster-internal".to_string(), logging: ValidatedLogging { - vector_aggregator_config_map_name: None, - opensearch_container: test_config.log_config, - vector_container: ValidatedContainerLogConfigChoice::Automatic( + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), ), + vector_container: None, }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, @@ -640,21 +465,21 @@ mod tests { assert_eq!( EnvVarSet::new() - .with_value(EnvVarName::from_str_unsafe("TEST"), "value",) + .with_value(&EnvVarName::from_str_unsafe("TEST"), "value") .with_value( - EnvVarName::from_str_unsafe("cluster.initial_cluster_manager_nodes"), + &EnvVarName::from_str_unsafe("cluster.initial_cluster_manager_nodes"), "my-opensearch-cluster-nodes-default-0,my-opensearch-cluster-nodes-default-1", ) .with_value( - EnvVarName::from_str_unsafe("discovery.seed_hosts"), + &EnvVarName::from_str_unsafe("discovery.seed_hosts"), "my-opensearch-cluster-manager", ) .with_field_path( - EnvVarName::from_str_unsafe("node.name"), + &EnvVarName::from_str_unsafe("node.name"), FieldPathEnvVar::Name ) .with_value( - EnvVarName::from_str_unsafe("node.roles"), + &EnvVarName::from_str_unsafe("node.roles"), "cluster_manager,data,ingest,remote_cluster_client" ), node_config.environment_variables() @@ -709,81 +534,4 @@ mod tests { node_config_multiple_nodes.initial_cluster_manager_nodes() ); } - - #[test] - pub fn test_automatic_log_config_file_content() { - let automatic_log_config_node_config = node_config(TestConfig { - log_config: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), - ..TestConfig::default() - }); - - let custom_log_config_node_config = node_config(TestConfig { - log_config: ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( - "custom-log-config", - )), - ..TestConfig::default() - }); - - assert_eq!( - Some(concat!( - "appenders = FILE, CONSOLE\n\n", - "appender.CONSOLE.type = Console\n", - "appender.CONSOLE.name = CONSOLE\n", - "appender.CONSOLE.target = SYSTEM_ERR\n", - "appender.CONSOLE.layout.type = PatternLayout\n", - "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", - "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", - "appender.CONSOLE.filter.threshold.level = INFO\n\n", - "appender.FILE.type = RollingFile\n", - "appender.FILE.name = FILE\n", - "appender.FILE.fileName = /stackable/log/opensearch/opensearch.log4j2.xml\n", - "appender.FILE.filePattern = /stackable/log/opensearch/opensearch.log4j2.xml.%i\n", - "appender.FILE.layout.type = XMLLayout\n", - "appender.FILE.policies.type = Policies\n", - "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", - "appender.FILE.policies.size.size = 5MB\n", - "appender.FILE.strategy.type = DefaultRolloverStrategy\n", - "appender.FILE.strategy.max = 1\n", - "appender.FILE.filter.threshold.type = ThresholdFilter\n", - "appender.FILE.filter.threshold.level = INFO\n\n\n", - "rootLogger.level=INFO\n", - "rootLogger.appenderRefs = CONSOLE, FILE\n", - "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", - "rootLogger.appenderRef.FILE.ref = FILE" - ).to_owned()), - automatic_log_config_node_config.automatic_log_config_file_content() - ); - assert_eq!( - None, - custom_log_config_node_config.automatic_log_config_file_content() - ); - } - - #[test] - pub fn test_custom_log_config_map() { - let custom_log_config_node_config = node_config(TestConfig { - log_config: ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( - "custom-log-config", - )), - ..TestConfig::default() - }); - - let automatic_log_config_node_config = node_config(TestConfig { - log_config: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), - ..TestConfig::default() - }); - - assert_eq!( - Some(ConfigMapName::from_str_unsafe("custom-log-config")), - custom_log_config_node_config.custom_log_config_map() - ); - assert_eq!( - None, - automatic_log_config_node_config.custom_log_config_map() - ); - } } diff --git a/rust/operator-binary/src/controller/build/opensearch_server.vrl b/rust/operator-binary/src/controller/build/opensearch_server.vrl deleted file mode 100644 index e69de29..0000000 diff --git a/rust/operator-binary/src/controller/build/product_logging.rs b/rust/operator-binary/src/controller/build/product_logging.rs new file mode 100644 index 0000000..ef68c36 --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging.rs @@ -0,0 +1 @@ +pub mod config; diff --git a/rust/operator-binary/src/controller/build/product_logging/config.rs b/rust/operator-binary/src/controller/build/product_logging/config.rs new file mode 100644 index 0000000..2877ab1 --- /dev/null +++ b/rust/operator-binary/src/controller/build/product_logging/config.rs @@ -0,0 +1,230 @@ +use std::cmp; + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::spec::AutomaticContainerLogConfig, +}; + +use crate::{crd::v1alpha1, framework::product_logging::framework::STACKABLE_LOG_DIR}; + +/// The log configuration file +pub const CONFIGURATION_FILE_LOG4J2_PROPERTIES: &str = "log4j2.properties"; + +const OPENSEARCH_SERVER_LOG_FILE: &str = "opensearch_server.json"; + +pub const MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { + let log_path = format!( + "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", + container = v1alpha1::Container::OpenSearch.to_container_name() + ); + + let number_of_archived_log_files = 1; + let max_log_files_size_in_mib = MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32; + let max_log_file_size_in_mib = cmp::max( + 1, + max_log_files_size_in_mib / (1 + number_of_archived_log_files), + ); + + let loggers = config + .loggers + .iter() + .filter(|(name, _)| name.as_str() != AutomaticContainerLogConfig::ROOT_LOGGER) + .enumerate() + .flat_map(|(index, (name, logger_config))| { + [ + ( + format!("logger.{index}.name"), + name.escape_default().to_string(), + ), + ( + format!("logger.{index}.level"), + logger_config.level.to_log4j_literal(), + ), + ] + }) + .collect::>(); + + let root_logger = vec![ + ( + "rootLogger.level".to_owned(), + config.root_log_level().to_log4j2_literal(), + ), + ( + "rootLogger.appenderRef.CONSOLE.ref".to_owned(), + "CONSOLE".to_owned(), + ), + ( + "rootLogger.appenderRef.FILE.ref".to_owned(), + "FILE".to_owned(), + ), + ]; + + let console_appender = vec![ + ("appender.CONSOLE.type".to_owned(), "Console".to_owned()), + ("appender.CONSOLE.name".to_owned(), "CONSOLE".to_owned()), + ( + "appender.CONSOLE.target".to_owned(), + "SYSTEM_ERR".to_owned(), + ), + ( + "appender.CONSOLE.layout.type".to_owned(), + "PatternLayout".to_owned(), + ), + // Same as the default layout pattern of the console appender + // see https://github.com/opensearch-project/OpenSearch/blob/3.1.0/distribution/src/config/log4j2.properties#L17 + ( + "appender.CONSOLE.layout.pattern".to_owned(), + "[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.CONSOLE.filter.threshold.level".to_owned(), + config + .console + .as_ref() + .and_then(|console| console.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ]; + + let file_appender = vec![ + ("appender.FILE.type".to_owned(), "RollingFile".to_owned()), + ("appender.FILE.name".to_owned(), "FILE".to_owned()), + ("appender.FILE.fileName".to_owned(), log_path.to_owned()), + ( + "appender.FILE.filePattern".to_owned(), + format!("{log_path}.%i"), + ), + ( + "appender.FILE.layout.type".to_owned(), + "OpenSearchJsonLayout".to_owned(), + ), + ( + "appender.FILE.layout.type_name".to_owned(), + "server".to_owned(), + ), + ( + "appender.FILE.policies.type".to_owned(), + "Policies".to_owned(), + ), + ( + "appender.FILE.policies.size.type".to_owned(), + "SizeBasedTriggeringPolicy".to_owned(), + ), + ( + "appender.FILE.policies.size.size".to_owned(), + format!("{max_log_file_size_in_mib}MB"), + ), + ( + "appender.FILE.strategy.type".to_owned(), + "DefaultRolloverStrategy".to_owned(), + ), + ( + "appender.FILE.strategy.max".to_owned(), + number_of_archived_log_files.to_string(), + ), + ( + "appender.FILE.filter.threshold.type".to_owned(), + "ThresholdFilter".to_owned(), + ), + ( + "appender.FILE.filter.threshold.level".to_owned(), + config + .file + .as_ref() + .and_then(|file| file.level) + .unwrap_or_default() + .to_log4j2_literal(), + ), + ]; + + [root_logger, loggers, console_appender, file_appender] + .iter() + .flatten() + .map(|(key, value)| format!("{key} = {value}\n")) + .collect() +} + +pub fn vector_config_file_content() -> String { + include_str!("vector.yaml").to_owned() +} + +#[cfg(test)] +mod tests { + use stackable_operator::product_logging::spec::{ + AppenderConfig, AutomaticContainerLogConfig, LogLevel, LoggerConfig, + }; + + use super::create_log4j2_config; + + #[test] + pub fn test_create_log4j2_config() { + let log4j2_config = create_log4j2_config(&AutomaticContainerLogConfig { + loggers: [ + ( + "org.opensearch.index.reindex".to_owned(), + LoggerConfig { + level: LogLevel::DEBUG, + }, + ), + ( + "org.opensearch.indices.recovery".to_owned(), + LoggerConfig { + level: LogLevel::TRACE, + }, + ), + ] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::WARN), + }), + file: Some(AppenderConfig { + level: Some(LogLevel::DEBUG), + }), + }); + + let expected_config = concat!( + "rootLogger.level = INFO\n", + "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", + "rootLogger.appenderRef.FILE.ref = FILE\n", + "logger.0.name = org.opensearch.index.reindex\n", + "logger.0.level = DEBUG\n", + "logger.1.name = org.opensearch.indices.recovery\n", + "logger.1.level = TRACE\n", + "appender.CONSOLE.type = Console\n", + "appender.CONSOLE.name = CONSOLE\n", + "appender.CONSOLE.target = SYSTEM_ERR\n", + "appender.CONSOLE.layout.type = PatternLayout\n", + "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", + "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", + "appender.CONSOLE.filter.threshold.level = WARN\n", + "appender.FILE.type = RollingFile\n", + "appender.FILE.name = FILE\n", + "appender.FILE.fileName = /stackable/log/opensearch/opensearch_server.json\n", + "appender.FILE.filePattern = /stackable/log/opensearch/opensearch_server.json.%i\n", + "appender.FILE.layout.type = OpenSearchJsonLayout\n", + "appender.FILE.layout.type_name = server\n", + "appender.FILE.policies.type = Policies\n", + "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", + "appender.FILE.policies.size.size = 5MB\n", + "appender.FILE.strategy.type = DefaultRolloverStrategy\n", + "appender.FILE.strategy.max = 1\n", + "appender.FILE.filter.threshold.type = ThresholdFilter\n", + "appender.FILE.filter.threshold.level = DEBUG\n", + ).to_owned(); + + assert_eq!(expected_config, log4j2_config); + } +} diff --git a/rust/operator-binary/src/controller/build/test-vector.sh b/rust/operator-binary/src/controller/build/product_logging/test-vector.sh similarity index 85% rename from rust/operator-binary/src/controller/build/test-vector.sh rename to rust/operator-binary/src/controller/build/product_logging/test-vector.sh index 19be25c..cf43eab 100755 --- a/rust/operator-binary/src/controller/build/test-vector.sh +++ b/rust/operator-binary/src/controller/build/product_logging/test-vector.sh @@ -6,6 +6,6 @@ NAMESPACE=default \ CLUSTER_NAME=opensearch \ ROLE_NAME=nodes \ ROLE_GROUP_NAME=cluster-manager \ -VECTOR_AGGREGATOR=vector-aggregator \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ VECTOR_FILE_LOG_LEVEL=info \ vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/vector-test.yaml b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml similarity index 100% rename from rust/operator-binary/src/controller/build/vector-test.yaml rename to rust/operator-binary/src/controller/build/product_logging/vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/vector.yaml b/rust/operator-binary/src/controller/build/product_logging/vector.yaml similarity index 99% rename from rust/operator-binary/src/controller/build/vector.yaml rename to rust/operator-binary/src/controller/build/product_logging/vector.yaml index 8233111..dc33be7 100644 --- a/rust/operator-binary/src/controller/build/vector.yaml +++ b/rust/operator-binary/src/controller/build/product_logging/vector.yaml @@ -138,8 +138,7 @@ sinks: inputs: - extended_logs type: vector - # TODO - address: ${VECTOR_AGGREGATOR} + address: ${VECTOR_AGGREGATOR_ADDRESS} console: inputs: - vector diff --git a/rust/operator-binary/src/controller/build/role_builder.rs b/rust/operator-binary/src/controller/build/role_builder.rs index 166a0a3..fb23199 100644 --- a/rust/operator-binary/src/controller/build/role_builder.rs +++ b/rust/operator-binary/src/controller/build/role_builder.rs @@ -218,6 +218,7 @@ mod tests { str::FromStr, }; + use pretty_assertions::assert_eq; use serde_json::json; use stackable_operator::{ commons::{ @@ -264,13 +265,10 @@ mod tests { affinity: StackableAffinity::default(), listener_class: "cluster-internal".to_string(), logging: ValidatedLogging { - vector_aggregator_config_map_name: None, opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), ), - vector_container: ValidatedContainerLogConfigChoice::Automatic( - AutomaticContainerLogConfig::default(), - ), + vector_container: None, }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 02e09de..42e8a4f 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -19,19 +19,25 @@ use stackable_operator::{ apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kvp::{Annotations, Label, Labels}, - memory::MemoryQuantity, product_logging::framework::{ - create_vector_shutdown_file_command, remove_vector_shutdown_file_command, + VECTOR_CONFIG_FILE, calculate_log_volume_size_limit, create_vector_shutdown_file_command, + remove_vector_shutdown_file_command, }, utils::COMMON_BASH_TRAP_FUNCTIONS, }; -use super::node_config::{ - CONFIGURATION_FILE_LOG4J2_PROPERTIES, CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig, - STACKABLE_LOG_DIR, VECTOR_CONFIG_FILE, +use super::{ + node_config::{CONFIGURATION_FILE_OPENSEARCH_YML, NodeConfig}, + product_logging::config::{ + CONFIGURATION_FILE_LOG4J2_PROPERTIES, create_log4j2_config, vector_config_file_content, + }, }; use crate::{ - controller::{ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster}, + constant, + controller::{ + ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, + build::product_logging::config::MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, + }, crd::v1alpha1, framework::{ PersistentVolumeClaimName, RoleGroupName, ServiceAccountName, ServiceName, VolumeName, @@ -43,7 +49,9 @@ use crate::{ }, }, kvp::label::{recommended_labels, role_group_selector, role_selector}, - product_logging::framework::vector_container, + product_logging::framework::{ + STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, + }, role_group_utils::ResourceNames, }, }; @@ -53,39 +61,19 @@ pub const HTTP_PORT: u16 = 9200; pub const TRANSPORT_PORT_NAME: &str = "transport"; pub const TRANSPORT_PORT: u16 = 9300; -const CONFIG_VOLUME_NAME: &str = "config"; -const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; -const DATA_VOLUME_NAME: &str = "data"; +constant!(CONFIG_VOLUME_NAME: VolumeName = "config"); -const LISTENER_VOLUME_NAME: &str = "listener"; +constant!(LOG_CONFIG_VOLUME_NAME: VolumeName = "log-config"); +constant!(DATA_VOLUME_NAME: VolumeName = "data"); + +constant!(LISTENER_VOLUME_NAME: PersistentVolumeClaimName = "listener"); const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; -const LOG_VOLUME_NAME: &str = "log"; +constant!(LOG_VOLUME_NAME: VolumeName = "log"); const LOG_VOLUME_DIR: &str = "/stackable/log"; const DEFAULT_OPENSEARCH_HOME: &str = "/stackable/opensearch"; -fn config_volume_name() -> VolumeName { - VolumeName::from_str(CONFIG_VOLUME_NAME).expect("should be a valid Volume name") -} - -fn log_config_volume_name() -> VolumeName { - VolumeName::from_str(LOG_CONFIG_VOLUME_NAME).expect("should be a valid Volume name") -} - -fn data_volume_name() -> VolumeName { - VolumeName::from_str(DATA_VOLUME_NAME).expect("should be a valid Volume name") -} - -fn listener_volume_name() -> PersistentVolumeClaimName { - PersistentVolumeClaimName::from_str(LISTENER_VOLUME_NAME) - .expect("should be a valid PersistentVolumeClaim name") -} - -fn log_volume_name() -> VolumeName { - VolumeName::from_str(LOG_VOLUME_NAME).expect("should be a valid Volume name") -} - /// Builder for role-group resources pub struct RoleGroupBuilder<'a> { service_account_name: ServiceAccountName, @@ -139,13 +127,14 @@ impl<'a> RoleGroupBuilder<'a> { self.node_config.static_opensearch_config_file_content(), ); - if let Some(log_config_file_content) = self.node_config.automatic_log_config_file_content() + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &self.role_group_config.config.logging.opensearch_container { data.insert( CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned(), - log_config_file_content, + create_log4j2_config(log_config), ); - } + }; if self .role_group_config @@ -153,10 +142,7 @@ impl<'a> RoleGroupBuilder<'a> { .logging .is_vector_agent_enabled() { - data.insert( - VECTOR_CONFIG_FILE.to_owned(), - include_str!("vector.yaml").to_owned(), - ); + data.insert(VECTOR_CONFIG_FILE.to_owned(), vector_config_file_content()); } ConfigMap { @@ -180,7 +166,7 @@ impl<'a> RoleGroupBuilder<'a> { .resources .storage .data - .build_pvc(data_volume_name().as_ref(), Some(vec!["ReadWriteOnce"])); + .build_pvc(DATA_VOLUME_NAME.as_ref(), Some(vec!["ReadWriteOnce"])); let listener_group_name = self.resource_names.listener_name(); @@ -192,7 +178,7 @@ impl<'a> RoleGroupBuilder<'a> { let listener_volume_claim_template = listener_operator_volume_source_builder_build_pvc( &ListenerReference::Listener(listener_group_name), &self.recommended_labels(), - &listener_volume_name(), + &LISTENER_VOLUME_NAME, ); let pvcs: Option> = Some(vec![ @@ -234,26 +220,37 @@ impl<'a> RoleGroupBuilder<'a> { .build(); let opensearch_container = self.build_opensearch_container(); - let vector_container = if let Some(vector_container_log_config) = - &self.role_group_config.config.logging.vector_container - { - vector_container( - &v1alpha1::Container::Vector.to_container_name(), - vector_container_log_config, - &self.resource_names.cluster_name, - &self.resource_names.role_name, - &self.resource_names.role_group_name, - &self.cluster.image, - &log_config_volume_name(), - &log_volume_name(), - ) - } else { - None - }; + let vector_container = self + .role_group_config + .config + .logging + .vector_container + .as_ref() + .map(|vector_container_log_config| { + vector_container( + &v1alpha1::Container::Vector.to_container_name(), + vector_container_log_config, + &self.resource_names.cluster_name, + &self.resource_names.role_name, + &self.resource_names.role_group_name, + &self.cluster.image, + &LOG_CONFIG_VOLUME_NAME, + &LOG_VOLUME_NAME, + ) + }); + + let log_config_volume_config_map = + if let ValidatedContainerLogConfigChoice::Custom(config_map_name) = + &self.role_group_config.config.logging.opensearch_container + { + config_map_name.clone() + } else { + self.resource_names.role_group_config_map() + }; let volumes = vec![ Volume { - name: config_volume_name().to_string(), + name: CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: self.resource_names.role_group_config_map().to_string(), ..Default::default() @@ -261,21 +258,19 @@ impl<'a> RoleGroupBuilder<'a> { ..Volume::default() }, Volume { - name: log_config_volume_name().to_string(), + name: LOG_CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { - name: self - .node_config - .custom_log_config_map() - .unwrap_or_else(|| self.resource_names.role_group_config_map()) - .to_string(), + name: log_config_volume_config_map.to_string(), ..Default::default() }), ..Volume::default() }, Volume { - name: log_volume_name().to_string(), + name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { - size_limit: Some(MemoryQuantity::from_mebi(100.0).into()), + size_limit: Some(calculate_log_volume_size_limit(&[ + MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, + ])), ..EmptyDirVolumeSource::default() }), ..Volume::default() @@ -392,20 +387,20 @@ impl<'a> RoleGroupBuilder<'a> { // Use `OPENSEARCH_HOME` from envOverrides or default to `DEFAULT_OPENSEARCH_HOME`. let opensearch_home = env_vars - .get(EnvVarName::from_str_unsafe("OPENSEARCH_HOME")) + .get(&EnvVarName::from_str_unsafe("OPENSEARCH_HOME")) .and_then(|env_var| env_var.value.clone()) .unwrap_or(DEFAULT_OPENSEARCH_HOME.to_owned()); // Use `OPENSEARCH_PATH_CONF` from envOverrides or default to `OPENSEARCH_HOME/config`, // i.e. depend on `OPENSEARCH_HOME`. let opensearch_path_conf = env_vars - .get(EnvVarName::from_str_unsafe("OPENSEARCH_PATH_CONF")) + .get(&EnvVarName::from_str_unsafe("OPENSEARCH_PATH_CONF")) .and_then(|env_var| env_var.value.clone()) .unwrap_or(format!("{opensearch_home}/config")); let volume_mounts = [ VolumeMount { mount_path: format!("{opensearch_path_conf}/{CONFIGURATION_FILE_OPENSEARCH_YML}"), - name: config_volume_name().to_string(), + name: CONFIG_VOLUME_NAME.to_string(), read_only: Some(true), sub_path: Some(CONFIGURATION_FILE_OPENSEARCH_YML.to_owned()), ..VolumeMount::default() @@ -414,24 +409,24 @@ impl<'a> RoleGroupBuilder<'a> { mount_path: format!( "{opensearch_path_conf}/{CONFIGURATION_FILE_LOG4J2_PROPERTIES}" ), - name: log_config_volume_name().to_string(), + name: LOG_CONFIG_VOLUME_NAME.to_string(), read_only: Some(true), sub_path: Some(CONFIGURATION_FILE_LOG4J2_PROPERTIES.to_owned()), ..VolumeMount::default() }, VolumeMount { mount_path: format!("{opensearch_home}/data"), - name: data_volume_name().to_string(), + name: DATA_VOLUME_NAME.to_string(), ..VolumeMount::default() }, VolumeMount { mount_path: LISTENER_VOLUME_DIR.to_owned(), - name: LISTENER_VOLUME_NAME.to_owned(), + name: LISTENER_VOLUME_NAME.to_string(), ..VolumeMount::default() }, VolumeMount { mount_path: LOG_VOLUME_DIR.to_owned(), - name: log_volume_name().to_string(), + name: LOG_VOLUME_NAME.to_string(), ..VolumeMount::default() }, ]; @@ -632,6 +627,7 @@ mod tests { str::FromStr, }; + use pretty_assertions::assert_eq; use serde_json::json; use stackable_operator::{ commons::{ @@ -647,8 +643,8 @@ mod tests { use uuid::uuid; use super::{ - RoleGroupBuilder, config_volume_name, data_volume_name, listener_volume_name, - log_config_volume_name, + CONFIG_VOLUME_NAME, DATA_VOLUME_NAME, LISTENER_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, RoleGroupBuilder, }; use crate::{ controller::{ @@ -657,20 +653,22 @@ mod tests { }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, ServiceAccountName, ServiceName, builder::pod::container::EnvVarSet, + ClusterName, ConfigMapName, ControllerName, NamespaceName, OperatorName, ProductName, + ProductVersion, RoleGroupName, ServiceAccountName, ServiceName, + builder::pod::container::EnvVarSet, product_logging::framework::VectorContainerLogConfig, role_utils::GenericProductSpecificCommonConfig, }, }; #[test] - fn test_volume_names() { + fn test_constants() { // Test that the functions do not panic - config_volume_name(); - log_config_volume_name(); - data_volume_name(); - listener_volume_name(); + let _ = CONFIG_VOLUME_NAME; + let _ = LOG_CONFIG_VOLUME_NAME; + let _ = DATA_VOLUME_NAME; + let _ = LISTENER_VOLUME_NAME; + let _ = LOG_VOLUME_NAME; } fn context_names() -> ContextNames { @@ -700,13 +698,14 @@ mod tests { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), ), - vector_container: None, - // Some(VectorContainerLogConfig { - // log_config: ValidatedContainerLogConfigChoice::Automatic( - // AutomaticContainerLogConfig::default(), - // ), - // vector_aggregator_config_map_name: None, - // }), + vector_container: Some(VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_aggregator_config_map_name: ConfigMapName::from_str_unsafe( + "vector-aggregator", + ), + }), }, node_roles: NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, @@ -765,9 +764,18 @@ mod tests { let context_names = context_names(); let role_group_builder = role_group_builder(&context_names); - let config_map = serde_json::to_value(role_group_builder.build_config_map()) + let mut config_map = serde_json::to_value(role_group_builder.build_config_map()) .expect("should be serializable"); + // The content of log4j2.properties is already tested in the + // `conrtoller::build::product_logging::config` module. + config_map["data"]["log4j2.properties"].take(); + // The content of opensearch.yml is already tested in the `controller::build::node_config` + // module. + config_map["data"]["opensearch.yml"].take(); + // vector.yaml is a static file and does not have to be repeated here. + config_map["data"]["vector.yaml"].take(); + assert_eq!( json!({ "apiVersion": "v1", @@ -795,38 +803,9 @@ mod tests { ] }, "data": { - "log4j2.properties": concat!( - "appenders = FILE, CONSOLE\n\n", - "appender.CONSOLE.type = Console\n", - "appender.CONSOLE.name = CONSOLE\n", - "appender.CONSOLE.target = SYSTEM_ERR\n", - "appender.CONSOLE.layout.type = PatternLayout\n", - "appender.CONSOLE.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n\n", - "appender.CONSOLE.filter.threshold.type = ThresholdFilter\n", - "appender.CONSOLE.filter.threshold.level = INFO\n\n", - "appender.FILE.type = RollingFile\n", - "appender.FILE.name = FILE\n", - "appender.FILE.fileName = /stackable/log/opensearch/opensearch.log4j2.xml\n", - "appender.FILE.filePattern = /stackable/log/opensearch/opensearch.log4j2.xml.%i\n", - "appender.FILE.layout.type = XMLLayout\n", - "appender.FILE.policies.type = Policies\n", - "appender.FILE.policies.size.type = SizeBasedTriggeringPolicy\n", - "appender.FILE.policies.size.size = 5MB\n", - "appender.FILE.strategy.type = DefaultRolloverStrategy\n", - "appender.FILE.strategy.max = 1\n", - "appender.FILE.filter.threshold.type = ThresholdFilter\n", - "appender.FILE.filter.threshold.level = INFO\n\n\n", - "rootLogger.level=INFO\n", - "rootLogger.appenderRefs = CONSOLE, FILE\n", - "rootLogger.appenderRef.CONSOLE.ref = CONSOLE\n", - "rootLogger.appenderRef.FILE.ref = FILE" - ), - "opensearch.yml": concat!( - "cluster.name: \"my-opensearch-cluster\"\n", - "discovery.type: \"single-node\"\n", - "network.host: \"0.0.0.0\"\n", - "plugins.security.nodes_dn: [\"CN=generated certificate for pod\"]" - ) + "log4j2.properties": null, + "opensearch.yml": null, + "vector.yaml": null } }), config_map @@ -899,9 +878,52 @@ mod tests { "affinity": {}, "containers": [ { - "args": [], + "args": [ + concat!( + "\n", + "prepare_signal_handlers()\n", + "{\n", + " unset term_child_pid\n", + " unset term_kill_needed\n", + " trap 'handle_term_signal' TERM\n", + "}\n", + "\n", + "handle_term_signal()\n", + "{\n", + " if [ \"${term_child_pid}\" ]; then\n", + " kill -TERM \"${term_child_pid}\" 2>/dev/null\n", + " else\n", + " term_kill_needed=\"yes\"\n", + " fi\n", + "}\n", + "\n", + "wait_for_termination()\n", + "{\n", + " set +e\n", + " term_child_pid=$1\n", + " if [[ -v term_kill_needed ]]; then\n", + " kill -TERM \"${term_child_pid}\" 2>/dev/null\n", + " fi\n", + " wait ${term_child_pid} 2>/dev/null\n", + " trap - TERM\n", + " wait ${term_child_pid} 2>/dev/null\n", + " set -e\n", + "}\n", + "\n", + "rm -f /stackable/log/_vector/shutdown\n", + "prepare_signal_handlers\n", + "containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n", + "/stackable/opensearch/opensearch-docker-entrypoint.sh &\n", + "wait_for_termination $!\n", + "mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown" + ) + ], "command": [ - "/stackable/opensearch/opensearch-docker-entrypoint.sh" + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c" ], "env": [ { @@ -976,9 +998,110 @@ mod tests { { "mountPath": "/stackable/listener", "name": "listener" + }, + { + "mountPath": "/stackable/log", + "name": "log" } ] - } + }, + { + "args": [ + concat!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n", + "vector & vector_pid=$!\n", + "if [ ! -f \"/stackable/log/_vector/shutdown\" ]; then\n", + "mkdir -p /stackable/log/_vector\n", + "inotifywait -qq --event create /stackable/log/_vector;\n", + "fi\n", + "sleep 1\n", + "kill $vector_pid" + ), + ], + "command": [ + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c" + ], + "env": [ + { + "name": "CLUSTER_NAME", + "value":"my-opensearch-cluster", + }, + { + "name": "LOG_DIR", + "value": "/stackable/log", + }, + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace", + }, + }, + }, + { + "name": "OPENSEARCH_SERVER_LOG_FILE", + "value": "opensearch_server.json", + }, + { + "name": "ROLE_GROUP_NAME", + "value": "default", + }, + { + "name": "ROLE_NAME", + "value": "nodes", + }, + { + "name": "VECTOR_AGGREGATOR_ADDRESS", + "valueFrom": { + "configMapKeyRef": { + "key": "ADDRESS", + "name": "vector-aggregator", + }, + }, + }, + { + "name": "VECTOR_CONFIG_YAML", + "value": "/stackable/config/vector.yaml", + }, + { + "name": "VECTOR_FILE_LOG_LEVEL", + "value": "info", + }, + { + "name": "VECTOR_LOG", + "value": "info", + }, + ], + "image": "oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev", + "imagePullPolicy": "Always", + "name": "vector", + "resources": { + "limits": { + "cpu": "500m", + "memory": "128Mi", + }, + "requests": { + "cpu": "250m", + "memory": "128Mi", + }, + }, + "volumeMounts": [ + { + "mountPath": "/stackable/config/vector.yaml", + "name": "log-config", + "readOnly": true, + "subPath": "vector.yaml", + }, + { + "mountPath": "/stackable/log", + "name": "log", + }, + ], + }, ], "securityContext": { "fsGroup": 1000 @@ -997,7 +1120,13 @@ mod tests { "name": "my-opensearch-cluster-nodes-default" }, "name": "log-config" - } + }, + { + "emptyDir": { + "sizeLimit": "30Mi" + }, + "name": "log" + } ] } }, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 24dd20e..5b9ea77 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -204,7 +204,7 @@ fn validate_role_group_config( for (env_var_name, env_var_value) in merged_role_group.config.env_overrides { env_overrides = env_overrides.with_value( - EnvVarName::from_str(&env_var_name).context(ParseEnvironmentVariableSnafu)?, + &EnvVarName::from_str(&env_var_name).context(ParseEnvironmentVariableSnafu)?, env_var_value, ); } @@ -229,7 +229,6 @@ fn validate_logging_configuration( validate_logging_configuration_for_container(logging, v1alpha1::Container::OpenSearch) .context(ValidateLoggingConfigSnafu)?; - // TODO Move to framework? let vector_container = if logging.enable_vector_agent { let vector_aggregator_config_map_name = vector_aggregator_config_map_name.context(GetVectorAggregatorConfigMapNameSnafu)?; @@ -253,8 +252,9 @@ fn validate_logging_configuration( #[cfg(test)] mod tests { - use std::str::FromStr; + use std::{collections::BTreeMap, str::FromStr}; + use pretty_assertions::assert_eq; use stackable_operator::{ commons::{ affinity::StackableAffinity, @@ -291,7 +291,9 @@ mod tests { ClusterName, ConfigMapName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, RoleGroupName, builder::pod::container::{EnvVarName, EnvVarSet}, - product_logging::framework::ValidatedContainerLogConfigChoice, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + }, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, }; @@ -360,27 +362,44 @@ mod tests { }, listener_class: "listener-class-from-role-group-level".to_owned(), logging: ValidatedLogging { - opensearch_container: ValidatedContainerLogConfigChoice::Custom( - ConfigMapName::from_str_unsafe("custom-log-config-map") + opensearch_container: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig { + loggers: [( + "ROOT".to_owned(), + LoggerConfig { + level: LogLevel::INFO + } + )] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + file: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + }, ), - vector_container: None, - // ValidatedContainerLogConfigChoice::Automatic( - // AutomaticContainerLogConfig { - // loggers: [( - // "ROOT".to_owned(), - // LoggerConfig { - // level: LogLevel::INFO - // } - // )] - // .into(), - // console: Some(AppenderConfig { - // level: Some(LogLevel::INFO) - // }), - // file: Some(AppenderConfig { - // level: Some(LogLevel::INFO) - // }), - // }, - // ), + vector_container: Some(VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig { + loggers: [( + "ROOT".to_owned(), + LoggerConfig { + level: LogLevel::INFO + }, + )] + .into(), + console: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + file: Some(AppenderConfig { + level: Some(LogLevel::INFO) + }), + }, + ), + vector_aggregator_config_map_name: + ConfigMapName::from_str_unsafe("vector-aggregator"), + }), }, node_roles: NodeRoles( [ @@ -561,30 +580,53 @@ mod tests { ); } - // #[test] - // fn test_validate_err_parse_container_name() { - // test_validate_err( - // |cluster| { - // cluster.spec.nodes.config.config.logging = LoggingFragment { - // enable_vector_agent: Some(true), - // containers: [( - // v1alpha1::Container::OpenSearch, - // ContainerLogConfigFragment { - // choice: Some(ContainerLogConfigChoiceFragment::Custom( - // CustomContainerLogConfigFragment { - // custom: ConfigMapLogConfigFragment { - // config_map: Some("invalid ConfigMap name".to_owned()), - // }, - // }, - // )), - // }, - // )] - // .into(), - // } - // }, - // ErrorDiscriminants::ParseContainerName, - // ); - // } + #[test] + fn test_validate_err_parse_vector_aggregator_config_map_name() { + test_validate_err( + |cluster| { + cluster + .spec + .cluster_config + .vector_aggregator_config_map_name = Some("invalid ConfigMap name".to_owned()) + }, + ErrorDiscriminants::ParseVectorAggregatorConfigMapName, + ); + } + + #[test] + fn test_validate_err_validate_logging_config() { + test_validate_err( + |cluster| { + cluster.spec.nodes.config.config.logging.containers = [( + v1alpha1::Container::OpenSearch, + ContainerLogConfigFragment { + choice: Some(ContainerLogConfigChoiceFragment::Custom( + CustomContainerLogConfigFragment { + custom: ConfigMapLogConfigFragment { + config_map: Some("invalid ConfigMap name".to_owned()), + }, + }, + )), + }, + )] + .into() + }, + ErrorDiscriminants::ValidateLoggingConfig, + ); + } + + #[test] + fn test_validate_err_get_vector_aggregator_config_map_name() { + test_validate_err( + |cluster| { + cluster + .spec + .cluster_config + .vector_aggregator_config_map_name = None + }, + ErrorDiscriminants::GetVectorAggregatorConfigMapName, + ); + } #[test] fn test_validate_err_termination_grace_period_too_long() { @@ -643,7 +685,7 @@ mod tests { image: serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage structure"), cluster_config: v1alpha1::OpenSearchClusterConfig { - vector_aggregator_config_map_name: None, + vector_aggregator_config_map_name: Some("vector-aggregator".to_owned()), }, cluster_operation: ClusterOperation::default(), nodes: Role { @@ -653,21 +695,7 @@ mod tests { listener_class: Some("listener-class-from-role-level".to_owned()), logging: LoggingFragment { enable_vector_agent: Some(true), - containers: [( - v1alpha1::Container::OpenSearch, - ContainerLogConfigFragment { - choice: Some(ContainerLogConfigChoiceFragment::Custom( - CustomContainerLogConfigFragment { - custom: ConfigMapLogConfigFragment { - config_map: Some( - "custom-log-config-map".to_owned(), - ), - }, - }, - )), - }, - )] - .into(), + containers: BTreeMap::default(), }, ..v1alpha1::OpenSearchConfigFragment::default() }, diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index b338d4b..e04d5e5 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -32,8 +32,9 @@ pub mod kvp; pub mod product_logging; pub mod role_group_utils; pub mod role_utils; +pub mod validation; -#[derive(Snafu, Debug, EnumDiscriminants)] +#[derive(Debug, EnumDiscriminants, Snafu)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { #[snafu(display("empty strings are not allowed"))] @@ -42,6 +43,11 @@ pub enum Error { #[snafu(display("maximum length exceeded"))] LengthExceeded { length: usize, max_length: usize }, + #[snafu(display("not a valid ConfigMap key"))] + InvalidConfigMapKey { + source: crate::framework::validation::Error, + }, + #[snafu(display("not a valid label value"))] InvalidLabelValue { source: stackable_operator::kvp::LabelValueError, @@ -168,15 +174,18 @@ macro_rules! attributed_string_type { } ); }; + (@from_str $name:ident, $s:expr, is_config_map_key) => { + $crate::framework::validation::is_config_map_key($s).context($crate::framework::InvalidConfigMapKeySnafu)?; + }; + (@from_str $name:ident, $s:expr, is_rfc_1035_label_name) => { + stackable_operator::validation::is_lowercase_rfc_1035_label($s).context($crate::framework::InvalidRfc1035LabelNameSnafu)?; + }; (@from_str $name:ident, $s:expr, is_rfc_1123_dns_subdomain_name) => { stackable_operator::validation::is_lowercase_rfc_1123_subdomain($s).context($crate::framework::InvalidRfc1123DnsSubdomainNameSnafu)?; }; (@from_str $name:ident, $s:expr, is_rfc_1123_label_name) => { stackable_operator::validation::is_lowercase_rfc_1123_label($s).context($crate::framework::InvalidRfc1123LabelNameSnafu)?; }; - (@from_str $name:ident, $s:expr, is_rfc_1035_label_name) => { - stackable_operator::validation::is_lowercase_rfc_1035_label($s).context($crate::framework::InvalidRfc1035LabelNameSnafu)?; - }; (@from_str $name:ident, $s:expr, is_valid_label_value) => { stackable_operator::kvp::LabelValue::from_str($s).context($crate::framework::InvalidLabelValueSnafu)?; }; @@ -189,6 +198,8 @@ macro_rules! attributed_string_type { pub const MAX_LENGTH: usize = $max_length; } }; + (@trait_impl $name:ident, is_config_map_key) => { + }; (@trait_impl $name:ident, is_rfc_1035_label_name) => { impl $name { pub const IS_RFC_1035_LABEL_NAME: bool = true; @@ -233,6 +244,14 @@ macro_rules! attributed_string_type { }; } +#[macro_export(local_inner_macros)] +macro_rules! constant { + ($qualifier:vis $name:ident: $type:ident = $value:literal) => { + $qualifier static $name: std::sync::LazyLock<$type> = + std::sync::LazyLock::new(|| $type::from_str($value).expect("should be a valid $name")); + }; +} + /// Returns the minimum of the given values. /// /// As opposed to [`std::cmp::min`], this function can be used at compile-time. @@ -257,6 +276,15 @@ attributed_string_type! { (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), is_rfc_1123_dns_subdomain_name } +attributed_string_type! { + ConfigMapKey, + "The key for a ConfigMap or Secret", + "log4j2.properties", + // see https://github.com/kubernetes/kubernetes/blob/v1.34.1/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L435-L451 + (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), + // TODO Use a custom regex? + is_config_map_key +} attributed_string_type! { ContainerName, "The name of a container in a Pod", diff --git a/rust/operator-binary/src/framework/builder/pod/container.rs b/rust/operator-binary/src/framework/builder/pod/container.rs index de71f2b..2eb488b 100644 --- a/rust/operator-binary/src/framework/builder/pod/container.rs +++ b/rust/operator-binary/src/framework/builder/pod/container.rs @@ -7,7 +7,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::framework::{ConfigMapName, ContainerName}; +use crate::framework::{ConfigMapKey, ConfigMapName, ContainerName}; #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -24,6 +24,7 @@ pub fn new_container_builder(container_name: &ContainerName) -> ContainerBuilder ContainerBuilder::new(container_name.as_ref()).expect("should be a valid container name") } +// TODO Use attributed_string_type instead? /// Validated environment variable name #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct EnvVarName(String); @@ -72,8 +73,8 @@ impl EnvVarSet { } /// Returns a reference to the [`EnvVar`] with the given name - pub fn get(&self, env_var_name: impl Into) -> Option<&EnvVar> { - self.0.get(&env_var_name.into()) + pub fn get(&self, env_var_name: &EnvVarName) -> Option<&EnvVar> { + self.0.get(env_var_name) } /// Moves all [`EnvVar`]s from the given set into this one. @@ -88,25 +89,22 @@ impl EnvVarSet { /// Adds the given [`EnvVar`]s to this set /// /// [`EnvVar`]s with the same name are overridden. - pub fn with_values(self, env_vars: I) -> Self + pub fn with_values(self, env_vars: I) -> Self where - I: IntoIterator, - K: Into, + I: IntoIterator, V: Into, { env_vars .into_iter() .fold(self, |extended_env_vars, (name, value)| { - extended_env_vars.with_value(name, value) + extended_env_vars.with_value(&name, value) }) } /// Adds an environment variable with the given name and string value to this set /// /// An [`EnvVar`] with the same name is overridden. - pub fn with_value(mut self, name: impl Into, value: impl Into) -> Self { - let name: EnvVarName = name.into(); - + pub fn with_value(mut self, name: &EnvVarName, value: impl Into) -> Self { self.0.insert( name.clone(), EnvVar { @@ -122,13 +120,7 @@ impl EnvVarSet { /// Adds an environment variable with the given name and field path to this set /// /// An [`EnvVar`] with the same name is overridden. - pub fn with_field_path( - mut self, - name: impl Into, - field_path: FieldPathEnvVar, - ) -> Self { - let name: EnvVarName = name.into(); - + pub fn with_field_path(mut self, name: &EnvVarName, field_path: FieldPathEnvVar) -> Self { self.0.insert( name.clone(), EnvVar { @@ -152,12 +144,10 @@ impl EnvVarSet { /// An [`EnvVar`] with the same name is overridden. pub fn with_config_map_key_ref( mut self, - name: impl Into, + name: &EnvVarName, config_map_name: &ConfigMapName, - config_map_key: impl Into, + config_map_key: &ConfigMapKey, ) -> Self { - let name: EnvVarName = name.into(); - self.0.insert( name.clone(), EnvVar { @@ -165,7 +155,7 @@ impl EnvVarSet { value: None, value_from: Some(EnvVarSource { config_map_key_ref: Some(ConfigMapKeySelector { - key: config_map_key.into(), + key: config_map_key.to_string(), name: config_map_name.to_string(), ..ConfigMapKeySelector::default() }), @@ -235,12 +225,12 @@ mod tests { ]); let env_var_set2 = EnvVarSet::new() .with_value( - EnvVarName::from_str_unsafe("ENV2"), + &EnvVarName::from_str_unsafe("ENV2"), "value2 from env_var_set2", ) - .with_field_path(EnvVarName::from_str_unsafe("ENV3"), FieldPathEnvVar::Name) + .with_field_path(&EnvVarName::from_str_unsafe("ENV3"), FieldPathEnvVar::Name) .with_value( - EnvVarName::from_str_unsafe("ENV4"), + &EnvVarName::from_str_unsafe("ENV4"), "value4 from env_var_set2", ); @@ -305,7 +295,7 @@ mod tests { #[test] fn test_envvarset_with_value() { - let env_var_set = EnvVarSet::new().with_value(EnvVarName::from_str_unsafe("ENV"), "value"); + let env_var_set = EnvVarSet::new().with_value(&EnvVarName::from_str_unsafe("ENV"), "value"); assert_eq!( Some(&EnvVar { @@ -313,14 +303,14 @@ mod tests { value: Some("value".to_owned()), value_from: None }), - env_var_set.get(EnvVarName::from_str_unsafe("ENV")) + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) ); } #[test] fn test_envvarset_with_field_path() { let env_var_set = EnvVarSet::new() - .with_field_path(EnvVarName::from_str_unsafe("ENV"), FieldPathEnvVar::Name); + .with_field_path(&EnvVarName::from_str_unsafe("ENV"), FieldPathEnvVar::Name); assert_eq!( Some(&EnvVar { @@ -334,7 +324,7 @@ mod tests { ..EnvVarSource::default() }), }), - env_var_set.get(EnvVarName::from_str_unsafe("ENV")) + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) ); } } diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs index 2c630c4..bd29c5a 100644 --- a/rust/operator-binary/src/framework/product_logging/framework.rs +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -5,20 +5,42 @@ use stackable_operator::{ builder::pod::{container::FieldPathEnvVar, resources::ResourceRequirementsBuilder}, commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::{Container, VolumeMount}, - product_logging::spec::{ - AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, LogLevel, Logging, + product_logging::{ + framework::VECTOR_CONFIG_FILE, + spec::{ + AppenderConfig, AutomaticContainerLogConfig, ConfigMapLogConfig, + ContainerLogConfigChoice, CustomContainerLogConfig, LogLevel, Logging, + }, }, }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::framework::{ - ClusterName, ConfigMapName, ContainerName, RoleGroupName, RoleName, VolumeName, - builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, +use crate::{ + constant, + framework::{ + ClusterName, ConfigMapKey, ConfigMapName, ContainerName, RoleGroupName, RoleName, + VolumeName, + builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + }, }; -const STACKABLE_LOG_DIR: &str = "/stackable/log"; -const VECTOR_CONFIG_FILE: &str = "vector.yaml"; +// Public variant of `stackable_operator::product_logging::framework::STACKABLE_CONFIG_DIR` +const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; + +// Public variant of `stackable_operator::product_logging::framework::VECTOR_LOG_DIR` +const VECTOR_CONTROL_DIR: &str = "_vector"; + +// Public variant of `stackable_operator::product_logging::framework::SHUTDOWN_FILE` +const SHUTDOWN_FILE: &str = "shutdown"; + +// Public variant of `stackable_operator::product_logging::framework::STACKABLE_LOG_DIR` +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; + +// Public variant of `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_CM_KEY` +constant!(pub VECTOR_AGGREGATOR_CM_KEY: ConfigMapKey = "ADDRESS"); + +// Public variant of `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_ADDRESS` +constant!(pub VECTOR_AGGREGATOR_ENV_NAME: EnvVarName = "VECTOR_AGGREGATOR_ADDRESS"); #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] @@ -85,7 +107,7 @@ pub fn vector_container( image: &ResolvedProductImage, log_config_volume_name: &VolumeName, log_volume_name: &VolumeName, -) -> Option { +) -> Container { let log_level = if let ValidatedContainerLogConfigChoice::Automatic(log_config) = &vector_container_log_config.log_config { @@ -107,37 +129,37 @@ pub fn vector_container( }; let env_vars = EnvVarSet::new() - .with_value(EnvVarName::from_str_unsafe("CLUSTER_NAME"), cluster_name) - .with_value(EnvVarName::from_str_unsafe("LOG_DIR"), "/stackable/log") + .with_value(&EnvVarName::from_str_unsafe("CLUSTER_NAME"), cluster_name) + .with_value(&EnvVarName::from_str_unsafe("LOG_DIR"), "/stackable/log") .with_field_path( - EnvVarName::from_str_unsafe("NAMESPACE"), + &EnvVarName::from_str_unsafe("NAMESPACE"), FieldPathEnvVar::Namespace, ) .with_value( - EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), + // TODO parameter + &EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), "opensearch_server.json", ) .with_value( - EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), + &EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), role_group_name, ) - .with_value(EnvVarName::from_str_unsafe("ROLE_NAME"), role_name) + .with_value(&EnvVarName::from_str_unsafe("ROLE_NAME"), role_name) .with_config_map_key_ref( - EnvVarName::from_str_unsafe("VECTOR_AGGREGATOR"), + &VECTOR_AGGREGATOR_ENV_NAME, &vector_container_log_config.vector_aggregator_config_map_name, - // TODO type-safe? - "ADDRESS", + &VECTOR_AGGREGATOR_CM_KEY, ) .with_value( - EnvVarName::from_str_unsafe("VECTOR_CONFIG_YAML"), - format!("/stackable/config/{VECTOR_CONFIG_FILE}"), + &EnvVarName::from_str_unsafe("VECTOR_CONFIG_YAML"), + format!("{STACKABLE_CONFIG_DIR}/{VECTOR_CONFIG_FILE}"), ) .with_value( - EnvVarName::from_str_unsafe("VECTOR_FILE_LOG_LEVEL"), + &EnvVarName::from_str_unsafe("VECTOR_FILE_LOG_LEVEL"), vector_file_log_level.to_vector_literal(), ) .with_value( - EnvVarName::from_str_unsafe("VECTOR_LOG"), + &EnvVarName::from_str_unsafe("VECTOR_LOG"), log_level.to_vector_literal(), ); @@ -148,7 +170,7 @@ pub fn vector_container( .with_memory_limit("128Mi") .build(); - let container = new_container_builder(container_name) + new_container_builder(container_name) .image_from_product_image(image) .command(vec![ "/bin/bash".to_string(), @@ -166,15 +188,13 @@ pub fn vector_container( fi\n\ sleep 1\n\ kill $vector_pid", - vector_control_directory = format!("{STACKABLE_LOG_DIR}/_vector"), - // TODO - SHUTDOWN_FILE = "shutdown" + vector_control_directory = format!("{STACKABLE_LOG_DIR}/{VECTOR_CONTROL_DIR}"), )]) .add_env_vars(env_vars.into()) .add_volume_mounts([ VolumeMount { mount_path: format!( - "/stackable/config/{VECTOR_CONFIG_FILE}" + "{STACKABLE_CONFIG_DIR}/{VECTOR_CONFIG_FILE}" ), name: log_config_volume_name.to_string(), read_only: Some(true), @@ -189,7 +209,30 @@ pub fn vector_container( ]) .expect("The mount paths are statically defined and there should be no duplicates.") .resources(resources) - .build(); - - Some(container) + .build() } + +// #[test] +// fn test_validate_err_parse_container_name() { +// test_validate_err( +// |cluster| { +// cluster.spec.nodes.config.config.logging = LoggingFragment { +// enable_vector_agent: Some(true), +// containers: [( +// v1alpha1::Container::OpenSearch, +// ContainerLogConfigFragment { +// choice: Some(ContainerLogConfigChoiceFragment::Custom( +// CustomContainerLogConfigFragment { +// custom: ConfigMapLogConfigFragment { +// config_map: Some("invalid ConfigMap name".to_owned()), +// }, +// }, +// )), +// }, +// )] +// .into(), +// } +// }, +// ErrorDiscriminants::ParseContainerName, +// ); +// } diff --git a/rust/operator-binary/src/framework/validation.rs b/rust/operator-binary/src/framework/validation.rs new file mode 100644 index 0000000..f92e804 --- /dev/null +++ b/rust/operator-binary/src/framework/validation.rs @@ -0,0 +1,56 @@ +use std::sync::LazyLock; + +use regex::Regex; +use snafu::{Snafu, ensure}; +use stackable_operator::validation::RFC_1123_SUBDOMAIN_MAX_LENGTH; + +/// Format of a key for a ConfigMap or Secret +pub const CONFIG_MAP_KEY_FMT: &str = "[-._a-zA-Z0-9]+"; +const CONFIG_MAP_KEY_ERROR_MSG: &str = + "a valid config key must consist of alphanumeric characters, '-', '_' or '.'"; +static CONFIG_MAP_KEY_REGEX: LazyLock = LazyLock::new(|| { + Regex::new(&format!("^{CONFIG_MAP_KEY_FMT}$")).expect("failed to compile ConfigMap key regex") +}); + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("value does not match the regular expression"))] + Regex { + value: String, + regex: &'static str, + message: &'static str, + }, + + #[snafu(display("value exceeds the maximum length"))] + TooLong { value: String, max_length: usize }, +} + +type Result = std::result::Result<(), Error>; + +/// Tests if the given value is a valid key for a ConfigMap or Secret +/// +/// see +pub fn is_config_map_key(value: &str) -> Result { + // When adding this function to stackable_operator, use the private functions like + // validate_all. + + let max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH; + ensure!( + value.len() < max_length, + TooLongSnafu { + value: value.to_owned(), + max_length + } + ); + + ensure!( + CONFIG_MAP_KEY_REGEX.is_match(value), + RegexSnafu { + value: value.to_owned(), + regex: CONFIG_MAP_KEY_FMT, + message: CONFIG_MAP_KEY_ERROR_MSG + } + ); + + Ok(()) +} From 45dfd7c3aa05826e9c79c7fd906928ea0e39f6f0 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 8 Oct 2025 13:09:54 +0200 Subject: [PATCH 03/14] chore: Use safe types already in the CRD --- Cargo.toml | 2 +- .../helm/opensearch-operator/crds/crds.yaml | 6 + rust/operator-binary/src/controller.rs | 12 +- rust/operator-binary/src/controller/build.rs | 6 +- .../src/controller/build/node_config.rs | 4 +- .../src/controller/build/role_builder.rs | 6 +- .../controller/build/role_group_builder.rs | 10 +- .../src/controller/validate.rs | 63 +++---- rust/operator-binary/src/crd/mod.rs | 17 +- rust/operator-binary/src/framework.rs | 169 +++++++++++++++++- 10 files changed, 223 insertions(+), 72 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9e8641f..b548faf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ built = { version = "0.8.0", features = ["chrono", "git2"] } clap = "4.5" futures = { version = "0.3", features = ["compat"] } pretty_assertions = "1.4" -regex = "1.11.3" +regex = "1.11" rstest = "0.26" schemars = { version = "1.0.0", features = ["url2"] } # same as in operator-rs serde = { version = "1.0", features = ["derive"] } diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index e48aac2..ce589e7 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -38,6 +38,8 @@ spec: It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 nullable: true type: string type: object @@ -183,6 +185,8 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + maxLength: 253 + minLength: 1 nullable: true type: string logging: @@ -526,6 +530,8 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + maxLength: 253 + minLength: 1 nullable: true type: string logging: diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a7ba99f..8a29efc 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -33,8 +33,8 @@ use crate::{ v1alpha1::{self}, }, framework::{ - ClusterName, ControllerName, HasName, HasUid, NameIsValidLabelValue, NamespaceName, - OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, Uid, + ClusterName, ControllerName, HasName, HasUid, ListenerClassName, NameIsValidLabelValue, + NamespaceName, OperatorName, ProductName, ProductVersion, RoleGroupName, RoleName, Uid, product_logging::framework::{ValidatedContainerLogConfigChoice, VectorContainerLogConfig}, role_utils::{GenericProductSpecificCommonConfig, RoleGroupConfig}, }, @@ -128,7 +128,7 @@ type OpenSearchNodeResources = #[derive(Clone, Debug, PartialEq)] pub struct ValidatedOpenSearchConfig { pub affinity: StackableAffinity, - pub listener_class: String, + pub listener_class: ListenerClassName, pub logging: ValidatedLogging, pub node_roles: NodeRoles, pub resources: OpenSearchNodeResources, @@ -379,8 +379,8 @@ mod tests { controller::{OpenSearchNodeResources, ValidatedOpenSearchConfig}, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, NamespaceName, OperatorName, ProductVersion, RoleGroupName, - builder::pod::container::EnvVarSet, + ClusterName, ListenerClassName, NamespaceName, OperatorName, ProductVersion, + RoleGroupName, builder::pod::container::EnvVarSet, product_logging::framework::ValidatedContainerLogConfigChoice, role_utils::GenericProductSpecificCommonConfig, }, @@ -504,7 +504,7 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), - listener_class: "external-stable".to_owned(), + listener_class: ListenerClassName::from_str_unsafe("external-stable"), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 768fdcf..43c3bda 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -79,8 +79,8 @@ mod tests { }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, builder::pod::container::EnvVarSet, + ClusterName, ControllerName, ListenerClassName, NamespaceName, OperatorName, + ProductName, ProductVersion, RoleGroupName, builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -202,7 +202,7 @@ mod tests { replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), - listener_class: "external-stable".to_owned(), + listener_class: ListenerClassName::from_str_unsafe("external-stable"), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 56d5800..b11b2af 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -285,7 +285,7 @@ mod tests { controller::{ValidatedLogging, ValidatedOpenSearchConfig}, crd::NodeRoles, framework::{ - ClusterName, NamespaceName, ProductVersion, RoleGroupName, + ClusterName, ListenerClassName, NamespaceName, ProductVersion, RoleGroupName, product_logging::framework::ValidatedContainerLogConfigChoice, role_utils::GenericProductSpecificCommonConfig, }, @@ -315,7 +315,7 @@ mod tests { replicas: test_config.replicas, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), - listener_class: "cluster-internal".to_string(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), diff --git a/rust/operator-binary/src/controller/build/role_builder.rs b/rust/operator-binary/src/controller/build/role_builder.rs index fb23199..454d7d5 100644 --- a/rust/operator-binary/src/controller/build/role_builder.rs +++ b/rust/operator-binary/src/controller/build/role_builder.rs @@ -241,8 +241,8 @@ mod tests { }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ControllerName, NamespaceName, OperatorName, ProductName, ProductVersion, - RoleGroupName, builder::pod::container::EnvVarSet, + ClusterName, ControllerName, ListenerClassName, NamespaceName, OperatorName, + ProductName, ProductVersion, RoleGroupName, builder::pod::container::EnvVarSet, role_utils::GenericProductSpecificCommonConfig, }, }; @@ -263,7 +263,7 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), - listener_class: "cluster-internal".to_string(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 42e8a4f..d307491 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -569,7 +569,7 @@ impl<'a> RoleGroupBuilder<'a> { listener::v1alpha1::Listener { metadata, spec: listener::v1alpha1::ListenerSpec { - class_name: Some(listener_class), + class_name: Some(listener_class.to_string()), ports: Some(ports.to_vec()), ..listener::v1alpha1::ListenerSpec::default() }, @@ -653,9 +653,9 @@ mod tests { }, crd::{NodeRoles, v1alpha1}, framework::{ - ClusterName, ConfigMapName, ControllerName, NamespaceName, OperatorName, ProductName, - ProductVersion, RoleGroupName, ServiceAccountName, ServiceName, - builder::pod::container::EnvVarSet, + ClusterName, ConfigMapName, ControllerName, ListenerClassName, NamespaceName, + OperatorName, ProductName, ProductVersion, RoleGroupName, ServiceAccountName, + ServiceName, builder::pod::container::EnvVarSet, product_logging::framework::VectorContainerLogConfig, role_utils::GenericProductSpecificCommonConfig, }, @@ -693,7 +693,7 @@ mod tests { replicas: 1, config: ValidatedOpenSearchConfig { affinity: StackableAffinity::default(), - listener_class: "cluster-internal".to_string(), + listener_class: ListenerClassName::from_str_unsafe("cluster-internal"), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig::default(), diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 5b9ea77..e3dd839 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -64,9 +64,6 @@ pub enum Error { #[snafu(display("failed to set role-group name"))] ParseRoleGroupName { source: crate::framework::Error }, - #[snafu(display("failed to set vectorAggregatorConfigMapName"))] - ParseVectorAggregatorConfigMapName { source: crate::framework::Error }, - #[snafu(display("failed to resolve product image"))] ResolveProductImage { source: stackable_operator::commons::product_image_selection::Error, @@ -166,22 +163,12 @@ fn validate_role_group_config( ) .context(ValidateOpenSearchConfigSnafu)?; - let vector_aggregator_config_map_name = if let Some(config_map_name) = &cluster - .spec - .cluster_config - .vector_aggregator_config_map_name - { - Some( - ConfigMapName::from_str(config_map_name) - .context(ParseVectorAggregatorConfigMapNameSnafu)?, - ) - } else { - None - }; - let logging = validate_logging_configuration( &merged_role_group.config.config.logging, - vector_aggregator_config_map_name, + &cluster + .spec + .cluster_config + .vector_aggregator_config_map_name, )?; let graceful_shutdown_timeout = merged_role_group.config.config.graceful_shutdown_timeout; @@ -223,15 +210,16 @@ fn validate_role_group_config( fn validate_logging_configuration( logging: &Logging, - vector_aggregator_config_map_name: Option, + vector_aggregator_config_map_name: &Option, ) -> Result { let opensearch_container = validate_logging_configuration_for_container(logging, v1alpha1::Container::OpenSearch) .context(ValidateLoggingConfigSnafu)?; let vector_container = if logging.enable_vector_agent { - let vector_aggregator_config_map_name = - vector_aggregator_config_map_name.context(GetVectorAggregatorConfigMapNameSnafu)?; + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(GetVectorAggregatorConfigMapNameSnafu)?; Some(VectorContainerLogConfig { log_config: validate_logging_configuration_for_container( logging, @@ -288,8 +276,8 @@ mod tests { v1alpha1::{self}, }, framework::{ - ClusterName, ConfigMapName, ControllerName, NamespaceName, OperatorName, ProductName, - ProductVersion, RoleGroupName, + ClusterName, ConfigMapName, ControllerName, ListenerClassName, NamespaceName, + OperatorName, ProductName, ProductVersion, RoleGroupName, builder::pod::container::{EnvVarName, EnvVarSet}, product_logging::framework::{ ValidatedContainerLogConfigChoice, VectorContainerLogConfig, @@ -360,7 +348,9 @@ mod tests { }), ..StackableAffinity::default() }, - listener_class: "listener-class-from-role-group-level".to_owned(), + listener_class: ListenerClassName::from_str_unsafe( + "listener-class-from-role-group-level" + ), logging: ValidatedLogging { opensearch_container: ValidatedContainerLogConfigChoice::Automatic( AutomaticContainerLogConfig { @@ -580,19 +570,6 @@ mod tests { ); } - #[test] - fn test_validate_err_parse_vector_aggregator_config_map_name() { - test_validate_err( - |cluster| { - cluster - .spec - .cluster_config - .vector_aggregator_config_map_name = Some("invalid ConfigMap name".to_owned()) - }, - ErrorDiscriminants::ParseVectorAggregatorConfigMapName, - ); - } - #[test] fn test_validate_err_validate_logging_config() { test_validate_err( @@ -685,14 +662,18 @@ mod tests { image: serde_json::from_str(r#"{"productVersion": "3.1.0"}"#) .expect("should be a valid ProductImage structure"), cluster_config: v1alpha1::OpenSearchClusterConfig { - vector_aggregator_config_map_name: Some("vector-aggregator".to_owned()), + vector_aggregator_config_map_name: Some(ConfigMapName::from_str_unsafe( + "vector-aggregator", + )), }, cluster_operation: ClusterOperation::default(), nodes: Role { config: CommonConfiguration { config: v1alpha1::OpenSearchConfigFragment { graceful_shutdown_timeout: Some(Duration::from_minutes_unchecked(5)), - listener_class: Some("listener-class-from-role-level".to_owned()), + listener_class: Some(ListenerClassName::from_str_unsafe( + "listener-class-from-role-level", + )), logging: LoggingFragment { enable_vector_agent: Some(true), containers: BTreeMap::default(), @@ -740,9 +721,9 @@ mod tests { RoleGroup { config: CommonConfiguration { config: v1alpha1::OpenSearchConfigFragment { - listener_class: Some( - "listener-class-from-role-group-level".to_owned(), - ), + listener_class: Some(ListenerClassName::from_str_unsafe( + "listener-class-from-role-group-level", + )), ..v1alpha1::OpenSearchConfigFragment::default() }, config_overrides: [( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index af72612..28989b2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -26,12 +26,15 @@ use stackable_operator::{ }; use strum::{Display, EnumIter}; -use crate::framework::{ - ClusterName, ContainerName, NameIsValidLabelValue, ProductName, RoleName, - role_utils::GenericProductSpecificCommonConfig, +use crate::{ + constant, + framework::{ + ClusterName, ConfigMapName, ContainerName, ListenerClassName, NameIsValidLabelValue, + ProductName, RoleName, role_utils::GenericProductSpecificCommonConfig, + }, }; -const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; +constant!(DEFAULT_LISTENER_CLASS: ListenerClassName = "cluster-internal"); #[versioned( version(name = "v1alpha1"), @@ -82,7 +85,7 @@ pub mod versioned { /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) /// to learn how to configure log aggregation with Vector. #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + pub vector_aggregator_config_map_name: Option, } // The possible node roles are by default the built-in roles and the search role, see @@ -151,7 +154,7 @@ pub mod versioned { /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. #[fragment_attrs(serde(default))] - pub listener_class: String, + pub listener_class: ListenerClassName, #[fragment_attrs(serde(default))] pub logging: Logging, @@ -257,7 +260,7 @@ impl v1alpha1::OpenSearchConfig { ), // Defaults taken from the Helm chart, see // https://github.com/opensearch-project/helm-charts/blob/opensearch-3.0.0/charts/opensearch/values.yaml#L16-L20 - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_owned()), logging: product_logging::spec::default_logging(), node_roles: Some(NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index e04d5e5..e7e6b5c 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -20,6 +20,8 @@ //! become less frequent, then this module can be incorporated into stackable-operator. The module //! structure should already resemble the one of stackable-operator. +use std::str::FromStr; + use snafu::Snafu; use stackable_operator::validation::{ RFC_1035_LABEL_MAX_LENGTH, RFC_1123_LABEL_MAX_LENGTH, RFC_1123_SUBDOMAIN_MAX_LENGTH, @@ -149,6 +151,27 @@ macro_rules! attributed_string_type { } } + impl<'de> serde::Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string: String = serde::Deserialize::deserialize(deserializer)?; + $name::from_str(&string).map_err(|err| serde::de::Error::custom(&err)) + } + } + + impl serde::Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.serialize(serializer) + } + } + + impl stackable_operator::config::merge::Atomic for $name {} + #[cfg(test)] impl $name { #[allow(dead_code)] @@ -197,6 +220,21 @@ macro_rules! attributed_string_type { // type arithmetic would be better pub const MAX_LENGTH: usize = $max_length; } + + // The JsonSchema implementation requires `max_length`. + impl schemars::JsonSchema for $name { + fn schema_name() -> std::borrow::Cow<'static, str> { + std::stringify!($name).into() + } + + fn json_schema(_generator: &mut schemars::generate::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "minLength": 1, + "maxLength": $name::MAX_LENGTH + }) + } + } }; (@trait_impl $name:ident, is_config_map_key) => { }; @@ -282,7 +320,6 @@ attributed_string_type! { "log4j2.properties", // see https://github.com/kubernetes/kubernetes/blob/v1.34.1/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L435-L451 (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), - // TODO Use a custom regex? is_config_map_key } attributed_string_type! { @@ -458,19 +495,26 @@ attributed_string_type! { mod tests { use std::str::FromStr; + use schemars::{JsonSchema, SchemaGenerator}; + use serde_json::{Number, Value, json}; use uuid::uuid; use super::{ - ClusterName, ClusterRoleName, ConfigMapName, ControllerName, ErrorDiscriminants, - NamespaceName, OperatorName, PersistentVolumeClaimName, ProductVersion, RoleBindingName, - RoleGroupName, RoleName, ServiceAccountName, ServiceName, StatefulSetName, Uid, VolumeName, + ClusterName, ClusterRoleName, ConfigMapKey, ConfigMapName, ContainerName, ControllerName, + ErrorDiscriminants, ListenerClassName, ListenerName, NamespaceName, OperatorName, + PersistentVolumeClaimName, ProductVersion, RoleBindingName, RoleGroupName, RoleName, + ServiceAccountName, ServiceName, StatefulSetName, Uid, VolumeName, }; use crate::framework::{NameIsValidLabelValue, ProductName}; #[test] fn test_attributed_string_type_examples() { ConfigMapName::test_example(); + ConfigMapKey::test_example(); + ContainerName::test_example(); ClusterRoleName::test_example(); + ListenerName::test_example(); + ListenerClassName::test_example(); NamespaceName::test_example(); PersistentVolumeClaimName::test_example(); RoleBindingName::test_example(); @@ -540,6 +584,123 @@ mod tests { ); } + attributed_string_type! { + JsonSchemaTest, + "JsonSchemaTest test", + "test", + (max_length = 4) + } + + #[test] + fn test_attributed_string_type_json_schema() { + type T = JsonSchemaTest; + + T::test_example(); + assert_eq!("JsonSchemaTest", JsonSchemaTest::schema_name()); + assert_eq!( + json!({ + "type": "string", + "minLength": 1, + "maxLength": 4 + }), + JsonSchemaTest::json_schema(&mut SchemaGenerator::default()) + ); + } + + attributed_string_type! { + SerializeTest, + "serde::Serialize test", + "test" + } + + #[test] + fn test_attributed_string_type_serialize() { + type T = SerializeTest; + + T::test_example(); + assert_eq!( + "\"test\"".to_owned(), + serde_json::to_string(&T::from_str_unsafe("test")).expect("should be serializable") + ); + } + + attributed_string_type! { + DeserializeTest, + "serde::Deserialize test", + "test", + (max_length = 4), + is_rfc_1123_label_name + } + + #[test] + fn test_attributed_string_type_deserialize() { + type T = DeserializeTest; + + T::test_example(); + assert_eq!( + T::from_str_unsafe("test"), + serde_json::from_value(Value::String("test".to_owned())) + .expect("should be deserializable") + ); + assert_eq!( + Err("empty strings are not allowed".to_owned()), + serde_json::from_value::(Value::String("".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("maximum length exceeded".to_owned()), + serde_json::from_value::(Value::String("testx".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("not a valid label name as defined in RFC 1123".to_owned()), + serde_json::from_value::(Value::String("-".to_owned())) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: null, expected a string".to_owned()), + serde_json::from_value::(Value::Null).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: boolean `true`, expected a string".to_owned()), + serde_json::from_value::(Value::Bool(true)).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: integer `1`, expected a string".to_owned()), + serde_json::from_value::(Value::Number( + Number::from_i128(1).expect("should be a valid number") + )) + .map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: sequence, expected a string".to_owned()), + serde_json::from_value::(Value::Array(vec![])).map_err(|err| err.to_string()) + ); + assert_eq!( + Err("invalid type: map, expected a string".to_owned()), + serde_json::from_value::(Value::Object(serde_json::Map::new())) + .map_err(|err| err.to_string()) + ); + } + + attributed_string_type! { + IsConfigMapKeyTest, + "is_config_map_key test", + "a_B-c.1", + is_config_map_key + } + + #[test] + fn test_attributed_string_type_is_config_map_key() { + type T = IsConfigMapKeyTest; + + T::test_example(); + assert_eq!( + Err(ErrorDiscriminants::InvalidConfigMapKey), + T::from_str(" ").map_err(ErrorDiscriminants::from) + ); + } + attributed_string_type! { IsRfc1035LabelNameTest, "is_rfc_1035_label_name test", From 610d8ddedc16fb1a8452f65339f3759f16dc14c2 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 8 Oct 2025 17:08:41 +0200 Subject: [PATCH 04/14] test: Fix the integration tests --- .../controller/build/role_group_builder.rs | 12 +- ...pensearch-vector-aggregator-values.yaml.j2 | 88 +-- .../templates/kuttl/logging/20-assert.yaml.j2 | 664 +----------------- .../logging/20-install-opensearch.yaml.j2 | 109 ++- tests/templates/kuttl/logging/30-assert.yaml | 2 +- .../kuttl/logging/30-test-opensearch.yaml | 178 ++--- .../metrics/20-install-opensearch.yaml.j2 | 18 +- .../kuttl/metrics/30-check-metrics.yaml | 1 + tests/templates/kuttl/smoke/10-assert.yaml.j2 | 110 ++- tests/test-definition.yaml | 19 +- 10 files changed, 270 insertions(+), 931 deletions(-) diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index d307491..3899240 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -234,7 +234,7 @@ impl<'a> RoleGroupBuilder<'a> { &self.resource_names.role_name, &self.resource_names.role_group_name, &self.cluster.image, - &LOG_CONFIG_VOLUME_NAME, + &CONFIG_VOLUME_NAME, &LOG_VOLUME_NAME, ) }); @@ -433,10 +433,6 @@ impl<'a> RoleGroupBuilder<'a> { new_container_builder(&v1alpha1::Container::OpenSearch.to_container_name()) .image_from_product_image(&self.cluster.image) - .command(vec![format!( - "{opensearch_home}/opensearch-docker-entrypoint.sh" - )]) - .args(self.role_group_config.cli_overrides_to_vec()) .command(vec![ "/bin/bash".to_string(), "-x".to_string(), @@ -448,8 +444,12 @@ impl<'a> RoleGroupBuilder<'a> { "{COMMON_BASH_TRAP_FUNCTIONS}\n\ {remove_vector_shutdown_file_command}\n\ prepare_signal_handlers\n\ + if command --search containerdebug >/dev/null 2>&1; then\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &\n\ - {opensearch_home}/opensearch-docker-entrypoint.sh {extra_args} &\n\ + else\n\ + echo >&2 \"containerdebug not installed; Proceed without it.\"\n\ + fi\n\ + ./opensearch-docker-entrypoint.sh {extra_args} &\n\ wait_for_termination $!\n\ {create_vector_shutdown_file_command}", extra_args = self.role_group_config.cli_overrides_to_vec().join(" "), diff --git a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 index 9bc5fed..c6cb2ed 100644 --- a/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/10_opensearch-vector-aggregator-values.yaml.j2 @@ -22,83 +22,41 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) - filteredAutomaticLogConfigMasterHbase: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-master-automatic-log-config-0" && - .container == "opensearch" - filteredAutomaticLogConfigMasterVector: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-master-automatic-log-config-0" && - .container == "vector" - filteredCustomLogConfigMasterHbase: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-master-custom-log-config-0" && - .container == "opensearch" - filteredCustomLogConfigMasterVector: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-master-custom-log-config-0" && - .container == "vector" - filteredAutomaticLogConfigRegionserverHbase: + filteredAutomaticLogConfigOpenSearch: type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-regionserver-automatic-log-config-0" && - .container == "opensearch" - filteredAutomaticLogConfigRegionserverVector: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-regionserver-automatic-log-config-0" && - .container == "vector" - filteredCustomLogConfigRegionserverHbase: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-regionserver-custom-log-config-0" && - .container == "opensearch" - filteredCustomLogConfigRegionserverVector: - type: filter - inputs: [validEvents] - condition: >- - .pod == "test-opensearch-regionserver-custom-log-config-0" && - .container == "vector" - filteredAutomaticLogConfigRestserverHbase: - type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- - .pod == "test-opensearch-restserver-automatic-log-config-0" && + .pod == "opensearch-nodes-automatic-0" && .container == "opensearch" - filteredAutomaticLogConfigRestserverVector: + filteredAutomaticLogConfigVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- - .pod == "test-opensearch-restserver-automatic-log-config-0" && + .pod == "opensearch-nodes-automatic-0" && .container == "vector" - filteredCustomLogConfigRestserverHbase: + filteredCustomLogConfigOpenSearch: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- - .pod == "test-opensearch-restserver-custom-log-config-0" && + .pod == "opensearch-nodes-custom-0" && .container == "opensearch" - filteredCustomLogConfigRestserverVector: + filteredCustomLogConfigVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- - .pod == "test-opensearch-restserver-custom-log-config-0" && + .pod == "opensearch-nodes-custom-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -106,7 +64,8 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole stdout: inputs: @@ -116,7 +75,8 @@ customConfig: codec: json {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector address: {{ lookup('env', 'VECTOR_AGGREGATOR') }} buffer: diff --git a/tests/templates/kuttl/logging/20-assert.yaml.j2 b/tests/templates/kuttl/logging/20-assert.yaml.j2 index ca660d1..e705ea3 100644 --- a/tests/templates/kuttl/logging/20-assert.yaml.j2 +++ b/tests/templates/kuttl/logging/20-assert.yaml.j2 @@ -1,6 +1,3 @@ -# All fields are checked that are set by the operator. -# This helps to detect unintentional changes. It is also a good reference for the output of the -# operator. The maintenance effort should be okay as long as it is only done in the smoke test. --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert @@ -9,662 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-cluster-manager - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - podManagementPolicy: Parallel - replicas: 3 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - serviceName: opensearch-nodes-cluster-manager-headless - template: - metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/opensearch-role.cluster_manager: "true" - stackable.tech/vendor: Stackable - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - topologyKey: kubernetes.io/hostname - weight: 1 - containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh - env: - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - - name: OPENSEARCH_HOME - value: {{ test_scenario['values']['opensearch_home'] }} - - name: cluster.initial_cluster_manager_nodes - value: opensearch-nodes-cluster-manager-0,opensearch-nodes-cluster-manager-1,opensearch-nodes-cluster-manager-2 - - name: discovery.seed_hosts - value: opensearch - - name: node.name - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: node.roles - value: cluster_manager - imagePullPolicy: IfNotPresent - name: opensearch - ports: - - containerPort: 9200 - name: http - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - readinessProbe: - failureThreshold: 3 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - resources: - limits: - cpu: "4" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - startupProbe: - failureThreshold: 30 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - volumeMounts: - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch.yml - name: config - readOnly: true - subPath: opensearch.yml - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties - name: log-config - readOnly: true - subPath: log4j2.properties - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data - name: data - - mountPath: /stackable/listener - name: listener - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security - name: security-config - readOnly: true - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls - name: tls - readOnly: true - securityContext: - fsGroup: 1000 - serviceAccount: opensearch-serviceaccount - serviceAccountName: opensearch-serviceaccount - terminationGracePeriodSeconds: 120 - volumes: - - configMap: - defaultMode: 420 - name: opensearch-nodes-cluster-manager - name: config - - configMap: - defaultMode: 420 - name: opensearch-nodes-cluster-manager - name: log-config - - name: security-config - secret: - defaultMode: 420 - secretName: opensearch-security-config - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - volumeMode: Filesystem - name: tls - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - volumeMode: Filesystem - status: - phase: Pending - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - annotations: - listeners.stackable.tech/listener-name: opensearch-nodes-cluster-manager - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: listener - spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: "1" - storageClassName: listeners.stackable.tech - volumeMode: Filesystem - status: - phase: Pending + name: opensearch-nodes-automatic status: - readyReplicas: 3 - replicas: 3 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-data - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - podManagementPolicy: Parallel - replicas: 2 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - serviceName: opensearch-nodes-data-headless - template: - metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/opensearch-role.data: "true" - stackable.tech/opensearch-role.ingest: "true" - stackable.tech/opensearch-role.remote_cluster_client: "true" - stackable.tech/vendor: Stackable - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - topologyKey: kubernetes.io/hostname - weight: 1 - containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh - env: - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - - name: OPENSEARCH_HOME - value: {{ test_scenario['values']['opensearch_home'] }} - - name: cluster.initial_cluster_manager_nodes - - name: discovery.seed_hosts - value: opensearch - - name: node.name - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: node.roles - value: ingest,data,remote_cluster_client - imagePullPolicy: IfNotPresent - name: opensearch - ports: - - containerPort: 9200 - name: http - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - readinessProbe: - failureThreshold: 3 - periodSeconds: 5 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - resources: - limits: - cpu: "4" - memory: 2Gi - requests: - cpu: "1" - memory: 2Gi - startupProbe: - failureThreshold: 30 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - tcpSocket: - port: http - timeoutSeconds: 3 - volumeMounts: - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch.yml - name: config - readOnly: true - subPath: opensearch.yml - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/log4j2.properties - name: log-config - readOnly: true - subPath: log4j2.properties - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/data - name: data - - mountPath: /stackable/listener - name: listener - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security - name: security-config - readOnly: true - - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls - name: tls - readOnly: true - securityContext: - fsGroup: 1000 - serviceAccount: opensearch-serviceaccount - serviceAccountName: opensearch-serviceaccount - terminationGracePeriodSeconds: 120 - volumes: - - configMap: - defaultMode: 420 - name: opensearch-nodes-data - name: config - - configMap: - defaultMode: 420 - name: opensearch-nodes-data - name: log-config - - name: security-config - secret: - defaultMode: 420 - secretName: opensearch-security-config - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - volumeMode: Filesystem - name: tls - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - volumeMode: Filesystem - status: - phase: Pending - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - annotations: - listeners.stackable.tech/listener-name: opensearch-nodes-data - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: listener - spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: "1" - storageClassName: listeners.stackable.tech - volumeMode: Filesystem - status: - phase: Pending -status: - readyReplicas: 2 - replicas: 2 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-cluster-manager - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -data: - opensearch.yml: |- - cluster.name: "opensearch" - cluster.routing.allocation.disk.threshold_enabled: "false" - discovery.type: "zen" - network.host: "0.0.0.0" - node.store.allow_mmap: "false" - plugins.security.allow_default_init_securityindex: "true" - plugins.security.nodes_dn: ["CN=generated certificate for pod"] - plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" - plugins.security.ssl.http.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" - plugins.security.ssl.http.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" - plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" - plugins.security.ssl.transport.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" - plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-data - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -data: - opensearch.yml: |- - cluster.name: "opensearch" - cluster.routing.allocation.disk.threshold_enabled: "false" - discovery.type: "zen" - network.host: "0.0.0.0" - node.store.allow_mmap: "false" - plugins.security.allow_default_init_securityindex: "true" - plugins.security.nodes_dn: ["CN=generated certificate for pod"] - plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" - plugins.security.ssl.http.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" - plugins.security.ssl.http.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" - plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt" - plugins.security.ssl.transport.pemkey_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key" - plugins.security.ssl.transport.pemtrustedcas_filepath: "{{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt" ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-cluster-manager-headless -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-nodes-data-headless -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - ports: - - name: http - port: 9200 - protocol: TCP - targetPort: 9200 - - name: transport - port: 9300 - protocol: TCP - targetPort: 9300 - publishNotReadyAddresses: true - selector: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch - stackable.tech/opensearch-role.cluster_manager: "true" - type: ClusterIP ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-serviceaccount - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} - stackable.tech/vendor: Stackable - name: opensearch-rolebinding - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: opensearch-clusterrole -subjects: -- kind: ServiceAccount - name: opensearch-serviceaccount ---- -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - name: opensearch-nodes -spec: - maxUnavailable: 1 - selector: - matchLabels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/name: opensearch ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: Listener -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.1.0 - stackable.tech/vendor: Stackable - name: opensearch-nodes-cluster-manager - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - className: external-stable - extraPodSelectorLabels: {} - ports: - - name: http - port: 9200 - protocol: TCP - publishNotReadyAddresses: null ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: Listener -metadata: - labels: - app.kubernetes.io/component: nodes - app.kubernetes.io/instance: opensearch - app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster - app.kubernetes.io/name: opensearch - app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.1.0 - stackable.tech/vendor: Stackable - name: opensearch-nodes-data - ownerReferences: - - apiVersion: opensearch.stackable.tech/v1alpha1 - controller: true - kind: OpenSearchCluster - name: opensearch -spec: - className: cluster-internal - extraPodSelectorLabels: {} - ports: - - name: http - port: 9200 - protocol: TCP - publishNotReadyAddresses: null + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 index 025da48..f17b427 100644 --- a/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/logging/20-install-opensearch.yaml.j2 @@ -1,4 +1,18 @@ --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-log-config +data: + log4j2.properties: | + rootLogger.level = INFO + rootLogger.appenderRef.FILE.ref = FILE + appender.FILE.type = File + appender.FILE.name = FILE + appender.FILE.fileName = /stackable/log/opensearch/opensearch_server.json + appender.FILE.layout.type = OpenSearchJsonLayout + appender.FILE.layout.type_name = server +--- apiVersion: opensearch.stackable.tech/v1alpha1 kind: OpenSearchCluster metadata: @@ -15,37 +29,29 @@ spec: clusterConfig: vectorAggregatorConfigMapName: opensearch-vector-aggregator-discovery nodes: - config: - logging: - enableVectorAgent: true - containers: - opensearch: - console: - level: INFO - file: - level: INFO - loggers: - ROOT: - level: INFO - vector: - console: - level: INFO - file: - level: INFO - loggers: - ROOT: - level: INFO roleGroups: - cluster-manager: + automatic: config: - nodeRoles: - - cluster_manager - resources: - storage: - data: - capacity: 100Mi - listenerClass: external-stable - replicas: 3 + logging: + enableVectorAgent: true + containers: + opensearch: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + replicas: 1 podOverrides: spec: volumes: @@ -54,19 +60,16 @@ spec: volumeClaimTemplate: metadata: annotations: - secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-cluster-manager-headless - data: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-automatic-headless + custom: config: - nodeRoles: - - ingest - - data - - remote_cluster_client - resources: - storage: - data: - capacity: 2Gi - listenerClass: cluster-internal - replicas: 2 + logging: + enableVectorAgent: true + containers: + opensearch: + custom: + configMap: custom-log-config + replicas: 1 podOverrides: spec: volumes: @@ -75,13 +78,7 @@ spec: volumeClaimTemplate: metadata: annotations: - secrets.stackable.tech/scope: node,pod,service=opensearch-nodes-data-headless - envOverrides: - # Only required for the official image - # The official image (built with https://github.com/opensearch-project/opensearch-build) - # installs a demo configuration if not disabled explicitly. - DISABLE_INSTALL_DEMO_CONFIG: "true" - OPENSEARCH_HOME: {{ test_scenario['values']['opensearch_home'] }} + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-custom-headless configOverrides: opensearch.yml: # Disable memory mapping in this test; If memory mapping were activated, the kernel setting @@ -94,23 +91,23 @@ spec: cluster.routing.allocation.disk.threshold_enabled: "false" plugins.security.allow_default_init_securityindex: "true" plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.transport.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.transport.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.http.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.http.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt podOverrides: spec: containers: - name: opensearch volumeMounts: - name: security-config - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + mountPath: /stackable/opensearch/config/opensearch-security readOnly: true - name: tls - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + mountPath: /stackable/opensearch/config/tls readOnly: true volumes: - name: security-config diff --git a/tests/templates/kuttl/logging/30-assert.yaml b/tests/templates/kuttl/logging/30-assert.yaml index bebbaa9..cae2443 100644 --- a/tests/templates/kuttl/logging/30-assert.yaml +++ b/tests/templates/kuttl/logging/30-assert.yaml @@ -6,6 +6,6 @@ timeout: 600 apiVersion: batch/v1 kind: Job metadata: - name: test-opensearch + name: test-log-aggregation status: succeeded: 1 diff --git a/tests/templates/kuttl/logging/30-test-opensearch.yaml b/tests/templates/kuttl/logging/30-test-opensearch.yaml index b7cfe4c..d3753a2 100644 --- a/tests/templates/kuttl/logging/30-test-opensearch.yaml +++ b/tests/templates/kuttl/logging/30-test-opensearch.yaml @@ -2,31 +2,21 @@ apiVersion: batch/v1 kind: Job metadata: - name: test-opensearch + name: test-log-aggregation spec: template: spec: containers: - - name: test-opensearch + - name: test-log-aggregation image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + # TODO Extend the metrics Job or reduce this one? command: - - /bin/bash - - -euxo - - pipefail - - -c + - python args: - - | - pip install opensearch-py==3.0.0 - python scripts/test.py - env: - # required for pip install - - name: HOME - value: /stackable + - scripts/test.py volumeMounts: - name: script mountPath: /stackable/scripts - - name: tls - mountPath: /stackable/tls securityContext: allowPrivilegeEscalation: false capabilities: @@ -43,20 +33,7 @@ spec: volumes: - name: script configMap: - name: test-opensearch - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" + name: test-log-aggregation serviceAccountName: test-service-account securityContext: fsGroup: 1000 @@ -65,100 +42,53 @@ spec: apiVersion: v1 kind: ConfigMap metadata: - name: test-opensearch + name: test-log-aggregation data: test.py: | - # https://docs.opensearch.org/docs/latest/clients/python-low-level/#sample-program - - from opensearchpy import OpenSearch - - host = 'opensearch' - port = 9200 - auth = ('admin', 'AJVFsGJBbpT6mChn') # For testing only. Don't store credentials in code. - ca_certs_path = '/stackable/tls/ca.crt' - - # Create the client with SSL/TLS enabled, but hostname verification disabled. - client = OpenSearch( - hosts = [{'host': host, 'port': port}], - http_compress = True, # enables gzip compression for request bodies - http_auth = auth, - use_ssl = True, - verify_certs = True, - ssl_assert_hostname = False, - ssl_show_warn = False, - ca_certs = ca_certs_path - ) - - # Create an index with non-default settings. - index_name = 'python-test-index' - index_body = { - 'settings': { - 'index': { - 'number_of_shards': 4 - } - } - } - - response = client.indices.create(index=index_name, body=index_body) - print('\nCreating index:') - print(response) - - # Add a document to the index. - document = { - 'title': 'Moneyball', - 'director': 'Bennett Miller', - 'year': '2011' - } - id = '1' - - response = client.index( - index = index_name, - body = document, - id = id, - refresh = True - ) - - print('\nAdding document:') - print(response) - - # Perform bulk operations - - movies = '{ "index" : { "_index" : "my-dsl-index", "_id" : "2" } } \n { "title" : "Interstellar", "director" : "Christopher Nolan", "year" : "2014"} \n { "create" : { "_index" : "my-dsl-index", "_id" : "3" } } \n { "title" : "Star Trek Beyond", "director" : "Justin Lin", "year" : "2015"} \n { "update" : {"_id" : "3", "_index" : "my-dsl-index" } } \n { "doc" : {"year" : "2016"} }' - - client.bulk(body=movies) - - # Search for the document. - q = 'miller' - query = { - 'size': 5, - 'query': { - 'multi_match': { - 'query': q, - 'fields': ['title^2', 'director'] - } - } - } - - response = client.search( - body = query, - index = index_name - ) - print('\nSearch results:') - print(response) - - # Delete the document. - response = client.delete( - index = index_name, - id = id - ) - - print('\nDeleting document:') - print(response) - - # Delete the index. - response = client.indices.delete( - index = index_name - ) - - print('\nDeleting index:') - print(response) + import requests + + + def check_sent_events(): + response = requests.post( + 'http://opensearch-vector-aggregator:8686/graphql', + json={ + 'query': """ + { + transforms(first:100) { + nodes { + componentId + metrics { + sentEventsTotal { + sentEventsTotal + } + } + } + } + } + """ + } + ) + + assert response.status_code == 200, \ + 'Cannot access the API of the vector aggregator.' + + result = response.json() + + transforms = result['data']['transforms']['nodes'] + for transform in transforms: + sentEvents = transform['metrics']['sentEventsTotal'] + componentId = transform['componentId'] + + if componentId == 'filteredInvalidEvents': + assert sentEvents is None or \ + sentEvents['sentEventsTotal'] == 0, \ + 'Invalid log events were sent.' + else: + assert sentEvents is not None and \ + sentEvents['sentEventsTotal'] > 0, \ + f'No events were sent in "{componentId}".' + + + if __name__ == '__main__': + check_sent_events() + print('Test successful!') diff --git a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 index c5cf3ad..e527a8c 100644 --- a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 @@ -22,8 +22,6 @@ spec: capacity: 100Mi listenerClass: external-stable replicas: 3 - envOverrides: - OPENSEARCH_HOME: {{ test_scenario['values']['opensearch_home'] }} configOverrides: opensearch.yml: # Disable memory mapping in this test; If memory mapping were activated, the kernel setting @@ -36,23 +34,23 @@ spec: cluster.routing.allocation.disk.threshold_enabled: "false" plugins.security.allow_default_init_securityindex: "true" plugins.security.ssl.transport.enabled: "true" - plugins.security.ssl.transport.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.transport.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.transport.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt plugins.security.ssl.http.enabled: "true" - plugins.security.ssl.http.pemcert_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.crt - plugins.security.ssl.http.pemkey_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/tls.key - plugins.security.ssl.http.pemtrustedcas_filepath: {{ test_scenario['values']['opensearch_home'] }}/config/tls/ca.crt + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt podOverrides: spec: containers: - name: opensearch volumeMounts: - name: security-config - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security + mountPath: /stackable/opensearch/config/opensearch-security readOnly: true - name: tls - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls + mountPath: /stackable/opensearch/config/tls readOnly: true volumes: - name: security-config diff --git a/tests/templates/kuttl/metrics/30-check-metrics.yaml b/tests/templates/kuttl/metrics/30-check-metrics.yaml index 0aa4f4d..7a8b9ee 100644 --- a/tests/templates/kuttl/metrics/30-check-metrics.yaml +++ b/tests/templates/kuttl/metrics/30-check-metrics.yaml @@ -14,6 +14,7 @@ spec: - -euo - pipefail - -c + args: - > curl http://prometheus-operated:9090/api/v1/query?query=opensearch_cluster_nodes_number%7Bpod%3D%22opensearch-nodes-default-0%22%7D | jq --exit-status '.data.result[0].value[1] == "3"' diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index ca660d1..3d4c8d3 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -57,8 +57,54 @@ spec: topologyKey: kubernetes.io/hostname weight: 1 containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + - args: + - |- + + prepare_signal_handlers() + { + unset term_child_pid + unset term_kill_needed + trap 'handle_term_signal' TERM + } + + handle_term_signal() + { + if [ "${term_child_pid}" ]; then + kill -TERM "${term_child_pid}" 2>/dev/null + else + term_kill_needed="yes" + fi + } + + wait_for_termination() + { + set +e + term_child_pid=$1 + if [[ -v term_kill_needed ]]; then + kill -TERM "${term_child_pid}" 2>/dev/null + fi + wait ${term_child_pid} 2>/dev/null + trap - TERM + wait ${term_child_pid} 2>/dev/null + set -e + } + + rm -f /stackable/log/_vector/shutdown + prepare_signal_handlers + if command --search containerdebug >/dev/null 2>&1; then + containerdebug --output=/stackable/log/containerdebug-state.json --loop & + else + echo >&2 "containerdebug not installed; Proceed without it." + fi + ./opensearch-docker-entrypoint.sh & + wait_for_termination $! + mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -119,6 +165,8 @@ spec: name: data - mountPath: /stackable/listener name: listener + - mountPath: /stackable/log + name: log - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security name: security-config readOnly: true @@ -139,6 +187,9 @@ spec: defaultMode: 420 name: opensearch-nodes-cluster-manager name: log-config + - emptyDir: + sizeLimit: 30Mi + name: log - name: security-config secret: defaultMode: 420 @@ -253,8 +304,54 @@ spec: topologyKey: kubernetes.io/hostname weight: 1 containers: - - command: - - {{ test_scenario['values']['opensearch_home'] }}/opensearch-docker-entrypoint.sh + - args: + - |- + + prepare_signal_handlers() + { + unset term_child_pid + unset term_kill_needed + trap 'handle_term_signal' TERM + } + + handle_term_signal() + { + if [ "${term_child_pid}" ]; then + kill -TERM "${term_child_pid}" 2>/dev/null + else + term_kill_needed="yes" + fi + } + + wait_for_termination() + { + set +e + term_child_pid=$1 + if [[ -v term_kill_needed ]]; then + kill -TERM "${term_child_pid}" 2>/dev/null + fi + wait ${term_child_pid} 2>/dev/null + trap - TERM + wait ${term_child_pid} 2>/dev/null + set -e + } + + rm -f /stackable/log/_vector/shutdown + prepare_signal_handlers + if command --search containerdebug >/dev/null 2>&1; then + containerdebug --output=/stackable/log/containerdebug-state.json --loop & + else + echo >&2 "containerdebug not installed; Proceed without it." + fi + ./opensearch-docker-entrypoint.sh & + wait_for_termination $! + mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c env: - name: DISABLE_INSTALL_DEMO_CONFIG value: "true" @@ -314,6 +411,8 @@ spec: name: data - mountPath: /stackable/listener name: listener + - mountPath: /stackable/log + name: log - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/opensearch-security name: security-config readOnly: true @@ -334,6 +433,9 @@ spec: defaultMode: 420 name: opensearch-nodes-data name: log-config + - emptyDir: + sizeLimit: 30Mi + name: log - name: security-config secret: defaultMode: 420 diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index f2dea4a..94eb420 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,14 +6,12 @@ dimensions: # To use a custom image, add a comma and the full name after the product version, e.g.: # - 3.1.0,oci.stackable.tech/sandbox/opensearch:3.1.0-stackable0.0.0-dev # - 3.1.0,localhost:5000/sdp/opensearch:3.1.0-stackable0.0.0-dev - # - 3.1.0,opensearchproject/opensearch:3.1.0 - name: openshift values: - "false" - name: opensearch_home values: - - /stackable/opensearch # for the Stackable image - # - /usr/share/opensearch # for the official image + - /stackable/opensearch tests: - name: smoke dimensions: @@ -25,21 +23,21 @@ tests: - opensearch - openshift - opensearch_home + # requires an image with the OpenSearch Prometheus exporter - name: metrics dimensions: - opensearch - openshift - - opensearch_home - name: ldap dimensions: - opensearch - openshift - opensearch_home + # requires an image with Vector - name: logging dimensions: - opensearch - openshift - - opensearch_home suites: - name: nightly patch: @@ -61,3 +59,14 @@ suites: expr: "true" - name: opensearch expr: last + - name: original-image + select: + - smoke + - external-access + - ldap + patch: + - dimensions: + - name: opensearch + expr: 3.1.0,opensearchproject/opensearch:3.1.0 + - name: opensearch_home + expr: /usr/share/opensearch From dc3c371f32ca03f26795f810d18f03548405e751 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 13:44:18 +0200 Subject: [PATCH 05/14] chore: Fix TODOs and add unit tests --- rust/operator-binary/src/controller.rs | 3 +- .../src/controller/build/node_config.rs | 5 - .../build/product_logging/config.rs | 141 ++++--- .../controller/build/role_group_builder.rs | 34 +- rust/operator-binary/src/crd/mod.rs | 10 +- rust/operator-binary/src/framework.rs | 22 ++ .../src/framework/builder/pod/container.rs | 42 ++- .../framework/product_logging/framework.rs | 350 +++++++++++++++--- .../src/framework/validation.rs | 44 ++- tests/templates/kuttl/smoke/10-assert.yaml.j2 | 4 + 10 files changed, 528 insertions(+), 127 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8a29efc..832e488 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -124,7 +124,7 @@ type OpenSearchRoleGroupConfig = type OpenSearchNodeResources = stackable_operator::commons::resources::Resources; -/// The validated [`v1alpha1::OpenSearchConfig`] +/// Validated [`v1alpha1::OpenSearchConfig`] #[derive(Clone, Debug, PartialEq)] pub struct ValidatedOpenSearchConfig { pub affinity: StackableAffinity, @@ -135,6 +135,7 @@ pub struct ValidatedOpenSearchConfig { pub termination_grace_period_seconds: i64, } +/// Validated log configuration per container #[derive(Clone, Debug, PartialEq)] pub struct ValidatedLogging { pub opensearch_container: ValidatedContainerLogConfigChoice, diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index b11b2af..db795f7 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -493,11 +493,6 @@ mod tests { ..TestConfig::default() }); - println!( - "node_config_single_node: {:?}", - node_config_single_node.role_group_config.config - ); - let node_config_multiple_nodes = node_config(TestConfig { replicas: 2, ..TestConfig::default() diff --git a/rust/operator-binary/src/controller/build/product_logging/config.rs b/rust/operator-binary/src/controller/build/product_logging/config.rs index 2877ab1..c70694c 100644 --- a/rust/operator-binary/src/controller/build/product_logging/config.rs +++ b/rust/operator-binary/src/controller/build/product_logging/config.rs @@ -1,13 +1,21 @@ -use std::cmp; +//! OpenSearch specific log configuration + +use std::{cmp, collections::BTreeMap}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - product_logging::spec::AutomaticContainerLogConfig, + product_logging::spec::{AppenderConfig, AutomaticContainerLogConfig, LogLevel, LoggerConfig}, }; -use crate::{crd::v1alpha1, framework::product_logging::framework::STACKABLE_LOG_DIR}; +use crate::{ + crd::v1alpha1::{self}, + framework::{ + builder::pod::container::{EnvVarName, EnvVarSet}, + product_logging::framework::STACKABLE_LOG_DIR, + }, +}; -/// The log configuration file +/// OpenSearch log configuration file pub const CONFIGURATION_FILE_LOG4J2_PROPERTIES: &str = "log4j2.properties"; const OPENSEARCH_SERVER_LOG_FILE: &str = "opensearch_server.json"; @@ -17,24 +25,39 @@ pub const MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity unit: BinaryMultiple::Mebi, }; +/// Create a log4j2 configuration from the given automatic log configuration pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { - let log_path = format!( - "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", - container = v1alpha1::Container::OpenSearch.to_container_name() - ); + [ + log4j2_root_logger_config(&config.root_log_level()), + log4j2_loggers_config(&config.loggers), + log4j2_console_appender_config(&config.console), + log4j2_file_appender_config(&config.file), + ] + .iter() + .flatten() + .map(|(key, value)| format!("{key} = {value}\n")) + .collect() +} - let number_of_archived_log_files = 1; - let max_log_files_size_in_mib = MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32; - let max_log_file_size_in_mib = cmp::max( - 1, - max_log_files_size_in_mib / (1 + number_of_archived_log_files), - ); +fn log4j2_root_logger_config(root_log_level: &LogLevel) -> Vec<(String, String)> { + vec![ + ( + "rootLogger.level".to_owned(), + root_log_level.to_log4j2_literal(), + ), + ( + "rootLogger.appenderRef.CONSOLE.ref".to_owned(), + "CONSOLE".to_owned(), + ), + ( + "rootLogger.appenderRef.FILE.ref".to_owned(), + "FILE".to_owned(), + ), + ] +} - let loggers = config - .loggers +fn log4j2_loggers_config(loggers_config: &BTreeMap) -> Vec<(String, String)> { + loggers_config .iter() .filter(|(name, _)| name.as_str() != AutomaticContainerLogConfig::ROOT_LOGGER) .enumerate() @@ -50,24 +73,13 @@ pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { ), ] }) - .collect::>(); - - let root_logger = vec![ - ( - "rootLogger.level".to_owned(), - config.root_log_level().to_log4j2_literal(), - ), - ( - "rootLogger.appenderRef.CONSOLE.ref".to_owned(), - "CONSOLE".to_owned(), - ), - ( - "rootLogger.appenderRef.FILE.ref".to_owned(), - "FILE".to_owned(), - ), - ]; + .collect::>() +} - let console_appender = vec![ +fn log4j2_console_appender_config( + console_appender_config: &Option, +) -> Vec<(String, String)> { + vec![ ("appender.CONSOLE.type".to_owned(), "Console".to_owned()), ("appender.CONSOLE.name".to_owned(), "CONSOLE".to_owned()), ( @@ -90,16 +102,34 @@ pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { ), ( "appender.CONSOLE.filter.threshold.level".to_owned(), - config - .console + console_appender_config .as_ref() .and_then(|console| console.level) .unwrap_or_default() .to_log4j2_literal(), ), - ]; + ] +} + +fn log4j2_file_appender_config( + file_appender_config: &Option, +) -> Vec<(String, String)> { + let log_path = format!( + "{STACKABLE_LOG_DIR}/{container}/{OPENSEARCH_SERVER_LOG_FILE}", + container = v1alpha1::Container::OpenSearch.to_container_name() + ); + + let number_of_archived_log_files = 1; + let max_log_files_size_in_mib = MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32; + let max_log_file_size_in_mib = cmp::max( + 1, + max_log_files_size_in_mib / (1 + number_of_archived_log_files), + ); - let file_appender = vec![ + vec![ ("appender.FILE.type".to_owned(), "RollingFile".to_owned()), ("appender.FILE.name".to_owned(), "FILE".to_owned()), ("appender.FILE.fileName".to_owned(), log_path.to_owned()), @@ -141,33 +171,38 @@ pub fn create_log4j2_config(config: &AutomaticContainerLogConfig) -> String { ), ( "appender.FILE.filter.threshold.level".to_owned(), - config - .file + file_appender_config .as_ref() .and_then(|file| file.level) .unwrap_or_default() .to_log4j2_literal(), ), - ]; - - [root_logger, loggers, console_appender, file_appender] - .iter() - .flatten() - .map(|(key, value)| format!("{key} = {value}\n")) - .collect() + ] } +/// Returns the Vector configuration file content as YAML pub fn vector_config_file_content() -> String { include_str!("vector.yaml").to_owned() } +/// Returns the OpenSearch specific environment variables used in the Vector configuration file +/// +/// The common environment variables are already set in +/// [`crate::framework::product_logging::framework::vector_container`]. +pub fn vector_config_file_extra_env_vars() -> EnvVarSet { + EnvVarSet::new().with_value( + &EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), + "opensearch_server.json", + ) +} + #[cfg(test)] mod tests { use stackable_operator::product_logging::spec::{ AppenderConfig, AutomaticContainerLogConfig, LogLevel, LoggerConfig, }; - use super::create_log4j2_config; + use super::{create_log4j2_config, vector_config_file_extra_env_vars}; #[test] pub fn test_create_log4j2_config() { @@ -227,4 +262,10 @@ mod tests { assert_eq!(expected_config, log4j2_config); } + + #[test] + pub fn test_vector_config_file_extra_env_vars() { + // Test that the function does not panic + vector_config_file_extra_env_vars(); + } } diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index 3899240..4b42052 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -18,7 +18,7 @@ use stackable_operator::{ }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, - kvp::{Annotations, Label, Labels}, + kvp::{Annotation, Annotations, Label, Labels}, product_logging::framework::{ VECTOR_CONFIG_FILE, calculate_log_volume_size_limit, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, @@ -36,7 +36,9 @@ use crate::{ constant, controller::{ ContextNames, OpenSearchRoleGroupConfig, ValidatedCluster, - build::product_logging::config::MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, + build::product_logging::config::{ + MAX_OPENSEARCH_SERVER_LOG_FILES_SIZE, vector_config_file_extra_env_vars, + }, }, crd::v1alpha1, framework::{ @@ -217,6 +219,13 @@ impl<'a> RoleGroupBuilder<'a> { let metadata = ObjectMetaBuilder::new() .with_labels(self.recommended_labels()) .with_labels(node_role_labels) + .with_annotation( + Annotation::try_from(( + "kubectl.kubernetes.io/default-container".to_owned(), + v1alpha1::Container::OpenSearch.to_container_name(), + )) + .expect("should be a valid annotation"), + ) .build(); let opensearch_container = self.build_opensearch_container(); @@ -229,13 +238,12 @@ impl<'a> RoleGroupBuilder<'a> { .map(|vector_container_log_config| { vector_container( &v1alpha1::Container::Vector.to_container_name(), - vector_container_log_config, - &self.resource_names.cluster_name, - &self.resource_names.role_name, - &self.resource_names.role_group_name, &self.cluster.image, + vector_container_log_config, + &self.resource_names, &CONFIG_VOLUME_NAME, &LOG_VOLUME_NAME, + vector_config_file_extra_env_vars(), ) }); @@ -294,7 +302,6 @@ impl<'a> RoleGroupBuilder<'a> { .pod_anti_affinity .clone(), }), - // TODO Add annotation that the opensearch container is the main one containers: [Some(opensearch_container), vector_container] .into_iter() .flatten() @@ -768,7 +775,7 @@ mod tests { .expect("should be serializable"); // The content of log4j2.properties is already tested in the - // `conrtoller::build::product_logging::config` module. + // `controller::build::product_logging::config` module. config_map["data"]["log4j2.properties"].take(); // The content of opensearch.yml is already tested in the `controller::build::node_config` // module. @@ -860,6 +867,9 @@ mod tests { "serviceName": "my-opensearch-cluster-nodes-default-headless", "template": { "metadata": { + "annotations": { + "kubectl.kubernetes.io/default-container": "opensearch", + }, "labels": { "app.kubernetes.io/component": "nodes", "app.kubernetes.io/instance": "my-opensearch-cluster", @@ -912,8 +922,12 @@ mod tests { "\n", "rm -f /stackable/log/_vector/shutdown\n", "prepare_signal_handlers\n", + "if command --search containerdebug >/dev/null 2>&1; then\n", "containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n", - "/stackable/opensearch/opensearch-docker-entrypoint.sh &\n", + "else\n", + "echo >&2 \"containerdebug not installed; Proceed without it.\"\n", + "fi\n", + "./opensearch-docker-entrypoint.sh &\n", "wait_for_termination $!\n", "mkdir -p /stackable/log/_vector && touch /stackable/log/_vector/shutdown" ) @@ -1092,7 +1106,7 @@ mod tests { "volumeMounts": [ { "mountPath": "/stackable/config/vector.yaml", - "name": "log-config", + "name": "config", "readOnly": true, "subPath": "vector.yaml", }, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 28989b2..4696806 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -152,10 +152,13 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Duration, - /// This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + /// This field controls which + /// [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + /// is used to expose the HTTP communication. #[fragment_attrs(serde(default))] pub listener_class: ListenerClassName, + /// Logging configuration #[fragment_attrs(serde(default))] pub logging: Logging, @@ -169,7 +172,6 @@ pub mod versioned { pub resources: Resources, } - // TODO All derives required? #[derive( Clone, Debug, @@ -258,10 +260,10 @@ impl v1alpha1::OpenSearchConfig { graceful_shutdown_timeout: Some( Duration::from_str("2m").expect("should be a valid duration"), ), - // Defaults taken from the Helm chart, see - // https://github.com/opensearch-project/helm-charts/blob/opensearch-3.0.0/charts/opensearch/values.yaml#L16-L20 listener_class: Some(DEFAULT_LISTENER_CLASS.to_owned()), logging: product_logging::spec::default_logging(), + // Defaults taken from the Helm chart, see + // https://github.com/opensearch-project/helm-charts/blob/opensearch-3.0.0/charts/opensearch/values.yaml#L16-L20 node_roles: Some(NodeRoles(vec![ v1alpha1::NodeRole::ClusterManager, v1alpha1::NodeRole::Ingest, diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index e7e6b5c..3442b2e 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -101,6 +101,18 @@ pub trait NameIsValidLabelValue { /// Restricted string type with attributes like maximum length. /// /// Fully-qualified types are used to ease the import into other modules. +/// +/// # Examples +/// +/// ```rust +/// attributed_string_type! { +/// ConfigMapName, +/// "The name of a ConfigMap", +/// "opensearch-nodes-default", +/// (max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH), +/// is_rfc_1123_dns_subdomain_name +/// } +/// ``` #[macro_export(local_inner_macros)] macro_rules! attributed_string_type { ($name:ident, $description:literal, $example:literal $(, $attribute:tt)*) => { @@ -282,6 +294,16 @@ macro_rules! attributed_string_type { }; } +/// Use [`std::sync::LazyLock`] to define a static "constant" from a string. +/// +/// The string is converted into the given type with [`std::str::FromStr::from_str`]. +/// +/// # Examples +/// +/// ```rust +/// constant!(DATA_VOLUME_NAME: VolumeName = "data"); +/// constant!(pub CONFIG_VOLUME_NAME: VolumeName = "config"); +/// ``` #[macro_export(local_inner_macros)] macro_rules! constant { ($qualifier:vis $name:ident: $type:ident = $value:literal) => { diff --git a/rust/operator-binary/src/framework/builder/pod/container.rs b/rust/operator-binary/src/framework/builder/pod/container.rs index 2eb488b..a7a8ced 100644 --- a/rust/operator-binary/src/framework/builder/pod/container.rs +++ b/rust/operator-binary/src/framework/builder/pod/container.rs @@ -24,7 +24,7 @@ pub fn new_container_builder(container_name: &ContainerName) -> ContainerBuilder ContainerBuilder::new(container_name.as_ref()).expect("should be a valid container name") } -// TODO Use attributed_string_type instead? +// TODO Use attributed_string_type instead /// Validated environment variable name #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] pub struct EnvVarName(String); @@ -139,7 +139,7 @@ impl EnvVarSet { self } - /// Adds an environment variable with the given name and field path to this set + /// Adds an environment variable with the given ConfigMap key reference to this set /// /// An [`EnvVar`] with the same name is overridden. pub fn with_config_map_key_ref( @@ -180,10 +180,15 @@ mod tests { use stackable_operator::{ builder::pod::container::FieldPathEnvVar, - k8s_openapi::api::core::v1::{EnvVar, EnvVarSource, ObjectFieldSelector}, + k8s_openapi::api::core::v1::{ + ConfigMapKeySelector, EnvVar, EnvVarSource, ObjectFieldSelector, + }, }; use super::{EnvVarName, EnvVarSet}; + use crate::framework::{ + ConfigMapKey, ConfigMapName, ContainerName, builder::pod::container::new_container_builder, + }; #[test] fn test_envvarname_fromstr() { @@ -199,6 +204,12 @@ mod tests { assert!(EnvVarName::from_str("=").is_err()); } + #[test] + fn test_new_container_builder() { + // Test that the function does not panic + new_container_builder(&ContainerName::from_str_unsafe("valid-container-name")); + } + #[test] fn test_envvarname_format() { assert_eq!( @@ -327,4 +338,29 @@ mod tests { env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) ); } + + #[test] + fn test_envvarset_with_config_map_key_ref() { + let env_var_set = EnvVarSet::new().with_config_map_key_ref( + &EnvVarName::from_str_unsafe("ENV"), + &ConfigMapName::from_str_unsafe("config-map"), + &ConfigMapKey::from_str_unsafe("key"), + ); + + assert_eq!( + Some(&EnvVar { + name: "ENV".to_owned(), + value: None, + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + key: "key".to_owned(), + name: "config-map".to_owned(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + }), + env_var_set.get(&EnvVarName::from_str_unsafe("ENV")) + ); + } } diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs index bd29c5a..3bb61f2 100644 --- a/rust/operator-binary/src/framework/product_logging/framework.rs +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -18,31 +18,32 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ constant, framework::{ - ClusterName, ConfigMapKey, ConfigMapName, ContainerName, RoleGroupName, RoleName, - VolumeName, + ConfigMapKey, ConfigMapName, ContainerName, VolumeName, builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + role_group_utils, }, }; -// Public variant of `stackable_operator::product_logging::framework::STACKABLE_CONFIG_DIR` +// Copy of the private constant `stackable_operator::product_logging::framework::STACKABLE_CONFIG_DIR` const STACKABLE_CONFIG_DIR: &str = "/stackable/config"; -// Public variant of `stackable_operator::product_logging::framework::VECTOR_LOG_DIR` +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_LOG_DIR` const VECTOR_CONTROL_DIR: &str = "_vector"; -// Public variant of `stackable_operator::product_logging::framework::SHUTDOWN_FILE` +// Copy of the private constant `stackable_operator::product_logging::framework::SHUTDOWN_FILE` const SHUTDOWN_FILE: &str = "shutdown"; // Public variant of `stackable_operator::product_logging::framework::STACKABLE_LOG_DIR` +/// Directory where the logs are stored pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -// Public variant of `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_CM_KEY` -constant!(pub VECTOR_AGGREGATOR_CM_KEY: ConfigMapKey = "ADDRESS"); +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_CM_KEY` +constant!(VECTOR_AGGREGATOR_CM_KEY: ConfigMapKey = "ADDRESS"); -// Public variant of `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_ADDRESS` -constant!(pub VECTOR_AGGREGATOR_ENV_NAME: EnvVarName = "VECTOR_AGGREGATOR_ADDRESS"); +// Copy of the private constant `stackable_operator::product_logging::framework::VECTOR_AGGREGATOR_ADDRESS` +constant!(VECTOR_AGGREGATOR_ENV_NAME: EnvVarName = "VECTOR_AGGREGATOR_ADDRESS"); -#[derive(Snafu, Debug, EnumDiscriminants)] +#[derive(Debug, EnumDiscriminants, Snafu)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { #[snafu(display("failed to get the container log configuration"))] @@ -54,18 +55,25 @@ pub enum Error { type Result = std::result::Result; +/// Validated [`ContainerLogConfigChoice`] +/// +/// The ConfigMap name in the Custom variant is valid. #[derive(Clone, Debug, PartialEq)] pub enum ValidatedContainerLogConfigChoice { Automatic(AutomaticContainerLogConfig), Custom(ConfigMapName), } +/// Validated [`ContainerLogConfigChoice`] for the Vector container +/// +/// It includes the discovery ConfigMap name of the Vector aggregator. #[derive(Clone, Debug, PartialEq)] pub struct VectorContainerLogConfig { pub log_config: ValidatedContainerLogConfigChoice, pub vector_aggregator_config_map_name: ConfigMapName, } +/// Validates the log configuration of the container pub fn validate_logging_configuration_for_container( logging: &Logging, container: T, @@ -77,36 +85,35 @@ where .containers .get(&container) .and_then(|container_log_config| container_log_config.choice.as_ref()) - // This should never happen because a default configuration should have been set in - // `v1alpha1::OpenSearchConfig` for all containers. + // This should never happen because default configurations should have been set for all + // containers. .context(GetContainerLogConfigurationSnafu { container: container.to_string(), })?; let validated_container_log_config_choice = match container_log_config_choice { + ContainerLogConfigChoice::Automatic(automatic_log_config) => { + ValidatedContainerLogConfigChoice::Automatic(automatic_log_config.clone()) + } ContainerLogConfigChoice::Custom(CustomContainerLogConfig { custom: ConfigMapLogConfig { config_map }, }) => ValidatedContainerLogConfigChoice::Custom( ConfigMapName::from_str(config_map).context(ParseContainerNameSnafu)?, ), - ContainerLogConfigChoice::Automatic(automatic_log_config) => { - ValidatedContainerLogConfigChoice::Automatic(automatic_log_config.clone()) - } }; Ok(validated_container_log_config_choice) } -/// Builds the container for the [`PodTemplateSpec`] +/// Builds the Vector container for the [`PodTemplateSpec`] pub fn vector_container( container_name: &ContainerName, - vector_container_log_config: &VectorContainerLogConfig, - cluster_name: &ClusterName, - role_name: &RoleName, - role_group_name: &RoleGroupName, image: &ResolvedProductImage, + vector_container_log_config: &VectorContainerLogConfig, + resource_names: &role_group_utils::ResourceNames, log_config_volume_name: &VolumeName, log_volume_name: &VolumeName, + extra_env_vars: EnvVarSet, ) -> Container { let log_level = if let ValidatedContainerLogConfigChoice::Automatic(log_config) = &vector_container_log_config.log_config @@ -129,22 +136,23 @@ pub fn vector_container( }; let env_vars = EnvVarSet::new() - .with_value(&EnvVarName::from_str_unsafe("CLUSTER_NAME"), cluster_name) + .with_value( + &EnvVarName::from_str_unsafe("CLUSTER_NAME"), + &resource_names.cluster_name, + ) .with_value(&EnvVarName::from_str_unsafe("LOG_DIR"), "/stackable/log") .with_field_path( &EnvVarName::from_str_unsafe("NAMESPACE"), FieldPathEnvVar::Namespace, ) .with_value( - // TODO parameter - &EnvVarName::from_str_unsafe("OPENSEARCH_SERVER_LOG_FILE"), - "opensearch_server.json", + &EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), + &resource_names.role_group_name, ) .with_value( - &EnvVarName::from_str_unsafe("ROLE_GROUP_NAME"), - role_group_name, + &EnvVarName::from_str_unsafe("ROLE_NAME"), + &resource_names.role_name, ) - .with_value(&EnvVarName::from_str_unsafe("ROLE_NAME"), role_name) .with_config_map_key_ref( &VECTOR_AGGREGATOR_ENV_NAME, &vector_container_log_config.vector_aggregator_config_map_name, @@ -161,7 +169,8 @@ pub fn vector_container( .with_value( &EnvVarName::from_str_unsafe("VECTOR_LOG"), log_level.to_vector_literal(), - ); + ) + .merge(extra_env_vars); let resources = ResourceRequirementsBuilder::new() .with_cpu_request("250m") @@ -212,27 +221,264 @@ pub fn vector_container( .build() } -// #[test] -// fn test_validate_err_parse_container_name() { -// test_validate_err( -// |cluster| { -// cluster.spec.nodes.config.config.logging = LoggingFragment { -// enable_vector_agent: Some(true), -// containers: [( -// v1alpha1::Container::OpenSearch, -// ContainerLogConfigFragment { -// choice: Some(ContainerLogConfigChoiceFragment::Custom( -// CustomContainerLogConfigFragment { -// custom: ConfigMapLogConfigFragment { -// config_map: Some("invalid ConfigMap name".to_owned()), -// }, -// }, -// )), -// }, -// )] -// .into(), -// } -// }, -// ErrorDiscriminants::ParseContainerName, -// ); -// } +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use pretty_assertions::assert_eq; + use serde_json::json; + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + kvp::LabelValue, + product_logging::spec::{ + AutomaticContainerLogConfig, ConfigMapLogConfig, ContainerLogConfig, + ContainerLogConfigChoice, CustomContainerLogConfig, Logging, + }, + }; + + use super::{ + ErrorDiscriminants, ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, vector_container, + }; + use crate::framework::{ + ClusterName, ConfigMapName, ContainerName, RoleGroupName, RoleName, VolumeName, + builder::pod::container::{EnvVarName, EnvVarSet}, + role_group_utils, + }; + + #[test] + fn test_validate_logging_configuration_for_container_ok_automatic_log_config() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }, + )] + .into(), + }; + + assert_eq!( + ValidatedContainerLogConfigChoice::Automatic(AutomaticContainerLogConfig::default()), + validate_logging_configuration_for_container(&logging, "container") + .expect("should be a valid log config") + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_ok_custom_log_config() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: "valid-config-map-name".to_owned(), + }, + })), + }, + )] + .into(), + }; + + assert_eq!( + ValidatedContainerLogConfigChoice::Custom(ConfigMapName::from_str_unsafe( + "valid-config-map-name" + )), + validate_logging_configuration_for_container(&logging, "container") + .expect("should be a valid log config") + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_err_get_container_log_configuration() { + let logging_without_container = Logging { + enable_vector_agent: false, + containers: [].into(), + }; + let logging_without_container_log_config_choice = Logging { + enable_vector_agent: false, + containers: [("container", ContainerLogConfig { choice: None })].into(), + }; + + assert_eq!( + Err(ErrorDiscriminants::GetContainerLogConfiguration), + validate_logging_configuration_for_container(&logging_without_container, "container") + .map_err(ErrorDiscriminants::from) + ); + + assert_eq!( + Err(ErrorDiscriminants::GetContainerLogConfiguration), + validate_logging_configuration_for_container( + &logging_without_container_log_config_choice, + "container" + ) + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_validate_logging_configuration_for_container_err_parse_container_name() { + let logging = Logging { + enable_vector_agent: false, + containers: [( + "container", + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: "invalid ConfigMap name".to_owned(), + }, + })), + }, + )] + .into(), + }; + + assert_eq!( + Err(ErrorDiscriminants::ParseContainerName), + validate_logging_configuration_for_container(&logging, "container") + .map_err(ErrorDiscriminants::from) + ); + } + + #[test] + fn test_vector_container() { + let image = ResolvedProductImage { + product_version: "1.0.0".to_owned(), + app_version_label_value: LabelValue::from_str("1.0.0-stackable0.0.0-dev") + .expect("should be a valid label value"), + image: "oci.stackable.tech/sdp/product:1.0.0-stackable0.0.0-dev".to_string(), + image_pull_policy: "Always".to_owned(), + pull_secrets: None, + }; + + let vector_container_log_config = VectorContainerLogConfig { + log_config: ValidatedContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + ), + vector_aggregator_config_map_name: ConfigMapName::from_str_unsafe("vector-aggregator"), + }; + + let resource_names = role_group_utils::ResourceNames { + cluster_name: ClusterName::from_str_unsafe("test-cluster"), + role_name: RoleName::from_str_unsafe("role"), + role_group_name: RoleGroupName::from_str_unsafe("role-group"), + }; + + let vector_container = vector_container( + &ContainerName::from_str_unsafe("vector"), + &image, + &vector_container_log_config, + &resource_names, + &VolumeName::from_str_unsafe("config"), + &VolumeName::from_str_unsafe("log"), + EnvVarSet::new().with_value(&EnvVarName::from_str_unsafe("CUSTOM_ENV"), "test"), + ); + + assert_eq!( + json!( + { + "args": [ + concat!( + "# Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file\n", + "vector & vector_pid=$!\n", + "if [ ! -f \"/stackable/log/_vector/shutdown\" ]; then\n", + "mkdir -p /stackable/log/_vector\n", + "inotifywait -qq --event create /stackable/log/_vector;\n", + "fi\n", + "sleep 1\n", + "kill $vector_pid" + ), + ], + "command": [ + "/bin/bash", + "-x", + "-euo", + "pipefail", + "-c", + ], + "env": [ + { + "name": "CLUSTER_NAME", + "value": "test-cluster", + }, + { + "name": "CUSTOM_ENV", + "value": "test", + }, + { + "name": "LOG_DIR", + "value": "/stackable/log", + }, + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace", + }, + }, + }, + { + "name": "ROLE_GROUP_NAME", + "value": "role-group", + }, + { + "name": "ROLE_NAME", + "value": "role", + }, + { + "name": "VECTOR_AGGREGATOR_ADDRESS", + "valueFrom": { + "configMapKeyRef": { + "key": "ADDRESS", + "name": "vector-aggregator", + }, + }, + }, + { + "name": "VECTOR_CONFIG_YAML", + "value": "/stackable/config/vector.yaml", + }, + { + "name": "VECTOR_FILE_LOG_LEVEL", + "value": "info", + }, + { + "name": "VECTOR_LOG", + "value": "info", + }, + ], + "image": "oci.stackable.tech/sdp/product:1.0.0-stackable0.0.0-dev", + "imagePullPolicy": "Always", + "name": "vector", + "resources": { + "limits": { + "cpu": "500m", + "memory": "128Mi", + }, + "requests": { + "cpu": "250m", + "memory": "128Mi", + }, + }, + "volumeMounts": [ + { + "mountPath": "/stackable/config/vector.yaml", + "name": "config", + "readOnly": true, + "subPath": "vector.yaml", + }, + { + "mountPath": "/stackable/log", + "name": "log", + }, + ], + }), + serde_json::to_value(vector_container).expect("should be serializable") + ); + } +} diff --git a/rust/operator-binary/src/framework/validation.rs b/rust/operator-binary/src/framework/validation.rs index f92e804..b4e3a20 100644 --- a/rust/operator-binary/src/framework/validation.rs +++ b/rust/operator-binary/src/framework/validation.rs @@ -12,8 +12,11 @@ static CONFIG_MAP_KEY_REGEX: LazyLock = LazyLock::new(|| { Regex::new(&format!("^{CONFIG_MAP_KEY_FMT}$")).expect("failed to compile ConfigMap key regex") }); -#[derive(Debug, Snafu)] +#[derive(Debug, Eq, PartialEq, Snafu)] pub enum Error { + #[snafu(display("value is empty"))] + Empty { value: String }, + #[snafu(display("value does not match the regular expression"))] Regex { value: String, @@ -34,9 +37,11 @@ pub fn is_config_map_key(value: &str) -> Result { // When adding this function to stackable_operator, use the private functions like // validate_all. + ensure!(!value.is_empty(), EmptySnafu { value }); + let max_length = RFC_1123_SUBDOMAIN_MAX_LENGTH; ensure!( - value.len() < max_length, + value.len() <= max_length, TooLongSnafu { value: value.to_owned(), max_length @@ -54,3 +59,38 @@ pub fn is_config_map_key(value: &str) -> Result { Ok(()) } + +#[cfg(test)] +mod tests { + use super::{CONFIG_MAP_KEY_ERROR_MSG, CONFIG_MAP_KEY_FMT, Error, is_config_map_key}; + + #[test] + fn test_is_config_map_key() { + assert_eq!(Ok(()), is_config_map_key("_a-A.1")); + + assert_eq!( + Err(Error::Empty { + value: "".to_owned() + }), + is_config_map_key("") + ); + + assert_eq!(Ok(()), is_config_map_key(&"a".repeat(253))); + assert_eq!( + Err(Error::TooLong { + value: "a".repeat(254), + max_length: 253 + }), + is_config_map_key(&"a".repeat(254)) + ); + + assert_eq!( + Err(Error::Regex { + value: " ".to_string(), + regex: CONFIG_MAP_KEY_FMT, + message: CONFIG_MAP_KEY_ERROR_MSG, + }), + is_config_map_key(" ") + ); + } +} diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 3d4c8d3..5bc3bd7 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -35,6 +35,8 @@ spec: serviceName: opensearch-nodes-cluster-manager-headless template: metadata: + annotations: + kubectl.kubernetes.io/default-container: opensearch labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch @@ -280,6 +282,8 @@ spec: serviceName: opensearch-nodes-data-headless template: metadata: + annotations: + kubectl.kubernetes.io/default-container: opensearch labels: app.kubernetes.io/component: nodes app.kubernetes.io/instance: opensearch From b575899413291d8d8a0c8112ab8b341bd1741d4b Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 14:24:25 +0200 Subject: [PATCH 06/14] chore: Run pre-commit --- deploy/helm/opensearch-operator/crds/crds.yaml | 10 ++++++++-- .../controller/build/product_logging/vector-test.yaml | 4 ++-- .../src/controller/build/product_logging/vector.yaml | 1 + rust/operator-binary/src/crd/mod.rs | 8 ++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/deploy/helm/opensearch-operator/crds/crds.yaml b/deploy/helm/opensearch-operator/crds/crds.yaml index ce589e7..12f3da0 100644 --- a/deploy/helm/opensearch-operator/crds/crds.yaml +++ b/deploy/helm/opensearch-operator/crds/crds.yaml @@ -184,7 +184,10 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + description: |- + This field controls which + [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the HTTP communication. maxLength: 253 minLength: 1 nullable: true @@ -529,7 +532,10 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the HTTP communication. + description: |- + This field controls which + [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the HTTP communication. maxLength: 253 minLength: 1 nullable: true diff --git a/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml index 162868b..7ffa2cd 100644 --- a/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/product_logging/vector-test.yaml @@ -2,7 +2,7 @@ # # A downside of these test cases is that they compare the whole event and that the message can # contain source code positions in vector.yaml, e.g. "function call error for \"parse_timestamp\" at (584:643)". Please adapt the tests if you change VRL code in vector.yaml. - +--- tests: - name: Test opensearch_server log entry without stacktrace inputs: @@ -42,7 +42,7 @@ tests: log_fields: file: /stackable/log/opensearch/opensearch_server.json message: | - {"type": "server", "timestamp": "2025-10-01T12:47:28,363Z", "level": "INFO", "component": "o.o.c.c.JoinHelper", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}", + {"type": "server", "timestamp": "2025-10-01T12:47:28,363Z", "level": "INFO", "component": "o.o.c.c.JoinHelper", "cluster.name": "opensearch", "node.name": "opensearch-nodes-cluster-manager-0", "message": "failed to join {opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true} with JoinRequest{sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, minimumTerm=0, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}, targetNode={opensearch-nodes-cluster-manager-0}{sk-r0P_TTYuPqaamTFbjKg}{fIRSMbQYSe2nKQZ_sPn4kg}{10.244.0.20}{10.244.0.20:9300}{m}{shard_indexing_pressure_enabled=true}}]}", "stacktrace": ["org.opensearch.transport.RemoteTransportException: [opensearch-nodes-cluster-manager-0][10.244.0.20:9300][internal:cluster/coordination/join]", "Caused by: org.opensearch.cluster.coordination.CoordinationStateRejectedException: became follower", "at org.opensearch.cluster.coordination.JoinHelper$$CandidateJoinAccumulator.lambda$$close$$3(JoinHelper.java:648) ~[opensearch-3.1.0.jar:3.1.0]", diff --git a/rust/operator-binary/src/controller/build/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/product_logging/vector.yaml index dc33be7..c01ac7c 100644 --- a/rust/operator-binary/src/controller/build/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/product_logging/vector.yaml @@ -1,3 +1,4 @@ +--- data_dir: /stackable/vector/var log_schema: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 4696806..4803e70 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -61,18 +61,18 @@ pub mod versioned { ))] #[serde(rename_all = "camelCase")] pub struct OpenSearchClusterSpec { - // no doc - docs in ProductImage struct. + // no doc - docs in ProductImage struct pub image: ProductImage, /// Configuration that applies to all roles and role groups #[serde(default)] pub cluster_config: v1alpha1::OpenSearchClusterConfig, - // no doc - docs in ClusterOperation struct. + // no doc - docs in ClusterOperation struct #[serde(default)] pub cluster_operation: ClusterOperation, - // no doc - docs in Role struct. + // no doc - docs in Role struct pub nodes: Role, } @@ -158,7 +158,7 @@ pub mod versioned { #[fragment_attrs(serde(default))] pub listener_class: ListenerClassName, - /// Logging configuration + // no doc - docs in Logging struct #[fragment_attrs(serde(default))] pub logging: Logging, From 6e1f209603fdb3466d0c1a3d60fee2f8698a2bd8 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 15:09:17 +0200 Subject: [PATCH 07/14] doc: Add logging documentation --- .../opensearch/pages/usage-guide/logging.adoc | 34 +++++++++++++++++++ docs/modules/opensearch/partials/nav.adoc | 1 + 2 files changed, 35 insertions(+) create mode 100644 docs/modules/opensearch/pages/usage-guide/logging.adoc diff --git a/docs/modules/opensearch/pages/usage-guide/logging.adoc b/docs/modules/opensearch/pages/usage-guide/logging.adoc new file mode 100644 index 0000000..d9d4594 --- /dev/null +++ b/docs/modules/opensearch/pages/usage-guide/logging.adoc @@ -0,0 +1,34 @@ += Log aggregation +:description: The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent. + +The logs can be forwarded to a Vector log aggregator by providing a discovery ConfigMap for the aggregator and by enabling the log agent: + +[source,yaml] +---- +spec: + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery + nodes: + config: + logging: + enableVectorAgent: true + containers: + opensearch: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO + vector: + console: + level: INFO + file: + level: INFO + loggers: + ROOT: + level: INFO +---- + +Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/docs/modules/opensearch/partials/nav.adoc b/docs/modules/opensearch/partials/nav.adoc index 42649b5..159b56d 100644 --- a/docs/modules/opensearch/partials/nav.adoc +++ b/docs/modules/opensearch/partials/nav.adoc @@ -7,6 +7,7 @@ ** xref:opensearch:usage-guide/storage-resource-configuration.adoc[] ** xref:opensearch:usage-guide/configuration-environment-overrides.adoc[] ** xref:opensearch:usage-guide/monitoring.adoc[] +** xref:opensearch:usage-guide/logging.adoc[] ** xref:opensearch:usage-guide/operations/index.adoc[] *** xref:opensearch:usage-guide/operations/cluster-operations.adoc[] *** xref:opensearch:usage-guide/operations/pod-placement.adoc[] From 5a755426c60c25c2004d850904787eb298def279 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 16:07:28 +0200 Subject: [PATCH 08/14] test: Tidy up integration tests --- .../templates/kuttl/ldap/11-create-ldap-user.yaml | 2 ++ .../kuttl/logging/30-test-opensearch.yaml | 1 - .../templates/kuttl/metrics/30-check-metrics.yaml | 15 +++++++++++++++ tests/test-definition.yaml | 10 ---------- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/templates/kuttl/ldap/11-create-ldap-user.yaml b/tests/templates/kuttl/ldap/11-create-ldap-user.yaml index 455e62d..0d0a016 100644 --- a/tests/templates/kuttl/ldap/11-create-ldap-user.yaml +++ b/tests/templates/kuttl/ldap/11-create-ldap-user.yaml @@ -78,6 +78,8 @@ spec: requests: storage: "1" serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 restartPolicy: OnFailure --- apiVersion: v1 diff --git a/tests/templates/kuttl/logging/30-test-opensearch.yaml b/tests/templates/kuttl/logging/30-test-opensearch.yaml index d3753a2..ee9f1f4 100644 --- a/tests/templates/kuttl/logging/30-test-opensearch.yaml +++ b/tests/templates/kuttl/logging/30-test-opensearch.yaml @@ -9,7 +9,6 @@ spec: containers: - name: test-log-aggregation image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - # TODO Extend the metrics Job or reduce this one? command: - python args: diff --git a/tests/templates/kuttl/metrics/30-check-metrics.yaml b/tests/templates/kuttl/metrics/30-check-metrics.yaml index 7a8b9ee..489c77c 100644 --- a/tests/templates/kuttl/metrics/30-check-metrics.yaml +++ b/tests/templates/kuttl/metrics/30-check-metrics.yaml @@ -18,5 +18,20 @@ spec: - > curl http://prometheus-operated:9090/api/v1/query?query=opensearch_cluster_nodes_number%7Bpod%3D%22opensearch-nodes-default-0%22%7D | jq --exit-status '.data.result[0].value[1] == "3"' + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 restartPolicy: OnFailure diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 94eb420..6e18a83 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,9 +6,6 @@ dimensions: # To use a custom image, add a comma and the full name after the product version, e.g.: # - 3.1.0,oci.stackable.tech/sandbox/opensearch:3.1.0-stackable0.0.0-dev # - 3.1.0,localhost:5000/sdp/opensearch:3.1.0-stackable0.0.0-dev - - name: openshift - values: - - "false" - name: opensearch_home values: - /stackable/opensearch @@ -16,28 +13,23 @@ tests: - name: smoke dimensions: - opensearch - - openshift - opensearch_home - name: external-access dimensions: - opensearch - - openshift - opensearch_home # requires an image with the OpenSearch Prometheus exporter - name: metrics dimensions: - opensearch - - openshift - name: ldap dimensions: - opensearch - - openshift - opensearch_home # requires an image with Vector - name: logging dimensions: - opensearch - - openshift suites: - name: nightly patch: @@ -55,8 +47,6 @@ suites: - dimensions: - expr: last - dimensions: - - name: openshift - expr: "true" - name: opensearch expr: last - name: original-image From 068c4af08eceb266f73bbf8118bc385bd44b5784 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 17:28:57 +0200 Subject: [PATCH 09/14] chore: Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7880e08..8a2fce9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. images can be used which have a different directory structure than the Stackable image ([#18]). - Add Prometheus labels and annotations to role-group services ([#26]). - Helm: Allow Pod `priorityClassName` to be configured ([#34]). +- Support log configuration and log aggregation ([#40]). [#10]: https://github.com/stackabletech/opensearch-operator/pull/10 [#17]: https://github.com/stackabletech/opensearch-operator/pull/17 @@ -32,3 +33,4 @@ All notable changes to this project will be documented in this file. [#26]: https://github.com/stackabletech/opensearch-operator/pull/26 [#34]: https://github.com/stackabletech/opensearch-operator/pull/34 [#38]: https://github.com/stackabletech/opensearch-operator/pull/38 +[#40]: https://github.com/stackabletech/opensearch-operator/pull/40 From 0697a44432ad3ea78827611b5a79334f538fba1b Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 9 Oct 2025 22:40:33 +0200 Subject: [PATCH 10/14] fix: Fix rustdoc error --- rust/operator-binary/src/framework/product_logging/framework.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/framework/product_logging/framework.rs b/rust/operator-binary/src/framework/product_logging/framework.rs index 3bb61f2..1b49109 100644 --- a/rust/operator-binary/src/framework/product_logging/framework.rs +++ b/rust/operator-binary/src/framework/product_logging/framework.rs @@ -105,7 +105,7 @@ where Ok(validated_container_log_config_choice) } -/// Builds the Vector container for the [`PodTemplateSpec`] +/// Builds the Vector container pub fn vector_container( container_name: &ContainerName, image: &ResolvedProductImage, From 10afcd47cf38bc9bd07af817bf79cddeb994a891 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 10 Oct 2025 13:39:39 +0200 Subject: [PATCH 11/14] test: Add Vector aggregator discovery ConfigMaps to the integration tests --- .../templates/kuttl/external-access/02-assert.yaml.j2 | 10 ++++++++++ ...tall-vector-aggregator-discovery-config-map.yaml.j2 | 9 +++++++++ .../templates/kuttl/external-access/opensearch.yaml.j2 | 4 ++++ tests/templates/kuttl/ldap/02-assert.yaml.j2 | 10 ++++++++++ ...tall-vector-aggregator-discovery-config-map.yaml.j2 | 9 +++++++++ .../templates/kuttl/ldap/21-install-opensearch.yaml.j2 | 4 ++++ tests/templates/kuttl/metrics/02-assert.yaml.j2 | 10 ++++++++++ ...tall-vector-aggregator-discovery-config-map.yaml.j2 | 9 +++++++++ .../kuttl/metrics/20-install-opensearch.yaml.j2 | 4 ++++ tests/templates/kuttl/smoke/03-assert.yaml.j2 | 10 ++++++++++ ...tall-vector-aggregator-discovery-config-map.yaml.j2 | 9 +++++++++ .../kuttl/smoke/10-install-opensearch.yaml.j2 | 4 ++++ 12 files changed, 92 insertions(+) create mode 100644 tests/templates/kuttl/external-access/02-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 create mode 100644 tests/templates/kuttl/ldap/02-assert.yaml.j2 create mode 100644 tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 create mode 100644 tests/templates/kuttl/metrics/02-assert.yaml.j2 create mode 100644 tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 create mode 100644 tests/templates/kuttl/smoke/03-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 diff --git a/tests/templates/kuttl/external-access/02-assert.yaml.j2 b/tests/templates/kuttl/external-access/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/external-access/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/external-access/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/opensearch.yaml.j2 b/tests/templates/kuttl/external-access/opensearch.yaml.j2 index 3dc66a3..bf04bae 100644 --- a/tests/templates/kuttl/external-access/opensearch.yaml.j2 +++ b/tests/templates/kuttl/external-access/opensearch.yaml.j2 @@ -12,6 +12,10 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: roleGroups: cluster-manager: diff --git a/tests/templates/kuttl/ldap/02-assert.yaml.j2 b/tests/templates/kuttl/ldap/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/ldap/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/ldap/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 index 79c13fb..dab3886 100644 --- a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 @@ -12,6 +12,10 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: roleGroups: default: diff --git a/tests/templates/kuttl/metrics/02-assert.yaml.j2 b/tests/templates/kuttl/metrics/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/metrics/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/metrics/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 index e527a8c..3187c03 100644 --- a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 @@ -12,6 +12,10 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: roleGroups: default: diff --git a/tests/templates/kuttl/smoke/03-assert.yaml.j2 b/tests/templates/kuttl/smoke/03-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/smoke/03-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/smoke/03-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 index 1ee05fc..772c3e7 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 @@ -12,6 +12,10 @@ spec: productVersion: "{{ test_scenario['values']['opensearch'] }}" {% endif %} pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} nodes: roleGroups: cluster-manager: From 8393709ada417b517179146fab21e582313da753 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 10 Oct 2025 16:20:07 +0200 Subject: [PATCH 12/14] test: Fix assertion --- tests/templates/kuttl/smoke/10-assert.yaml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 5bc3bd7..95e1bac 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -732,7 +732,7 @@ metadata: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: cluster-manager - app.kubernetes.io/version: 3.1.0 + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} stackable.tech/vendor: Stackable name: opensearch-nodes-cluster-manager ownerReferences: @@ -758,7 +758,7 @@ metadata: app.kubernetes.io/managed-by: opensearch.stackable.tech_opensearchcluster app.kubernetes.io/name: opensearch app.kubernetes.io/role-group: data - app.kubernetes.io/version: 3.1.0 + app.kubernetes.io/version: {{ test_scenario['values']['opensearch'].split(',')[0] }} stackable.tech/vendor: Stackable name: opensearch-nodes-data ownerReferences: From 7a0f32b01a109622ad4eb262964b86a09752e740 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 10 Oct 2025 17:07:28 +0200 Subject: [PATCH 13/14] test: Enable the Vector agent in all integration tests --- tests/templates/kuttl/external-access/opensearch.yaml.j2 | 3 +++ tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 | 3 +++ tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 | 3 +++ tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tests/templates/kuttl/external-access/opensearch.yaml.j2 b/tests/templates/kuttl/external-access/opensearch.yaml.j2 index bf04bae..484627d 100644 --- a/tests/templates/kuttl/external-access/opensearch.yaml.j2 +++ b/tests/templates/kuttl/external-access/opensearch.yaml.j2 @@ -17,6 +17,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: cluster-manager: config: diff --git a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 index dab3886..5f0b615 100644 --- a/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/ldap/21-install-opensearch.yaml.j2 @@ -17,6 +17,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 3 diff --git a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 index 3187c03..d6e2d91 100644 --- a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 @@ -17,6 +17,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: config: diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 index 772c3e7..56553df 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 @@ -17,6 +17,9 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: cluster-manager: config: From 96a52ca2aeec04e36a5dffe7bb9a78f632cac7fb Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 10 Oct 2025 17:59:11 +0200 Subject: [PATCH 14/14] test: Fix assertion --- tests/templates/kuttl/smoke/10-assert.yaml.j2 | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 95e1bac..c128217 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -175,6 +175,70 @@ spec: - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls name: tls readOnly: true +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - args: + - |- + # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file + vector & vector_pid=$! + if [ ! -f "/stackable/log/_vector/shutdown" ]; then + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi + sleep 1 + kill $vector_pid + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + env: + - name: CLUSTER_NAME + value: opensearch + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: OPENSEARCH_SERVER_LOG_FILE + value: opensearch_server.json + - name: ROLE_GROUP_NAME + value: cluster-manager + - name: ROLE_NAME + value: nodes + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: vector + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} securityContext: fsGroup: 1000 serviceAccount: opensearch-serviceaccount @@ -423,6 +487,70 @@ spec: - mountPath: {{ test_scenario['values']['opensearch_home'] }}/config/tls name: tls readOnly: true +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + - args: + - |- + # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file + vector & vector_pid=$! + if [ ! -f "/stackable/log/_vector/shutdown" ]; then + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi + sleep 1 + kill $vector_pid + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + env: + - name: CLUSTER_NAME + value: opensearch + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: OPENSEARCH_SERVER_LOG_FILE + value: opensearch_server.json + - name: ROLE_GROUP_NAME + value: data + - name: ROLE_NAME + value: nodes + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + name: vector + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} securityContext: fsGroup: 1000 serviceAccount: opensearch-serviceaccount