From ac8a646fbdecc09a51c590ce25f670dbe424f9c3 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Sat, 15 Feb 2025 19:07:44 +0100 Subject: [PATCH 01/76] api changes Signed-off-by: Kateryna Nezdolii --- .bazelrc | 13 +- api/BUILD | 1 + api/envoy/data/ip_tagging/v3/BUILD | 12 + api/envoy/data/ip_tagging/v3/ip_tagging.proto | 34 + .../filters/http/ip_tagging/v3/BUILD | 2 +- .../http/ip_tagging/v3/ip_tagging.proto | 31 +- api/versioning/BUILD | 1 + bazel/envoy_internal.bzl | 1 + envoy/registry/registry.h | 2 +- .../extensions/filters/http/ip_tagging/BUILD | 2 + .../filters/http/ip_tagging/config.cc | 9 +- .../filters/http/ip_tagging/config.h | 2 + .../http/ip_tagging/ip_tagging_filter.cc | 103 ++- .../http/ip_tagging/ip_tagging_filter.h | 54 +- test/extensions/filters/http/ip_tagging/BUILD | 6 + .../http/ip_tagging/ip_tagging_filter_test.cc | 843 ++++++++++-------- .../filters/http/ip_tagging/test_data/BUILD | 13 + .../test_data/ip_tags_internal_request.yaml | 4 + 18 files changed, 701 insertions(+), 432 deletions(-) create mode 100644 api/envoy/data/ip_tagging/v3/BUILD create mode 100644 api/envoy/data/ip_tagging/v3/ip_tagging.proto create mode 100644 test/extensions/filters/http/ip_tagging/test_data/BUILD create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml diff --git a/.bazelrc b/.bazelrc index 180ef0d0e4e5a..11a6ab188d33a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -64,6 +64,7 @@ build:linux --copt=-fdebug-types-section # (Workaround for https://github.com/bazelbuild/rules_foreign_cc/issues/421) build:linux --copt=-fPIC build:linux --copt=-Wno-deprecated-declarations +build:linux --copt=-Wno-implicit-function-declaration build:linux --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 build:linux --cxxopt=-fsized-deallocation --host_cxxopt=-fsized-deallocation build:linux --conlyopt=-fexceptions @@ -84,7 +85,8 @@ build:sanitizer --linkopt -ldl # Common flags for Clang build:clang --action_env=BAZEL_COMPILER=clang -build:clang --linkopt=-fuse-ld=lld +build:clang --linkopt=-fuse-ld=ld +build:clang --copt=-Wno-implicit-function-declaration build:clang --action_env=CC=clang --host_action_env=CC=clang build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ build:clang --incompatible_enable_cc_toolchain_resolution=false @@ -148,7 +150,7 @@ build:asan --copt -fno-optimize-sibling-calls # Clang ASAN/UBSAN build:clang-asan-common --config=clang build:clang-asan-common --config=asan -build:clang-asan-common --linkopt -fuse-ld=lld +build:clang-asan-common --linkopt -fuse-ld=ld build:clang-asan-common --linkopt --rtlib=compiler-rt build:clang-asan-common --linkopt --unwindlib=libgcc @@ -162,6 +164,7 @@ build:clang-asan --linkopt=-fsanitize=vptr,function # macOS build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 build:macos --copt=-Wno-deprecated-declarations +build:macos --copt=-Wno-implicit-function-declaration build:macos --action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin build:macos --host_action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin build:macos --define tcmalloc=disabled @@ -182,7 +185,7 @@ build:clang-tsan --config=sanitizer build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread -build:clang-tsan --linkopt -fuse-ld=lld +build:clang-tsan --linkopt -fuse-ld=ld build:clang-tsan --copt -DTHREAD_SANITIZER=1 build:clang-tsan --build_tag_filters=-no_san,-no_tsan build:clang-tsan --test_tag_filters=-no_san,-no_tsan @@ -203,7 +206,7 @@ build:clang-msan --test_tag_filters=-no_san build:clang-msan --define ENVOY_CONFIG_MSAN=1 build:clang-msan --copt -fsanitize=memory build:clang-msan --linkopt -fsanitize=memory -build:clang-msan --linkopt -fuse-ld=lld +build:clang-msan --linkopt -fuse-ld=ld build:clang-msan --copt -fsanitize-memory-track-origins=2 build:clang-msan --copt -DMEMORY_SANITIZER=1 build:clang-msan --test_env=MSAN_SYMBOLIZER_PATH @@ -292,7 +295,7 @@ build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-asan --config=clang-asan -build:rbe-toolchain-asan --linkopt -fuse-ld=lld +build:rbe-toolchain-asan --linkopt -fuse-ld=ld build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1 build:rbe-toolchain-asan --copt=-fsanitize=vptr,function build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function diff --git a/api/BUILD b/api/BUILD index 79e325dab2a55..646ddb35f15b9 100644 --- a/api/BUILD +++ b/api/BUILD @@ -129,6 +129,7 @@ proto_library( "//envoy/data/cluster/v3:pkg", "//envoy/data/core/v3:pkg", "//envoy/data/dns/v3:pkg", + "//envoy/data/ip_tagging/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", "//envoy/extensions/access_loggers/filters/cel/v3:pkg", diff --git a/api/envoy/data/ip_tagging/v3/BUILD b/api/envoy/data/ip_tagging/v3/BUILD new file mode 100644 index 0000000000000..09a37ad16b837 --- /dev/null +++ b/api/envoy/data/ip_tagging/v3/BUILD @@ -0,0 +1,12 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "@com_github_cncf_xds//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/data/ip_tagging/v3/ip_tagging.proto b/api/envoy/data/ip_tagging/v3/ip_tagging.proto new file mode 100644 index 0000000000000..5cc630fe2cf55 --- /dev/null +++ b/api/envoy/data/ip_tagging/v3/ip_tagging.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package envoy.data.ip_tagging.v3; + +import "envoy/config/core/v3/address.proto"; + +import "udpa/annotations/status.proto"; +import "udpa/annotations/versioning.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.data.ip_tagging.v3"; +option java_outer_classname = "IpTaggingProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/data/ip_tagging/v3;ip_taggingv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// Supplies the IP tag name and the IP address subnets. +message IPTag { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.ip_tagging.v2.IPTagging.IPTag"; + + // Specifies the IP tag name to apply. + string ip_tag_name = 1; + + // A list of IP address subnets that will be tagged with + // ip_tag_name. Both IPv4 and IPv6 are supported. + repeated config.core.v3.CidrRange ip_list = 2; +} + +// Specifies the content of the IP tag file. +// Allow the file to be created with no IP tags. +message IPTagFile { + repeated IPTag ip_tags = 1; +} diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD index 09a37ad16b837..9ebe0ec6dfa52 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/core/v3:pkg", + "//envoy/data/ip_tagging/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 87f725e1c8c25..ad848e8c402fe 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.filters.http.ip_tagging.v3; -import "envoy/config/core/v3/address.proto"; +import "envoy/data/ip_tagging/v3/ip_tagging.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; @@ -18,7 +18,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // IP tagging :ref:`configuration overview `. // [#extension: envoy.filters.http.ip_tagging] -// [#next-free-field: 6] +// [#next-free-field: 7] message IPTagging { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.ip_tagging.v2.IPTagging"; @@ -40,19 +40,6 @@ message IPTagging { EXTERNAL = 2; } - // Supplies the IP tag name and the IP address subnets. - message IPTag { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.ip_tagging.v2.IPTagging.IPTag"; - - // Specifies the IP tag name to apply. - string ip_tag_name = 1; - - // A list of IP address subnets that will be tagged with - // ip_tag_name. Both IPv4 and IPv6 are supported. - repeated config.core.v3.CidrRange ip_list = 2; - } - // Specify to which header the tags will be written. message IpTagHeader { // Describes how to apply the tags to the headers. @@ -88,13 +75,21 @@ message IPTagging { // The type of request the filter should apply to. RequestType request_type = 1 [(validate.rules).enum = {defined_only: true}]; - // [#comment:TODO(ccaraman): Extend functionality to load IP tags from file system. - // Tracked by issue https://github.com/envoyproxy/envoy/issues/2695] // The set of IP tags for the filter. - repeated IPTag ip_tags = 4 [(validate.rules).repeated = {min_items: 1}]; + // Only one of :ref:`ip_tags ` + // or :ref:`ip_tags_path ` + // can be set for the Ip Tagging filter. + repeated data.ip_tagging.v3.IPTag ip_tags = 4 [(validate.rules).repeated = {min_items: 1}]; // Specify to which header the tags will be written. // // If left unspecified, the tags will be appended to the ``x-envoy-ip-tags`` header. IpTagHeader ip_tag_header = 5; + + // Absolute path to the file with ip tags. + // Supports Yaml and Json file formats for ip tags. + // Only one of :ref:`ip_tags ` + // or :ref:`ip_tags_path ` + // can be set for the Ip Tagging filter. + string ip_tags_path = 6 [(validate.rules).string = {min_len: 1}]; } diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 2f72d3743bb9a..bd03a7090681e 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -67,6 +67,7 @@ proto_library( "//envoy/data/cluster/v3:pkg", "//envoy/data/core/v3:pkg", "//envoy/data/dns/v3:pkg", + "//envoy/data/ip_tagging/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", "//envoy/extensions/access_loggers/filters/cel/v3:pkg", diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 2346c2ec75882..4c8565d53a4b9 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -10,6 +10,7 @@ def envoy_copts(repository, test = False): "-Wextra", "-Werror", "-Wnon-virtual-dtor", + "-Wno-implicit-function-declaration", "-Woverloaded-virtual", "-Wold-style-cast", "-Wformat", diff --git a/envoy/registry/registry.h b/envoy/registry/registry.h index b13aef7c1c56c..18480cf6583cc 100644 --- a/envoy/registry/registry.h +++ b/envoy/registry/registry.h @@ -220,7 +220,7 @@ template class FactoryRegistry : public Logger::Loggable ip_tags_registry = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), + [] { return std::make_shared(); }); + IpTaggingFilterConfigSharedPtr config(new IpTaggingFilterConfig( - proto_config, stat_prefix, context.scope(), context.serverFactoryContext().runtime())); + proto_config, ip_tags_registry, stat_prefix, context.scope(), + context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), + context.messageValidationVisitor())); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(config)); diff --git a/source/extensions/filters/http/ip_tagging/config.h b/source/extensions/filters/http/ip_tagging/config.h index 750466d8eeb80..600a5112e6d8f 100644 --- a/source/extensions/filters/http/ip_tagging/config.h +++ b/source/extensions/filters/http/ip_tagging/config.h @@ -1,8 +1,10 @@ #pragma once +#include "envoy/data/ip_tagging/v3/ip_tagging.pb.validate.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" +#include "source/common/network/lc_trie.h" #include "source/extensions/filters/http/common/factory_base.h" namespace Envoy { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index c680230b2e175..17fb94652966d 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -13,28 +13,33 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -IpTaggingFilterConfig::IpTaggingFilterConfig( - const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime) - : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), - stat_name_set_(scope.symbolTable().makeSet("IpTagging")), - stats_prefix_(stat_name_set_->add(stat_prefix + "ip_tagging")), - no_hit_(stat_name_set_->add("no_hit")), total_(stat_name_set_->add("total")), - unknown_tag_(stat_name_set_->add("unknown_tag.hit")), - ip_tag_header_(config.has_ip_tag_header() ? config.ip_tag_header().header() : ""), - ip_tag_header_action_(config.has_ip_tag_header() - ? config.ip_tag_header().action() - : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE) { - - // Once loading IP tags from a file system is supported, the restriction on the size - // of the set should be removed and observability into what tags are loaded needs - // to be implemented. - // TODO(ccaraman): Remove size check once file system support is implemented. - // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. - if (config.ip_tags().empty()) { - throw EnvoyException("HTTP IP Tagging Filter requires ip_tags to be specified."); +IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor) + : api_(api), validation_visitor_(validation_visitor) {} + +LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path) { + if (!ip_tags_path.empty()) { + if (!absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml) && + !absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { + throw EnvoyException("Unsupported file format, unable to parse ip tags from file."); + } + auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); + THROW_IF_NOT_OK_REF(file_or_error.status()); + IpTagFileProto ip_tags_proto; + if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { + MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); + } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { + MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); + } + std::vector>> tag_data; + tag_data.reserve(ip_tags_proto.ip_tags().size()); + return std::make_shared>(tag_data); } + return nullptr; +} +LcTrieSharedPtr IpTagsLoader::parseInlineIpTags( + const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, + Stats::StatNameSetPtr& stat_name_set) { std::vector>> tag_data; tag_data.reserve(config.ip_tags().size()); for (const auto& ip_tag : config.ip_tags()) { @@ -52,11 +57,63 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( entry.address_prefix(), entry.prefix_len().value())); } } - tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); - stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); + stat_name_set->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); + } + return std::make_shared>(tag_data); +} + +LcTrieSharedPtr IpTagsRegistrySingleton::get(const std::string& ip_tags_path, + IpTagsLoader& tags_loader) { + LcTrieSharedPtr ip_tags; + const size_t key = std::hash()(ip_tags_path); + absl::MutexLock lock(&mu_); + auto it = ip_tags_registry_.find(key); + if (it != ip_tags_registry_.end()) { + ip_tags = it->second.lock(); + } else { + ip_tags = tags_loader.loadTags(ip_tags_path); + ip_tags_registry_[key] = ip_tags; + } + return ip_tags; +} + +IpTaggingFilterConfig::IpTaggingFilterConfig( + const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, + std::shared_ptr ip_tags_registry, const std::string& stat_prefix, + Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, + ProtobufMessage::ValidationVisitor& validation_visitor) + : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), + stat_name_set_(scope.symbolTable().makeSet("IpTagging")), + stats_prefix_(stat_name_set_->add(stat_prefix + "ip_tagging")), + no_hit_(stat_name_set_->add("no_hit")), total_(stat_name_set_->add("total")), + unknown_tag_(stat_name_set_->add("unknown_tag.hit")), + ip_tag_header_(config.has_ip_tag_header() ? config.ip_tag_header().header() : ""), + ip_tag_header_action_(config.has_ip_tag_header() + ? config.ip_tag_header().action() + : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE), + ip_tags_path_(config.ip_tags_path()), ip_tags_registry_(ip_tags_registry), + tags_loader_(api, validation_visitor) { + + // Once loading IP tags from a file system is supported, the restriction on the size + // of the set should be removed and observability into what tags are loaded needs + // to be implemented. + // TODO(ccaraman): Remove size check once file system support is implemented. + // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. + if (config.ip_tags().empty() && config.ip_tags_path().empty()) { + throw EnvoyException( + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); + } + + if (!config.ip_tags().empty() && !config.ip_tags_path().empty()) { + throw EnvoyException("Only one of ip_tags or ip_tags_path can be configured."); + } + + if (!config.ip_tags().empty()) { + trie_ = tags_loader_.parseInlineIpTags(config, stat_name_set_); + } else { + trie_ = ip_tags_registry_->get(ip_tags_path_, tags_loader_); } - trie_ = std::make_unique>(tag_data); } void IpTaggingFilterConfig::incCounter(Stats::StatName name) { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index f055c962877c6..02f4c8c5762f1 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -22,6 +22,49 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { +using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; +using LcTrieSharedPtr = std::shared_ptr>; +// TODO supports stats for ip tags +// Add tests for config and loading from file +// Add tests for singleton +// Support async reload of tags file +class IpTagsLoader { +public: + IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor); + + LcTrieSharedPtr loadTags(const std::string& ip_tags_path); + + LcTrieSharedPtr + parseInlineIpTags(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, + Stats::StatNameSetPtr& stat_name_set); + +private: + Api::Api& api_; + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + +/** + * A singleton for file based loading of ip tags and looking up parsed trie data structures with Ip + * tags. When given equivalent file paths to the Ip tags, the singleton returns pointers to the same + * trie structure. + */ +class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { +public: + IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } + LcTrieSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader); + +private: + absl::Mutex mu_; + // We keep weak_ptr here so the trie structures can be destroyed if the config is updated to stop + // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton + // from being destroyed unless it's no longer keeping track of any providers. (The singleton + // shared_ptr is *only* held by driver instances.) + absl::flat_hash_map>> + ip_tags_registry_ ABSL_GUARDED_BY(mu_); +}; + +SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); + /** * Type of requests the filter should apply to. */ @@ -36,8 +79,10 @@ class IpTaggingFilterConfig { envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IpTagHeader::HeaderAction; IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, + std::shared_ptr ip_tags_registry, const std::string& stat_prefix, Stats::Scope& scope, - Runtime::Loader& runtime); + Runtime::Loader& runtime, Api::Api& api, + ProtobufMessage::ValidationVisitor& validation_visitor); Runtime::Loader& runtime() { return runtime_; } FilterRequestType requestType() const { return request_type_; } @@ -82,10 +127,15 @@ class IpTaggingFilterConfig { const Stats::StatName no_hit_; const Stats::StatName total_; const Stats::StatName unknown_tag_; - std::unique_ptr> trie_; + LcTrieSharedPtr trie_; const Http::LowerCaseString ip_tag_header_; // An empty string indicates that no ip_tag_header is set. const HeaderAction ip_tag_header_action_; + const std::string ip_tags_path_; + // A shared_ptr to keep the ip tags registry singleton alive as long as any of its trie structures + // are in use. + const std::shared_ptr ip_tags_registry_; + IpTagsLoader tags_loader_; }; using IpTaggingFilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/ip_tagging/BUILD b/test/extensions/filters/http/ip_tagging/BUILD index f801cdd0aa39c..e7188b97072a3 100644 --- a/test/extensions/filters/http/ip_tagging/BUILD +++ b/test/extensions/filters/http/ip_tagging/BUILD @@ -14,6 +14,9 @@ envoy_package() envoy_extension_cc_test( name = "ip_tagging_filter_test", srcs = ["ip_tagging_filter_test.cc"], + data = [ + "//test/extensions/filters/http/ip_tagging/test_data:ip_tagging_files", + ], extension_names = ["envoy.filters.http.ip_tagging"], rbe_pool = "6gig", deps = [ @@ -21,9 +24,12 @@ envoy_extension_cc_test( "//source/common/http:header_map_lib", "//source/common/network:address_lib", "//source/common/network:utility_lib", + "//source/common/singleton:manager_impl_lib", "//source/extensions/filters/http/ip_tagging:config", "//source/extensions/filters/http/ip_tagging:ip_tagging_filter_lib", + "//test/mocks/api:api_mocks", "//test/mocks/http:http_mocks", + "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/stats:stats_mocks", "//test/test_common:utility_lib", diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 66890eb0b6903..2d6b336aa2e38 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -5,11 +5,15 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/network/address_impl.h" #include "source/common/network/utility.h" +#include "source/common/singleton/manager_impl.h" #include "source/extensions/filters/http/ip_tagging/ip_tagging_filter.h" +#include "test/mocks/api/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/test_common/environment.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -25,30 +29,55 @@ namespace { class IpTaggingFilterTest : public testing::Test { public: - IpTaggingFilterTest() { + IpTaggingFilterTest() + : api_(Api::createApiForTest()), + ip_tags_registry_(std::make_shared()) { ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) .WillByDefault(Return(true)); } + void SetUp() override { + // singleton_manager_ = std::make_unique(); + // std::cerr << "***Creating singleton" << std::endl; + // ip_tags_registry_ = + // singleton_manager_->getTyped( + // SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry_singleton), + // [] { return std::make_shared(); }); + } + const std::string internal_request_yaml = R"EOF( request_type: internal ip_tags: - ip_tag_name: internal_request ip_list: - {address_prefix: 1.2.3.5, prefix_len: 32} - )EOF"; - void initializeFilter(const std::string& yaml) { + void initializeFilter(const std::string& yaml, bool load_with_substitute = false) { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - TestUtility::loadFromYaml(yaml, config); - config_ = - std::make_shared(config, "prefix.", *stats_.rootScope(), runtime_); + if (load_with_substitute) { + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + } else { + TestUtility::loadFromYaml(yaml, config); + } + // IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& + // config, + // std::shared_ptr ip_tags_registry, + // const std::string& stat_prefix, Stats::Scope& scope, + // Runtime::Loader& runtime, Api::Api& api, + // ProtobufMessage::ValidationVisitor& validation_visitor + config_ = std::make_shared(config, ip_tags_registry_, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); } - ~IpTaggingFilterTest() override { filter_->onDestroy(); } + ~IpTaggingFilterTest() override { + if (filter_) { + filter_->onDestroy(); + } + } NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; @@ -56,386 +85,438 @@ request_type: internal NiceMock filter_callbacks_; Buffer::OwnedImpl data_; NiceMock runtime_; + Api::ApiPtr api_; + NiceMock validation_visitor_; + std::shared_ptr ip_tags_registry_; }; -TEST_F(IpTaggingFilterTest, InternalRequest) { - initializeFilter(internal_request_yaml); - EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - - // Check external requests don't get a tag. - request_headers = Http::TestRequestHeaderMapImpl{}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} - -TEST_F(IpTaggingFilterTest, ExternalRequest) { - const std::string external_request_yaml = R"EOF( -request_type: external -ip_tags: - - ip_tag_name: external_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - initializeFilter(external_request_yaml); - EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers; - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - - // Check internal requests don't get a tag. - request_headers = {{"x-envoy-internal", "true"}}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} - -TEST_F(IpTaggingFilterTest, BothRequest) { - const std::string both_request_yaml = R"EOF( -request_type: both -ip_tags: - - ip_tag_name: external_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} -)EOF"; - - initializeFilter(both_request_yaml); - EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - request_headers = Http::TestRequestHeaderMapImpl{}; - remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -} - -TEST_F(IpTaggingFilterTest, NoHits) { - initializeFilter(internal_request_yaml); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, AppendEntry) { - initializeFilter(internal_request_yaml); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-ip-tags", "test"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, ReplaceAlternateHeaderWhenActionIsDefaulted) { - const std::string internal_request_yaml = R"EOF( +// TODO nezdolik split config tests into separate test file +// TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE( +// initializeFilter(config_yaml), Envoy::EnvoyException, +// "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); +// } + +// TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// ip_tags: +// - ip_tag_name: internal_request +// ip_list: +// - {address_prefix: 1.2.3.5, prefix_len: 32} +// ip_tags_path: /test/tags.yaml +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, +// "Only one of ip_tags or ip_tags_path can be configured."); +// } + +// TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tags_path: /test/tags.csv +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, +// "Unsupported file format, unable to parse ip tags from file."); +// } + +TEST_F(IpTaggingFilterTest, InternalRequestIpTagsFile) { + const std::string config_yaml = R"EOF( request_type: internal -ip_tag_header: - header: x-envoy-optional-header -ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} +ip_tags_path: /test/tags.csv +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; - - initializeFilter(internal_request_yaml); - - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}, // foo will be removed - {"x-envoy-optional-header", "bar"}, // bar will be removed - {"x-envoy-optional-header", "baz"}, // baz will be removed - }; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, ReplaceAlternateHeader) { - const std::string internal_request_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: SANITIZE -ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - - initializeFilter(internal_request_yaml); - - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + initializeFilter(config_yaml, true); } -TEST_F(IpTaggingFilterTest, ClearAlternateHeaderWhenUnmatchedAndSanitized) { - const std::string internal_request_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: SANITIZE -ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - - initializeFilter(internal_request_yaml); - - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be removed - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, AppendForwardAlternateHeader) { - const std::string internal_request_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: APPEND_IF_EXISTS_OR_ADD -ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - - initializeFilter(internal_request_yaml); - - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}}; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("foo,internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { - const std::string internal_request_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: APPEND_IF_EXISTS_OR_ADD -ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - - initializeFilter(internal_request_yaml); - - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}}; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, NestedPrefixes) { - const std::string duplicate_request_yaml = R"EOF( -request_type: both -ip_tags: - - ip_tag_name: duplicate_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} -)EOF"; - - initializeFilter(duplicate_request_yaml); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-ip-tags", "test"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - - // There is no guarantee for the order tags are returned by the LC-Trie. - const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); - EXPECT_NE(std::string::npos, header_tag_data.find("test")); - EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); - EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, Ipv6Address) { - const std::string ipv6_addresses_yaml = R"EOF( -ip_tags: - - ip_tag_name: ipv6_request - ip_list: - - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} -)EOF"; - initializeFilter(ipv6_addresses_yaml); - Http::TestRequestHeaderMapImpl request_headers; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, RuntimeDisabled) { - initializeFilter(internal_request_yaml); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillOnce(Return(false)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -TEST_F(IpTaggingFilterTest, ClearRouteCache) { - initializeFilter(internal_request_yaml); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - // no tags, no call - EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); - request_headers = Http::TestRequestHeaderMapImpl{}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} +// TEST_F(IpTaggingFilterTest, InternalRequest) { +// initializeFilter(internal_request_yaml); +// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check external requests don't get a tag. +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// TEST_F(IpTaggingFilterTest, ExternalRequest) { +// const std::string external_request_yaml = R"EOF( +// request_type: external +// ip_tags: +// - ip_tag_name: external_request +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; +// initializeFilter(external_request_yaml); +// EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers; + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check internal requests don't get a tag. +// request_headers = {{"x-envoy-internal", "true"}}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// TEST_F(IpTaggingFilterTest, BothRequest) { +// const std::string both_request_yaml = R"EOF( +// request_type: both +// ip_tags: +// - ip_tag_name: external_request +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// - ip_tag_name: internal_request +// ip_list: +// - {address_prefix: 1.2.3.5, prefix_len: 32} +// )EOF"; + +// initializeFilter(both_request_yaml); +// EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// request_headers = Http::TestRequestHeaderMapImpl{}; +// remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); +// } + +// TEST_F(IpTaggingFilterTest, NoHits) { +// initializeFilter(internal_request_yaml); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, AppendEntry) { +// initializeFilter(internal_request_yaml); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-ip-tags", "test"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, ReplaceAlternateHeaderWhenActionIsDefaulted) { +// const std::string internal_request_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// ip_tags: +// - ip_tag_name: internal_request_with_optional_header +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(internal_request_yaml); + +// Http::TestRequestHeaderMapImpl request_headers{ +// {"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}, // foo will be removed +// {"x-envoy-optional-header", "bar"}, // bar will be removed +// {"x-envoy-optional-header", "baz"}, // baz will be removed +// }; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, ReplaceAlternateHeader) { +// const std::string internal_request_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// ip_tags: +// - ip_tag_name: internal_request_with_optional_header +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(internal_request_yaml); + +// Http::TestRequestHeaderMapImpl request_headers{ +// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, ClearAlternateHeaderWhenUnmatchedAndSanitized) { +// const std::string internal_request_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// ip_tags: +// - ip_tag_name: internal_request_with_optional_header +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(internal_request_yaml); + +// Http::TestRequestHeaderMapImpl request_headers{ +// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be +// removed +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, AppendForwardAlternateHeader) { +// const std::string internal_request_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: APPEND_IF_EXISTS_OR_ADD +// ip_tags: +// - ip_tag_name: internal_request_with_optional_header +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(internal_request_yaml); + +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}}; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("foo,internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { +// const std::string internal_request_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: APPEND_IF_EXISTS_OR_ADD +// ip_tags: +// - ip_tag_name: internal_request_with_optional_header +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(internal_request_yaml); + +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}}; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, NestedPrefixes) { +// const std::string duplicate_request_yaml = R"EOF( +// request_type: both +// ip_tags: +// - ip_tag_name: duplicate_request +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// - ip_tag_name: internal_request +// ip_list: +// - {address_prefix: 1.2.3.4, prefix_len: 32} +// )EOF"; + +// initializeFilter(duplicate_request_yaml); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-ip-tags", "test"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + +// // There is no guarantee for the order tags are returned by the LC-Trie. +// const std::string header_tag_data = +// request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); EXPECT_NE(std::string::npos, +// header_tag_data.find("test")); EXPECT_NE(std::string::npos, +// header_tag_data.find("internal_request")); EXPECT_NE(std::string::npos, +// header_tag_data.find("duplicate_request")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, Ipv6Address) { +// const std::string ipv6_addresses_yaml = R"EOF( +// ip_tags: +// - ip_tag_name: ipv6_request +// ip_list: +// - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} +// )EOF"; +// initializeFilter(ipv6_addresses_yaml); +// Http::TestRequestHeaderMapImpl request_headers; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, RuntimeDisabled) { +// initializeFilter(internal_request_yaml); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) +// .WillOnce(Return(false)); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// TEST_F(IpTaggingFilterTest, ClearRouteCache) { +// initializeFilter(internal_request_yaml); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// // no tags, no call +// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } } // namespace } // namespace IpTagging diff --git a/test/extensions/filters/http/ip_tagging/test_data/BUILD b/test/extensions/filters/http/ip_tagging/test_data/BUILD new file mode 100644 index 0000000000000..959996ce07a30 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/BUILD @@ -0,0 +1,13 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +filegroup( + name = "ip_tagging_files", + srcs = glob(["test_data/*"]), +) diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml new file mode 100644 index 0000000000000..fb384c8668057 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml @@ -0,0 +1,4 @@ +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} \ No newline at end of file From a3e5d675b7bdbed85e517c9188df1d3db726ba8c Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 27 Feb 2025 13:08:09 +0100 Subject: [PATCH 02/76] Refactor Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 24 ++++---- .../http/ip_tagging/ip_tagging_filter.h | 7 ++- .../http/ip_tagging/ip_tagging_filter_test.cc | 55 ++++++++++++++++++- .../filters/http/ip_tagging/test_data/BUILD | 2 +- .../test_data/ip_tags_internal_request.json | 13 +++++ 5 files changed, 82 insertions(+), 19 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 17fb94652966d..dfc45695da3f1 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -13,8 +13,9 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor) - : api_(api), validation_visitor_(validation_visitor) {} +IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, + Stats::StatNameSetPtr& stat_name_set) + : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path) { if (!ip_tags_path.empty()) { @@ -30,19 +31,16 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path) { } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); } - std::vector>> tag_data; - tag_data.reserve(ip_tags_proto.ip_tags().size()); - return std::make_shared>(tag_data); + return parseIpTags(ip_tags_proto.ip_tags()); } return nullptr; } -LcTrieSharedPtr IpTagsLoader::parseInlineIpTags( - const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - Stats::StatNameSetPtr& stat_name_set) { +LcTrieSharedPtr IpTagsLoader::parseIpTags( + const Protobuf::RepeatedPtrField& ip_tags) { std::vector>> tag_data; - tag_data.reserve(config.ip_tags().size()); - for (const auto& ip_tag : config.ip_tags()) { + tag_data.reserve(ip_tags.size()); + for (const auto& ip_tag : ip_tags) { std::vector cidr_set; cidr_set.reserve(ip_tag.ip_list().size()); for (const envoy::config::core::v3::CidrRange& entry : ip_tag.ip_list()) { @@ -58,7 +56,7 @@ LcTrieSharedPtr IpTagsLoader::parseInlineIpTags( } } tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); - stat_name_set->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); + stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } return std::make_shared>(tag_data); } @@ -93,7 +91,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( ? config.ip_tag_header().action() : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE), ip_tags_path_(config.ip_tags_path()), ip_tags_registry_(ip_tags_registry), - tags_loader_(api, validation_visitor) { + tags_loader_(api, validation_visitor, stat_name_set_) { // Once loading IP tags from a file system is supported, the restriction on the size // of the set should be removed and observability into what tags are loaded needs @@ -110,7 +108,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( } if (!config.ip_tags().empty()) { - trie_ = tags_loader_.parseInlineIpTags(config, stat_name_set_); + trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { trie_ = ip_tags_registry_->get(ip_tags_path_, tags_loader_); } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 02f4c8c5762f1..36d4e12aa0a6e 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -30,17 +30,18 @@ using LcTrieSharedPtr = std::shared_ptr>; // Support async reload of tags file class IpTagsLoader { public: - IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor); + IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, + Stats::StatNameSetPtr& stat_name_set); LcTrieSharedPtr loadTags(const std::string& ip_tags_path); LcTrieSharedPtr - parseInlineIpTags(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - Stats::StatNameSetPtr& stat_name_set); + parseIpTags(const Protobuf::RepeatedPtrField& ip_tags); private: Api::Api& api_; ProtobufMessage::ValidationVisitor& validation_visitor_; + Stats::StatNameSetPtr& stat_name_set_; }; /** diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 2d6b336aa2e38..dc172ec99c5df 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -128,13 +128,64 @@ request_type: internal // "Unsupported file format, unable to parse ip tags from file."); // } -TEST_F(IpTaggingFilterTest, InternalRequestIpTagsFile) { +TEST_F(IpTaggingFilterTest, InternalRequestIpTagsYamlFile) { const std::string config_yaml = R"EOF( request_type: internal -ip_tags_path: /test/tags.csv ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; initializeFilter(config_yaml, true); + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + // Check external requests don't get a tag. + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +} + +TEST_F(IpTaggingFilterTest, InternalRequestIpTagsJsonFile) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" +)EOF"; + initializeFilter(config_yaml, true); + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + // Check external requests don't get a tag. + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); } // TEST_F(IpTaggingFilterTest, InternalRequest) { diff --git a/test/extensions/filters/http/ip_tagging/test_data/BUILD b/test/extensions/filters/http/ip_tagging/test_data/BUILD index 959996ce07a30..92f9b6dfe9ac6 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/BUILD +++ b/test/extensions/filters/http/ip_tagging/test_data/BUILD @@ -9,5 +9,5 @@ envoy_package() filegroup( name = "ip_tagging_files", - srcs = glob(["test_data/*"]), + srcs = glob(["*"]), ) diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json new file mode 100644 index 0000000000000..68d3e4ecde66e --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json @@ -0,0 +1,13 @@ +{ + "ip_tags": [ + { + "ip_tag_name": "internal_request", + "ip_list": [ + { + "address_prefix": "1.2.3.5", + "prefix_len": 32 + } + ] + } + ] +} \ No newline at end of file From 54dc7ff0477429a2c3430316f6149ae87f489316 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 28 Feb 2025 16:28:15 +0100 Subject: [PATCH 03/76] Impl async file reload Signed-off-by: Kateryna Nezdolii --- source/common/common/logger.h | 3 +- .../http/ip_tagging/ip_tagging_filter.h | 48 ++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 761da45a97ca3..5a6845fb4f96d 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -98,7 +98,8 @@ const static bool should_log = true; FUNCTION(websocket) \ FUNCTION(golang) \ FUNCTION(stats_sinks) \ - FUNCTION(dynamic_modules) + FUNCTION(dynamic_modules) \ + FUNCTION(ip_tagging) // clang-format off enum class Id { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 36d4e12aa0a6e..6a18d698d6aed 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -24,8 +24,8 @@ namespace IpTagging { using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; using LcTrieSharedPtr = std::shared_ptr>; + // TODO supports stats for ip tags -// Add tests for config and loading from file // Add tests for singleton // Support async reload of tags file class IpTagsLoader { @@ -44,6 +44,52 @@ class IpTagsLoader { Stats::StatNameSetPtr& stat_name_set_; }; +class IpTagsProvider : public Logger::Loggable { +public: + IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner) + : ip_tags_path_(ip_tags_path), tags_loader_(tags_loader), owner_(owner) { + if (ip_tags_path.empty()) { + throw EnvoyException("Cannot load tags from empty file path."); + } + tags_ = tags_loader_.loadTags(ip_tags_path_); + ip_tags_reload_dispatcher_ = api.allocateDispatcher("ip_tags_reload_routine"); + ip_tags_file_watcher_ = dispatcher.createFilesystemWatcher(); + ip_tags_reload_thread_ = api.threadFactory().createThread( + [this]() -> void { + ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); + THROW_IF_NOT_OK( + ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, + [this](uint32_t) { return onIpTagsFileUpdate(); })); + ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); + }, + Thread::Options{std::string("ip_tags_reload_routine")}); + } + + ~IpTagsProvider() { + ENVOY_LOG(debug, "Shutting down ip tags provider"); + if (ip_tags_reload_dispatcher_) { + ip_tags_reload_dispatcher_->exit(); + } + if (ip_tags_reload_thread_) { + ip_tags_reload_thread_->join(); + ip_tags_reload_thread_.reset(); + } + }; + + absl::Status onIpTagsFileUpdate() { return absl::OkStatus(); } + +private: + const std::string ip_tags_path_; + IpTagsLoader& tags_loader_; + LcTrieSharedPtr tags_; + Thread::ThreadPtr ip_tags_reload_thread_; + Event::DispatcherPtr ip_tags_reload_dispatcher_; + Filesystem::WatcherPtr ip_tags_file_watcher_; + // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. + const Singleton::InstanceSharedPtr owner_; +}; + /** * A singleton for file based loading of ip tags and looking up parsed trie data structures with Ip * tags. When given equivalent file paths to the Ip tags, the singleton returns pointers to the same From 7c97081a0ec22decff247f01b151cda141984cfa Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 14 Mar 2025 15:43:09 +0100 Subject: [PATCH 04/76] Fixing more tests Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 535 +++++++++++------- .../ip_tagging/test_data/ip_tags_both.json | 22 + .../ip_tagging/test_data/ip_tags_both.yaml | 7 + .../test_data/ip_tags_external_request.json | 13 + .../test_data/ip_tags_external_request.yaml | 4 + .../test_data/ip_tags_internal_request.json | 2 +- .../test_data/ip_tags_internal_request.yaml | 2 +- .../test_data/ip_tags_with_header.json | 13 + .../test_data/ip_tags_with_header.yaml | 4 + 9 files changed, 395 insertions(+), 207 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index dc172ec99c5df..7c9cd846a4cf4 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -27,25 +27,10 @@ namespace HttpFilters { namespace IpTagging { namespace { -class IpTaggingFilterTest : public testing::Test { -public: - IpTaggingFilterTest() - : api_(Api::createApiForTest()), - ip_tags_registry_(std::make_shared()) { - ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillByDefault(Return(true)); - } + std::shared_ptr ip_tags_registry; - void SetUp() override { - // singleton_manager_ = std::make_unique(); - // std::cerr << "***Creating singleton" << std::endl; - // ip_tags_registry_ = - // singleton_manager_->getTyped( - // SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry_singleton), - // [] { return std::make_shared(); }); - } - - const std::string internal_request_yaml = R"EOF( + namespace { + const std::string internal_request_config = R"EOF( request_type: internal ip_tags: - ip_tag_name: internal_request @@ -53,20 +38,125 @@ request_type: internal - {address_prefix: 1.2.3.5, prefix_len: 32} )EOF"; - void initializeFilter(const std::string& yaml, bool load_with_substitute = false) { + const std::string internal_request_with_json_file_config = R"EOF( + request_type: internal + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" + )EOF"; + + const std::string internal_request_with_yaml_file_config = R"EOF( + request_type: internal + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" + )EOF"; + + const std::string external_request_config = R"EOF( +request_type: external +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +)EOF"; + + const std::string external_request_with_json_file_config = R"EOF( + request_type: external + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" + )EOF"; + + const std::string external_request_with_yaml_file_config = R"EOF( + request_type: external + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + )EOF"; + + const std::string both_request_config = R"EOF( +request_type: both +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} +)EOF"; + + const std::string both_request_with_json_file_config = R"EOF( +request_type: both +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" +)EOF"; + + const std::string both_request_with_yaml_file_config = R"EOF( +request_type: both +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" +)EOF"; + + const std::string internal_request_with_header_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header +ip_tags: + - ip_tag_name: internal_request_with_optional_header + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +)EOF"; + + const std::string internal_request_with_header_with_json_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +)EOF"; + + const std::string internal_request_with_header_with_yaml_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +)EOF"; + + const std::string internal_request_with_replace_header_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +ip_tags: + - ip_tag_name: internal_request_with_optional_header + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +)EOF"; + + const std::string internal_request_with_replace_header_with_json_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +)EOF"; + + const std::string internal_request_with_replace_header_with_yaml_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +)EOF"; + + }// namespace + +class IpTaggingFilterTest : public ::testing::TestWithParam { +public: + IpTaggingFilterTest() + : api_(Api::createApiForTest()) + { + ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillByDefault(Return(true)); + } + + static void SetUpTestSuite() { + ip_tags_registry = std::make_shared(); + } + + void initializeFilter(const std::string& yaml) { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - if (load_with_substitute) { TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); - } else { - TestUtility::loadFromYaml(yaml, config); - } - // IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& - // config, - // std::shared_ptr ip_tags_registry, - // const std::string& stat_prefix, Stats::Scope& scope, - // Runtime::Loader& runtime, Api::Api& api, - // ProtobufMessage::ValidationVisitor& validation_visitor - config_ = std::make_shared(config, ip_tags_registry_, "prefix.", + config_ = std::make_shared(config, ip_tags_registry, "prefix.", *stats_.rootScope(), runtime_, *api_, validation_visitor_); filter_ = std::make_unique(config_); @@ -87,7 +177,6 @@ request_type: internal NiceMock runtime_; Api::ApiPtr api_; NiceMock validation_visitor_; - std::shared_ptr ip_tags_registry_; }; // TODO nezdolik split config tests into separate test file @@ -128,12 +217,72 @@ request_type: internal // "Unsupported file format, unable to parse ip tags from file."); // } -TEST_F(IpTaggingFilterTest, InternalRequestIpTagsYamlFile) { - const std::string config_yaml = R"EOF( -request_type: internal -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" -)EOF"; - initializeFilter(config_yaml, true); +// TEST_F(IpTaggingFilterTest, InternalRequestIpTagsYamlFile) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" +// )EOF"; +// initializeFilter(config_yaml, true); +// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check external requests don't get a tag. +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// TEST_F(IpTaggingFilterTest, InternalRequestIpTagsJsonFile) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" +// )EOF"; +// initializeFilter(config_yaml, true); +// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check external requests don't get a tag. +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +class InternalRequestIpTaggingFilterTest: public IpTaggingFilterTest { +}; + +TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { + const std::string config = GetParam(); + initializeFilter(config); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -158,217 +307,186 @@ ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_da EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); } -TEST_F(IpTaggingFilterTest, InternalRequestIpTagsJsonFile) { - const std::string config_yaml = R"EOF( -request_type: internal -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" -)EOF"; - initializeFilter(config_yaml, true); - EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; +INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, + ::testing::ValuesIn( + {internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class ExternalRequestIpTaggingFilterTest: public IpTaggingFilterTest { +}; + +TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { + const std::string config = GetParam(); + initializeFilter(config); + EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers; + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - // Check external requests don't get a tag. - request_headers = Http::TestRequestHeaderMapImpl{}; + // Check internal requests don't get a tag. + request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); } -// TEST_F(IpTaggingFilterTest, InternalRequest) { -// initializeFilter(internal_request_yaml); -// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - -// // Check external requests don't get a tag. -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } +INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, + ::testing::ValuesIn( + {external_request_config, + external_request_with_json_file_config, + external_request_with_yaml_file_config + })); -// TEST_F(IpTaggingFilterTest, ExternalRequest) { -// const std::string external_request_yaml = R"EOF( -// request_type: external -// ip_tags: -// - ip_tag_name: external_request -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; -// initializeFilter(external_request_yaml); -// EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers; - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); +class BothRequestIpTaggingFilterTest: public IpTaggingFilterTest { +}; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); +TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { + const std::string config = GetParam(); + initializeFilter(config); + EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); -// // Check internal requests don't get a tag. -// request_headers = {{"x-envoy-internal", "true"}}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// TEST_F(IpTaggingFilterTest, BothRequest) { -// const std::string both_request_yaml = R"EOF( -// request_type: both -// ip_tags: -// - ip_tag_name: external_request -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// - ip_tag_name: internal_request -// ip_list: -// - {address_prefix: 1.2.3.5, prefix_len: 32} -// )EOF"; + request_headers = Http::TestRequestHeaderMapImpl{}; + remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); -// initializeFilter(both_request_yaml); -// EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); +} -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); +INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, + ::testing::ValuesIn( + {both_request_config, + both_request_with_json_file_config, + both_request_with_yaml_file_config + })); -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); +class NoHitsIpTaggingFilterTest: public IpTaggingFilterTest { +}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); +TEST_P(NoHitsIpTaggingFilterTest, NoHits) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; -// request_headers = Http::TestRequestHeaderMapImpl{}; -// remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// } + EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); -// TEST_F(IpTaggingFilterTest, NoHits) { -// initializeFilter(internal_request_yaml); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); +INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, + ::testing::ValuesIn( + {internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config + })); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +class AppendEntryFilterTest: public IpTaggingFilterTest { +}; -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } +TEST_P(AppendEntryFilterTest, AppendEntry) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-ip-tags", "test"}}; -// TEST_F(IpTaggingFilterTest, AppendEntry) { -// initializeFilter(internal_request_yaml); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-ip-tags", "test"}}; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } -// TEST_F(IpTaggingFilterTest, ReplaceAlternateHeaderWhenActionIsDefaulted) { -// const std::string internal_request_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// ip_tags: -// - ip_tag_name: internal_request_with_optional_header -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; +INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, + ::testing::ValuesIn( + {internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config + })); -// initializeFilter(internal_request_yaml); +class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest: public IpTaggingFilterTest { +}; -// Http::TestRequestHeaderMapImpl request_headers{ -// {"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}, // foo will be removed -// {"x-envoy-optional-header", "bar"}, // bar will be removed -// {"x-envoy-optional-header", "baz"}, // baz will be removed -// }; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); +TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, ReplaceAlternateHeaderWhenActionIsDefaulted) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{ + {"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}, // foo will be removed + {"x-envoy-optional-header", "bar"}, // bar will be removed + {"x-envoy-optional-header", "baz"}, // baz will be removed + }; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} -// TEST_F(IpTaggingFilterTest, ReplaceAlternateHeader) { -// const std::string internal_request_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// ip_tags: -// - ip_tag_name: internal_request_with_optional_header -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; +INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, + ::testing::ValuesIn( + {internal_request_with_header_config, + //internal_request_with_header_with_json_file_config, + internal_request_with_header_with_yaml_file_config + })); -// initializeFilter(internal_request_yaml); +// class ReplaceAlternateHeaderFilterTest: public IpTaggingFilterTest { +// }; +// TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { +// const std::string config = GetParam(); +// initializeFilter(config); // Http::TestRequestHeaderMapImpl request_headers{ // {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed // Network::Address::InstanceConstSharedPtr remote_address = @@ -385,6 +503,13 @@ ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_da // EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); // } +// INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, +// ::testing::ValuesIn( +// {internal_request_with_replace_header_config, +// internal_request_with_replace_header_with_json_file_config, +// internal_request_with_replace_header_with_yaml_file_config +// })); + // TEST_F(IpTaggingFilterTest, ClearAlternateHeaderWhenUnmatchedAndSanitized) { // const std::string internal_request_yaml = R"EOF( // request_type: internal @@ -401,7 +526,7 @@ ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_da // Http::TestRequestHeaderMapImpl request_headers{ // {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be -// removed +// //removed // Network::Address::InstanceConstSharedPtr remote_address = // Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); // filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json new file mode 100644 index 0000000000000..19481df2a6c62 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json @@ -0,0 +1,22 @@ +{ +"ip_tags": [ + { + "ip_tag_name": "external_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + }, + { + "ip_tag_name": "internal_request", + "ip_list": [ + { + "address_prefix": "1.2.3.5", + "prefix_len": 32 + } + ] + } +] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml new file mode 100644 index 0000000000000..002388277147e --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml @@ -0,0 +1,7 @@ +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json new file mode 100644 index 0000000000000..0b5f0ddc4f97d --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json @@ -0,0 +1,13 @@ +{ +"ip_tags": [ + { + "ip_tag_name": "external_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + } +] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml new file mode 100644 index 0000000000000..741868809deee --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml @@ -0,0 +1,4 @@ +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json index 68d3e4ecde66e..ce55c2d9af943 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json @@ -10,4 +10,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml index fb384c8668057..5ae7d7868ab96 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml @@ -1,4 +1,4 @@ ip_tags: - ip_tag_name: internal_request ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} \ No newline at end of file + - {address_prefix: 1.2.3.5, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json new file mode 100644 index 0000000000000..036582c21edbe --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json @@ -0,0 +1,13 @@ +{ +"ip_tags": [ + { + "ip_tag_name": "internal_request_with_optional_header", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + } +] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml new file mode 100644 index 0000000000000..cba803e6dd72b --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml @@ -0,0 +1,4 @@ +ip_tags: + - ip_tag_name: internal_request_with_optional_header + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} From 07f7df3e3b9041bec6e20c093271ca1431a0c797 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 14 Mar 2025 15:44:18 +0100 Subject: [PATCH 05/76] Add more tests Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 126 ++++++++---------- 1 file changed, 53 insertions(+), 73 deletions(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 7c9cd846a4cf4..11e4267ba4480 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -27,10 +27,10 @@ namespace HttpFilters { namespace IpTagging { namespace { - std::shared_ptr ip_tags_registry; +std::shared_ptr ip_tags_registry; - namespace { - const std::string internal_request_config = R"EOF( +namespace { +const std::string internal_request_config = R"EOF( request_type: internal ip_tags: - ip_tag_name: internal_request @@ -38,17 +38,17 @@ request_type: internal - {address_prefix: 1.2.3.5, prefix_len: 32} )EOF"; - const std::string internal_request_with_json_file_config = R"EOF( +const std::string internal_request_with_json_file_config = R"EOF( request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; - const std::string internal_request_with_yaml_file_config = R"EOF( +const std::string internal_request_with_yaml_file_config = R"EOF( request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; - const std::string external_request_config = R"EOF( +const std::string external_request_config = R"EOF( request_type: external ip_tags: - ip_tag_name: external_request @@ -56,17 +56,17 @@ request_type: external - {address_prefix: 1.2.3.4, prefix_len: 32} )EOF"; - const std::string external_request_with_json_file_config = R"EOF( +const std::string external_request_with_json_file_config = R"EOF( request_type: external ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" )EOF"; - const std::string external_request_with_yaml_file_config = R"EOF( +const std::string external_request_with_yaml_file_config = R"EOF( request_type: external ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" )EOF"; - const std::string both_request_config = R"EOF( +const std::string both_request_config = R"EOF( request_type: both ip_tags: - ip_tag_name: external_request @@ -77,17 +77,17 @@ request_type: both - {address_prefix: 1.2.3.5, prefix_len: 32} )EOF"; - const std::string both_request_with_json_file_config = R"EOF( +const std::string both_request_with_json_file_config = R"EOF( request_type: both ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" )EOF"; - const std::string both_request_with_yaml_file_config = R"EOF( +const std::string both_request_with_yaml_file_config = R"EOF( request_type: both ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" )EOF"; - const std::string internal_request_with_header_config = R"EOF( +const std::string internal_request_with_header_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header @@ -97,21 +97,21 @@ request_type: internal - {address_prefix: 1.2.3.4, prefix_len: 32} )EOF"; - const std::string internal_request_with_header_with_json_file_config = R"EOF( +const std::string internal_request_with_header_with_json_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; - const std::string internal_request_with_header_with_yaml_file_config = R"EOF( +const std::string internal_request_with_header_with_yaml_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; - const std::string internal_request_with_replace_header_config = R"EOF( +const std::string internal_request_with_replace_header_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header @@ -122,7 +122,7 @@ request_type: internal - {address_prefix: 1.2.3.4, prefix_len: 32} )EOF"; - const std::string internal_request_with_replace_header_with_json_file_config = R"EOF( +const std::string internal_request_with_replace_header_with_json_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header @@ -130,7 +130,7 @@ request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; - const std::string internal_request_with_replace_header_with_yaml_file_config = R"EOF( +const std::string internal_request_with_replace_header_with_yaml_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header @@ -138,24 +138,20 @@ request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; - }// namespace +} // namespace class IpTaggingFilterTest : public ::testing::TestWithParam { public: - IpTaggingFilterTest() - : api_(Api::createApiForTest()) - { + IpTaggingFilterTest() : api_(Api::createApiForTest()) { ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) .WillByDefault(Return(true)); } - static void SetUpTestSuite() { - ip_tags_registry = std::make_shared(); - } + static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } void initializeFilter(const std::string& yaml) { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); config_ = std::make_shared(config, ip_tags_registry, "prefix.", *stats_.rootScope(), runtime_, *api_, validation_visitor_); @@ -220,8 +216,8 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { // TEST_F(IpTaggingFilterTest, InternalRequestIpTagsYamlFile) { // const std::string config_yaml = R"EOF( // request_type: internal -// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" -// )EOF"; +// ip_tags_path: "{{ test_rundir +// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; // initializeFilter(config_yaml, true); // EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); // Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -250,8 +246,8 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { // TEST_F(IpTaggingFilterTest, InternalRequestIpTagsJsonFile) { // const std::string config_yaml = R"EOF( // request_type: internal -// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" -// )EOF"; +// ip_tags_path: "{{ test_rundir +// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; // initializeFilter(config_yaml, true); // EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); // Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -277,8 +273,7 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { // EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); // } -class InternalRequestIpTaggingFilterTest: public IpTaggingFilterTest { -}; +class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { const std::string config = GetParam(); @@ -308,13 +303,11 @@ TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { } INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, - ::testing::ValuesIn( - {internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); -class ExternalRequestIpTaggingFilterTest: public IpTaggingFilterTest { -}; +class ExternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { const std::string config = GetParam(); @@ -344,14 +337,11 @@ TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { } INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, - ::testing::ValuesIn( - {external_request_config, - external_request_with_json_file_config, - external_request_with_yaml_file_config - })); + ::testing::ValuesIn({external_request_config, + external_request_with_json_file_config, + external_request_with_yaml_file_config})); -class BothRequestIpTaggingFilterTest: public IpTaggingFilterTest { -}; +class BothRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { const std::string config = GetParam(); @@ -381,14 +371,11 @@ TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { } INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, - ::testing::ValuesIn( - {both_request_config, - both_request_with_json_file_config, - both_request_with_yaml_file_config - })); + ::testing::ValuesIn({both_request_config, + both_request_with_json_file_config, + both_request_with_yaml_file_config})); -class NoHitsIpTaggingFilterTest: public IpTaggingFilterTest { -}; +class NoHitsIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(NoHitsIpTaggingFilterTest, NoHits) { const std::string config = GetParam(); @@ -412,14 +399,11 @@ TEST_P(NoHitsIpTaggingFilterTest, NoHits) { } INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, - ::testing::ValuesIn( - {internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config - })); + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); -class AppendEntryFilterTest: public IpTaggingFilterTest { -}; +class AppendEntryFilterTest : public IpTaggingFilterTest {}; TEST_P(AppendEntryFilterTest, AppendEntry) { const std::string config = GetParam(); @@ -440,18 +424,15 @@ TEST_P(AppendEntryFilterTest, AppendEntry) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); } - INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, - ::testing::ValuesIn( - {internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config - })); + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); -class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest: public IpTaggingFilterTest { -}; +class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest : public IpTaggingFilterTest {}; -TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, ReplaceAlternateHeaderWhenActionIsDefaulted) { +TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, + ReplaceAlternateHeaderWhenActionIsDefaulted) { const std::string config = GetParam(); initializeFilter(config); Http::TestRequestHeaderMapImpl request_headers{ @@ -474,12 +455,11 @@ TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, ReplaceAlternateHe EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); } -INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, - ::testing::ValuesIn( - {internal_request_with_header_config, - //internal_request_with_header_with_json_file_config, - internal_request_with_header_with_yaml_file_config - })); +INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, + ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, + ::testing::ValuesIn({internal_request_with_header_config, + // internal_request_with_header_with_json_file_config, + internal_request_with_header_with_yaml_file_config})); // class ReplaceAlternateHeaderFilterTest: public IpTaggingFilterTest { // }; From f6969cb02e915594349de76091d549461c3659e3 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 18 Mar 2025 12:09:55 +0100 Subject: [PATCH 06/76] Fix unit tests Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 628 +++++++++--------- .../ip_tags_with_duplicate_request.json | 22 + .../ip_tags_with_duplicate_request.yaml | 7 + .../ip_tagging/test_data/ipv6_request.json | 13 + .../ip_tagging/test_data/ipv6_request.yaml | 4 + 5 files changed, 365 insertions(+), 309 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 11e4267ba4480..d8ab533f84ec7 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -138,6 +138,69 @@ request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; +const std::string internal_request_with_append_or_add_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: APPEND_IF_EXISTS_OR_ADD +ip_tags: + - ip_tag_name: internal_request_with_optional_header + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +)EOF"; + +const std::string internal_request_with_append_or_add_with_json_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: APPEND_IF_EXISTS_OR_ADD +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +)EOF"; + +const std::string internal_request_with_append_or_add_with_yaml_file_config = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: APPEND_IF_EXISTS_OR_ADD +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +)EOF"; + +const std::string duplicate_request_config = R"EOF( +request_type: both +ip_tags: + - ip_tag_name: duplicate_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +)EOF"; + +const std::string duplicate_request_with_json_file_config = R"EOF( +request_type: both +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json" +)EOF"; + +const std::string duplicate_request_with_yaml_file_config = R"EOF( +request_type: both +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml" +)EOF"; + +const std::string ipv6_config = R"EOF( +ip_tags: + - ip_tag_name: ipv6_request + ip_list: + - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} +)EOF"; + +const std::string ipv6_with_yaml_file_config = R"EOF( +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml" +)EOF"; + +const std::string ipv6_with_json_file_config = R"EOF( +ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json" +)EOF"; + } // namespace class IpTaggingFilterTest : public ::testing::TestWithParam { @@ -176,102 +239,42 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { }; // TODO nezdolik split config tests into separate test file -// TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE( -// initializeFilter(config_yaml), Envoy::EnvoyException, -// "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); -// } - -// TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// ip_tags: -// - ip_tag_name: internal_request -// ip_list: -// - {address_prefix: 1.2.3.5, prefix_len: 32} -// ip_tags_path: /test/tags.yaml -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, -// "Only one of ip_tags or ip_tags_path can be configured."); -// } - -// TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tags_path: /test/tags.csv -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, -// "Unsupported file format, unable to parse ip tags from file."); -// } - -// TEST_F(IpTaggingFilterTest, InternalRequestIpTagsYamlFile) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tags_path: "{{ test_rundir -// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; -// initializeFilter(config_yaml, true); -// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - -// // Check external requests don't get a tag. -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } - -// TEST_F(IpTaggingFilterTest, InternalRequestIpTagsJsonFile) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tags_path: "{{ test_rundir -// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; -// initializeFilter(config_yaml, true); -// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - -// // Check external requests don't get a tag. -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } +TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +)EOF"; + EXPECT_THROW_WITH_MESSAGE( + initializeFilter(config_yaml), Envoy::EnvoyException, + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); +} + +TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} +ip_tags_path: /test/tags.yaml +)EOF"; + EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, + "Only one of ip_tags or ip_tags_path can be configured."); +} + +TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_path: /test/tags.csv +)EOF"; + EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, + "Unsupported file format, unable to parse ip tags from file."); +} class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; @@ -458,221 +461,228 @@ TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, ::testing::ValuesIn({internal_request_with_header_config, - // internal_request_with_header_with_json_file_config, + internal_request_with_header_with_json_file_config, internal_request_with_header_with_yaml_file_config})); -// class ReplaceAlternateHeaderFilterTest: public IpTaggingFilterTest { -// }; - -// TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{ -// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, -// ::testing::ValuesIn( -// {internal_request_with_replace_header_config, -// internal_request_with_replace_header_with_json_file_config, -// internal_request_with_replace_header_with_yaml_file_config -// })); - -// TEST_F(IpTaggingFilterTest, ClearAlternateHeaderWhenUnmatchedAndSanitized) { -// const std::string internal_request_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// ip_tags: -// - ip_tag_name: internal_request_with_optional_header -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; - -// initializeFilter(internal_request_yaml); - -// Http::TestRequestHeaderMapImpl request_headers{ -// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be -// //removed -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, AppendForwardAlternateHeader) { -// const std::string internal_request_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: APPEND_IF_EXISTS_OR_ADD -// ip_tags: -// - ip_tag_name: internal_request_with_optional_header -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; - -// initializeFilter(internal_request_yaml); - -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("foo,internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { -// const std::string internal_request_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: APPEND_IF_EXISTS_OR_ADD -// ip_tags: -// - ip_tag_name: internal_request_with_optional_header -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; - -// initializeFilter(internal_request_yaml); - -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, NestedPrefixes) { -// const std::string duplicate_request_yaml = R"EOF( -// request_type: both -// ip_tags: -// - ip_tag_name: duplicate_request -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// - ip_tag_name: internal_request -// ip_list: -// - {address_prefix: 1.2.3.4, prefix_len: 32} -// )EOF"; - -// initializeFilter(duplicate_request_yaml); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-ip-tags", "test"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - -// // There is no guarantee for the order tags are returned by the LC-Trie. -// const std::string header_tag_data = -// request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); EXPECT_NE(std::string::npos, -// header_tag_data.find("test")); EXPECT_NE(std::string::npos, -// header_tag_data.find("internal_request")); EXPECT_NE(std::string::npos, -// header_tag_data.find("duplicate_request")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, Ipv6Address) { -// const std::string ipv6_addresses_yaml = R"EOF( -// ip_tags: -// - ip_tag_name: ipv6_request -// ip_list: -// - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} -// )EOF"; -// initializeFilter(ipv6_addresses_yaml); -// Http::TestRequestHeaderMapImpl request_headers; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, RuntimeDisabled) { -// initializeFilter(internal_request_yaml); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) -// .WillOnce(Return(false)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// TEST_F(IpTaggingFilterTest, ClearRouteCache) { -// initializeFilter(internal_request_yaml); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// // no tags, no call -// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } +class ReplaceAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{ + {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, + ::testing::ValuesIn({internal_request_with_replace_header_config, + internal_request_with_replace_header_with_json_file_config, + internal_request_with_replace_header_with_yaml_file_config})); + +class ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, + ClearAlternateHeaderWhenUnmatchedAndSanitized) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{ + {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be + // removed + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + ClearAlternateHeaderWhenUnmatchedAndSanitized, + ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, + ::testing::ValuesIn({internal_request_with_replace_header_config, + internal_request_with_replace_header_with_json_file_config, + internal_request_with_replace_header_with_yaml_file_config})); + +class AppendForwardAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +TEST_P(AppendForwardAlternateHeaderFilterTest, AppendForwardAlternateHeader) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("foo,internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + AppendForwardAlternateHeader, AppendForwardAlternateHeaderFilterTest, + ::testing::ValuesIn({internal_request_with_append_or_add_config, + internal_request_with_append_or_add_with_json_file_config, + internal_request_with_append_or_add_with_yaml_file_config})); + +class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest { +}; + +TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, + RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + RetainAlternateHeaderWhenUnmatchedAndAppendForwarded, + RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, + ::testing::ValuesIn({internal_request_with_append_or_add_config, + internal_request_with_append_or_add_with_json_file_config, + internal_request_with_append_or_add_with_yaml_file_config})); + +class NestedPrefixesFilterTest : public IpTaggingFilterTest {}; + +TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-ip-tags", "test"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + + // There is no guarantee for the order tags are returned by the LC-Trie. + const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); + EXPECT_NE(std::string::npos, header_tag_data.find("test")); + EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); + EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(NestedPrefixes, NestedPrefixesFilterTest, + ::testing::ValuesIn({duplicate_request_config, + duplicate_request_with_json_file_config, + duplicate_request_with_yaml_file_config})); + +class Ipv6AddressTest : public IpTaggingFilterTest {}; + +TEST_P(Ipv6AddressTest, Ipv6Address) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, + ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, + ipv6_with_yaml_file_config})); + +class RuntimeDisabledTest : public IpTaggingFilterTest {}; + +TEST_P(RuntimeDisabledTest, RuntimeDisabled) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillOnce(Return(false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class ClearRouteCacheTest : public IpTaggingFilterTest {}; + +TEST_P(ClearRouteCacheTest, ClearRouteCache) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + // no tags, no call + EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +} + +INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); } // namespace } // namespace IpTagging diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json new file mode 100644 index 0000000000000..9e5aaedd7dfce --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json @@ -0,0 +1,22 @@ +{ + "ip_tags": [ + { + "ip_tag_name": "duplicate_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + }, + { + "ip_tag_name": "internal_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + } + ] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml new file mode 100644 index 0000000000000..c3679cd06da19 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml @@ -0,0 +1,7 @@ +ip_tags: + - ip_tag_name: duplicate_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json new file mode 100644 index 0000000000000..a30a77b87a62c --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json @@ -0,0 +1,13 @@ +{ + "ip_tags": [ + { + "ip_tag_name": "ipv6_request", + "ip_list": [ + { + "address_prefix": "2001:abcd:ef01:2345:6789:abcd:ef01:234", + "prefix_len": 64 + } + ] + } + ] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml new file mode 100644 index 0000000000000..d74fdb1ccc60d --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml @@ -0,0 +1,4 @@ +ip_tags: + - ip_tag_name: ipv6_request + ip_list: + - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} From 5705e64c336d06e3d757a4c069b214fa805ba8c7 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 18 Mar 2025 18:17:12 +0100 Subject: [PATCH 07/76] Add more tests Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.h | 3 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 49 +++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 6a18d698d6aed..818582e6e5df0 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -165,7 +165,8 @@ class IpTaggingFilterConfig { } void incCounter(Stats::StatName name); - + // Allow the unit test to have access to private members. + friend class IpTaggingFilterConfigPeer; const FilterRequestType request_type_; Stats::Scope& scope_; Runtime::Loader& runtime_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index d8ab533f84ec7..79b135c70f8a8 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -25,6 +25,21 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace IpTagging { + +class IpTaggingFilterConfigPeer { +public: + static IpTagsLoader& ipTagsLoader(IpTaggingFilterConfig& filter_config) { + return filter_config.tags_loader_; + } + static const std::shared_ptr& + ipTagsRegistry(const IpTaggingFilterConfig& filter_config) { + return filter_config.ip_tags_registry_; + } + static const std::string& ipTagsPath(const IpTaggingFilterConfig& filter_config) { + return filter_config.ip_tags_path_; + } +}; + namespace { std::shared_ptr ip_tags_registry; @@ -276,6 +291,34 @@ ip_tags_path: /test/tags.csv "Unsupported file format, unable to parse ip tags from file."); } +TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; + TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), + proto_config1); + auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); + const std::string config2_string = R"EOF( + request_type: external + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" + )EOF"; + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; + TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); + auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); + auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); + auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); + EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); + LcTrieSharedPtr ip_tags1 = + ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), + IpTaggingFilterConfigPeer::ipTagsLoader(*config1)); + LcTrieSharedPtr ip_tags2 = + ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), + IpTaggingFilterConfigPeer::ipTagsLoader(*config2)); + EXPECT_EQ(ip_tags1.get(), ip_tags2.get()); +} + class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { @@ -497,9 +540,9 @@ TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, ClearAlternateHeaderWhenUnmatchedAndSanitized) { const std::string config = GetParam(); initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // header will be - // removed + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; // header will + // be removed Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( From a86724e20d316275c1bf659a2b3a65fda571986a Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 20 Mar 2025 14:39:43 +0100 Subject: [PATCH 08/76] Add ip tags file reload test cases Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 79b135c70f8a8..414139e93a453 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -45,6 +45,9 @@ namespace { std::shared_ptr ip_tags_registry; namespace { + +const std::string ip_tagging_prefix = "prefix.ip_tagging."; + const std::string internal_request_config = R"EOF( request_type: internal ip_tags: @@ -319,6 +322,34 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { EXPECT_EQ(ip_tags1.get(), ip_tags2.get()); } +TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; + TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), + proto_config1); + auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); + const std::string config2_string = R"EOF( + request_type: external + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" + )EOF"; + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; + TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); + auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); + auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); + auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); + EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); + LcTrieSharedPtr ip_tags1 = + ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), + IpTaggingFilterConfigPeer::ipTagsLoader(*config1)); + LcTrieSharedPtr ip_tags2 = + ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), + IpTaggingFilterConfigPeer::ipTagsLoader(*config2)); + EXPECT_NE(ip_tags1.get(), ip_tags2.get()); +} + class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { @@ -727,6 +758,110 @@ INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, internal_request_with_json_file_config, internal_request_with_yaml_file_config})); +struct IpTagsFileReloadTestCase { + + IpTagsFileReloadTestCase() = default; + IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, + const std::string& remote_address, + const std::string& source_ip_tags_file_path, + const std::string& reloaded_ip_tags_file_path, + const std::string& hit_counter_prefix, + const Http::TestRequestHeaderMapImpl tagged_headers, + const Http::TestRequestHeaderMapImpl not_tagged_headers) + : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), + source_ip_tags_file_path_(source_ip_tags_file_path), + reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), + hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), + not_tagged_headers_(not_tagged_headers) {} + IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; + + std::string yaml_config_; + FilterRequestType request_type_; + std::string remote_address_; + std::string source_ip_tags_file_path_; + std::string reloaded_ip_tags_file_path_; + std::string hit_counter_prefix_; + Http::TestRequestHeaderMapImpl tagged_headers_; + Http::TestRequestHeaderMapImpl not_tagged_headers_; +}; + +class IpTagsFileReloadImplTest : public ::testing::TestWithParam { +public: + IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { + ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillByDefault(Return(true)); + } + + static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } + + void initializeFilter(const std::string& yaml) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + config_ = std::make_shared(config, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + validation_visitor_); + filter_ = std::make_unique(config_); + filter_->setDecoderFilterCallbacks(filter_callbacks_); + } + + ~IpTagsFileReloadImplTest() override { + if (filter_) { + filter_->onDestroy(); + } + } + + NiceMock stats_; + IpTaggingFilterConfigSharedPtr config_; + std::unique_ptr filter_; + NiceMock filter_callbacks_; + Buffer::OwnedImpl data_; + NiceMock runtime_; + Api::ApiPtr api_; + NiceMock validation_visitor_; +}; + +TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { + IpTagsFileReloadTestCase test_case = GetParam(); + initializeFilter(test_case.yaml_config_); + EXPECT_EQ(test_case.request_type_, config_->requestType()); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(test_case.tagged_headers_, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, + test_case.tagged_headers_.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(test_case.not_tagged_headers_, false)); + EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); + // std::string source_db_file_path = TestEnvironment::substitute(test_case.source_db_file_path_); + // std::string reloaded_db_file_path = + // TestEnvironment::substitute(test_case.reloaded_db_file_path_); + // TestEnvironment::renameFile(source_db_file_path, source_db_file_path + "1"); + // TestEnvironment::renameFile(reloaded_db_file_path, source_db_file_path); +} + +struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { + {internal_request_with_yaml_file_config, + FilterRequestType::INTERNAL, + "1.2.3.5", + "", + "", + "internal_request", + {{"x-envoy-internal", "true"}}, + {}}, +}; + +INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, + ::testing::ValuesIn(ip_tags_file_reload_test_cases)); + } // namespace } // namespace IpTagging } // namespace HttpFilters From def4cc0121b6fe0b7a29e89aa67772ff7c9a2bca Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 24 Mar 2025 18:51:33 +0100 Subject: [PATCH 09/76] Fix reload tests Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/config.cc | 1 + .../http/ip_tagging/ip_tagging_filter.cc | 18 +++++----- .../http/ip_tagging/ip_tagging_filter.h | 10 ++++-- .../http/ip_tagging/ip_tagging_filter_test.cc | 36 +++++++++++++++---- .../ip_tags_updated_internal_request.yaml | 4 +++ 5 files changed, 51 insertions(+), 18 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml diff --git a/source/extensions/filters/http/ip_tagging/config.cc b/source/extensions/filters/http/ip_tagging/config.cc index f32ed62f91e66..89cc666697b7e 100644 --- a/source/extensions/filters/http/ip_tagging/config.cc +++ b/source/extensions/filters/http/ip_tagging/config.cc @@ -24,6 +24,7 @@ Http::FilterFactoryCb IpTaggingFilterFactory::createFilterFactoryFromProtoTyped( IpTaggingFilterConfigSharedPtr config(new IpTaggingFilterConfig( proto_config, ip_tags_registry, stat_prefix, context.scope(), context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), + context.serverFactoryContext().mainThreadDispatcher(), context.messageValidationVisitor())); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index dfc45695da3f1..73d2f1eeacc4b 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -44,7 +44,6 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( std::vector cidr_set; cidr_set.reserve(ip_tag.ip_list().size()); for (const envoy::config::core::v3::CidrRange& entry : ip_tag.ip_list()) { - absl::StatusOr cidr_or_error = Network::Address::CidrRange::create(entry); if (cidr_or_error.status().ok()) { @@ -61,25 +60,26 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( return std::make_shared>(tag_data); } -LcTrieSharedPtr IpTagsRegistrySingleton::get(const std::string& ip_tags_path, - IpTagsLoader& tags_loader) { - LcTrieSharedPtr ip_tags; +IpTagsProviderSharedPtr IpTagsRegistrySingleton::get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + Api::Api& api, Event::Dispatcher& dispatcher, std::shared_ptr singleton) { + IpTagsProviderSharedPtr ip_tags_provider; const size_t key = std::hash()(ip_tags_path); absl::MutexLock lock(&mu_); auto it = ip_tags_registry_.find(key); if (it != ip_tags_registry_.end()) { - ip_tags = it->second.lock(); + ip_tags_provider = it->second.lock(); } else { - ip_tags = tags_loader.loadTags(ip_tags_path); - ip_tags_registry_[key] = ip_tags; + ip_tags_provider = std::make_shared(ip_tags_path, tags_loader, dispatcher, api, singleton); + ip_tags_registry_[key] = ip_tags_provider; } - return ip_tags; + return ip_tags_provider; } IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, std::shared_ptr ip_tags_registry, const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, + Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor) : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), stat_name_set_(scope.symbolTable().makeSet("IpTagging")), @@ -110,7 +110,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { - trie_ = ip_tags_registry_->get(ip_tags_path_, tags_loader_); + trie_ = ip_tags_registry_->get(ip_tags_path_, tags_loader_, dispatcher, api, ip_tags_registry_); } } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 818582e6e5df0..b2bfd452405b1 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -44,6 +44,7 @@ class IpTagsLoader { Stats::StatNameSetPtr& stat_name_set_; }; +//todo nezdolik move to impl file class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, @@ -90,6 +91,8 @@ class IpTagsProvider : public Logger::Loggable { const Singleton::InstanceSharedPtr owner_; }; +using IpTagsProviderSharedPtr = std::shared_ptr; + /** * A singleton for file based loading of ip tags and looking up parsed trie data structures with Ip * tags. When given equivalent file paths to the Ip tags, the singleton returns pointers to the same @@ -98,7 +101,8 @@ class IpTagsProvider : public Logger::Loggable { class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } - LcTrieSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader); + IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + Api::Api& api, Event::Dispatcher& dispatcher, std::shared_ptr singleton); private: absl::Mutex mu_; @@ -106,7 +110,7 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton // from being destroyed unless it's no longer keeping track of any providers. (The singleton // shared_ptr is *only* held by driver instances.) - absl::flat_hash_map>> + absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); }; @@ -128,7 +132,7 @@ class IpTaggingFilterConfig { IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, std::shared_ptr ip_tags_registry, const std::string& stat_prefix, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, + Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor); Runtime::Loader& runtime() { return runtime_; } diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 414139e93a453..8042e0d225281 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -48,6 +48,12 @@ namespace { const std::string ip_tagging_prefix = "prefix.ip_tagging."; +const std::string internal_request_file_path = + "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml"; + +const std::string updated_internal_request_file_path = + "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml"; + const std::string internal_request_config = R"EOF( request_type: internal ip_tags: @@ -234,8 +240,26 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); config_ = std::make_shared(config, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, + *stats_.rootScope(), runtime_, *api_, dispatcher_, validation_visitor_); + EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) + .WillRepeatedly(Invoke([this, &conditional] { + Filesystem::MockWatcher* mock_watcher = new NiceMock(); + EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) + .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, + Filesystem::Watcher::OnChangedCb cb) { + { + absl::WriterMutexLock lock(&mutex_); + on_changed_cbs_.reserve(1); + on_changed_cbs_.emplace_back(std::move(cb)); + } + if (conditional.has_value()) { + conditional->setReady(); + } + return absl::OkStatus(); + })); + return mock_watcher; + })); filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); } @@ -246,6 +270,7 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { } } + Event::MockDispatcher dispatcher_; NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; std::unique_ptr filter_; @@ -841,11 +866,10 @@ TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(test_case.not_tagged_headers_, false)); EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); - // std::string source_db_file_path = TestEnvironment::substitute(test_case.source_db_file_path_); - // std::string reloaded_db_file_path = - // TestEnvironment::substitute(test_case.reloaded_db_file_path_); - // TestEnvironment::renameFile(source_db_file_path, source_db_file_path + "1"); - // TestEnvironment::renameFile(reloaded_db_file_path, source_db_file_path); + std::string source_ip_tags_file_path = TestEnvironment::substitute(test_case.source_ip_tags_file_path_); + std::string reloaded_db_file_path = TestEnvironment::substitute(test_case.reloaded_db_file_path_); + TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); + TestEnvironment::renameFile(reloaded_db_file_path, source_ip_tags_file_path); } struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml new file mode 100644 index 0000000000000..10c43f7144c29 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml @@ -0,0 +1,4 @@ +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.7, prefix_len: 32} From c206d20c33cb9b46726c5eb2eeb5a533472d70ad Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Sun, 6 Apr 2025 20:01:29 +0200 Subject: [PATCH 10/76] More fixes Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/config.cc | 3 +- .../http/ip_tagging/ip_tagging_filter.cc | 66 +- .../http/ip_tagging/ip_tagging_filter.h | 10 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 1159 +++++++++-------- 4 files changed, 651 insertions(+), 587 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/config.cc b/source/extensions/filters/http/ip_tagging/config.cc index 89cc666697b7e..fa34a76239aeb 100644 --- a/source/extensions/filters/http/ip_tagging/config.cc +++ b/source/extensions/filters/http/ip_tagging/config.cc @@ -24,8 +24,7 @@ Http::FilterFactoryCb IpTaggingFilterFactory::createFilterFactoryFromProtoTyped( IpTaggingFilterConfigSharedPtr config(new IpTaggingFilterConfig( proto_config, ip_tags_registry, stat_prefix, context.scope(), context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), - context.serverFactoryContext().mainThreadDispatcher(), - context.messageValidationVisitor())); + context.serverFactoryContext().mainThreadDispatcher(), context.messageValidationVisitor())); return [config](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter(std::make_shared(config)); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 73d2f1eeacc4b..dd5998672b7ff 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -13,6 +13,44 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { +/** + * A singleton for file based loading of ip tags and looking up parsed trie data structures with Ip + * tags. When given equivalent file paths to the Ip tags, the singleton returns pointers to the same + * trie structure. + */ +class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { +public: + IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } + IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + Api::Api& api, Event::Dispatcher& dispatcher, + std::shared_ptr singleton); + +IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + Api::Api& api, Event::Dispatcher& dispatcher, + std::shared_ptr singleton) { + std::shared_ptr ip_tags_provider; + const uint64_t key = std::hash()(ip_tags_path); + absl::MutexLock lock(&mu_); + auto it = ip_tags_registry_.find(key); + if (it != ip_tags_registry_.end()) { + ip_tags_provider = it->second.lock(); + } else { + ip_tags_provider = + std::make_shared(ip_tags_path, tags_loader, dispatcher, api, singleton); + ip_tags_registry_[key] = ip_tags_provider; + } + return ip_tags_provider; +} + +private: + absl::Mutex mu_; + // We keep weak_ptr here so the trie structures can be destroyed if the config is updated to stop + // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton + // from being destroyed unless it's no longer keeping track of any providers. (The singleton + // shared_ptr is *only* held by driver instances.) + absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); +}; + IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} @@ -60,26 +98,13 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( return std::make_shared>(tag_data); } -IpTagsProviderSharedPtr IpTagsRegistrySingleton::get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Api::Api& api, Event::Dispatcher& dispatcher, std::shared_ptr singleton) { - IpTagsProviderSharedPtr ip_tags_provider; - const size_t key = std::hash()(ip_tags_path); - absl::MutexLock lock(&mu_); - auto it = ip_tags_registry_.find(key); - if (it != ip_tags_registry_.end()) { - ip_tags_provider = it->second.lock(); - } else { - ip_tags_provider = std::make_shared(ip_tags_path, tags_loader, dispatcher, api, singleton); - ip_tags_registry_[key] = ip_tags_provider; - } - return ip_tags_provider; -} + +SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, std::shared_ptr ip_tags_registry, const std::string& stat_prefix, - Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, - Event::Dispatcher& dispatcher, + Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor) : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), stat_name_set_(scope.symbolTable().makeSet("IpTagging")), @@ -110,7 +135,14 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { - trie_ = ip_tags_registry_->get(ip_tags_path_, tags_loader_, dispatcher, api, ip_tags_registry_); + std::cerr << "***Resolving provider for: " << ip_tags_path_ <get(ip_tags_path_, tags_loader_, api, dispatcher, ip_tags_registry_); + if (provider && provider->ipTags()) { + trie_ = provider->ipTags(); + } else { + throw EnvoyException("Failed to get ip tags from provider"); + } } } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index b2bfd452405b1..8f35646f786f7 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -44,7 +44,7 @@ class IpTagsLoader { Stats::StatNameSetPtr& stat_name_set_; }; -//todo nezdolik move to impl file +// todo nezdolik move to impl file class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, @@ -78,6 +78,8 @@ class IpTagsProvider : public Logger::Loggable { } }; + LcTrieSharedPtr ipTags() { return tags_; }; + absl::Status onIpTagsFileUpdate() { return absl::OkStatus(); } private: @@ -102,7 +104,8 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Api::Api& api, Event::Dispatcher& dispatcher, std::shared_ptr singleton); + Api::Api& api, Event::Dispatcher& dispatcher, + std::shared_ptr singleton); private: absl::Mutex mu_; @@ -110,8 +113,7 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton // from being destroyed unless it's no longer keeping track of any providers. (The singleton // shared_ptr is *only* held by driver instances.) - absl::flat_hash_map> - ip_tags_registry_ ABSL_GUARDED_BY(mu_); + absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); }; SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 8042e0d225281..62b51438a8d25 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -20,6 +20,7 @@ #include "gtest/gtest.h" using testing::Return; +using testing::InvokeWithoutArgs; namespace Envoy { namespace Extensions { @@ -237,29 +238,17 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } void initializeFilter(const std::string& yaml) { + EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { + Filesystem::MockWatcher* mock_watcher = new NiceMock(); + EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) + .WillRepeatedly(Return(absl::OkStatus())); + return mock_watcher; + })); envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); config_ = std::make_shared(config, ip_tags_registry, "prefix.", *stats_.rootScope(), runtime_, *api_, dispatcher_, validation_visitor_); - EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) - .WillRepeatedly(Invoke([this, &conditional] { - Filesystem::MockWatcher* mock_watcher = new NiceMock(); - EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, - Filesystem::Watcher::OnChangedCb cb) { - { - absl::WriterMutexLock lock(&mutex_); - on_changed_cbs_.reserve(1); - on_changed_cbs_.emplace_back(std::move(cb)); - } - if (conditional.has_value()) { - conditional->setReady(); - } - return absl::OkStatus(); - })); - return mock_watcher; - })); filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(filter_callbacks_); } @@ -320,571 +309,613 @@ ip_tags_path: /test/tags.csv } TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { + EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { + Filesystem::MockWatcher* mock_watcher = new NiceMock(); + EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) + .WillRepeatedly(Return(absl::OkStatus())); + return mock_watcher; + })); envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config1); auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, + *stats_.rootScope(), runtime_, *api_, dispatcher_, validation_visitor_); const std::string config2_string = R"EOF( - request_type: external + request_type: internal ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, - validation_visitor_); - auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); - auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); - EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); - LcTrieSharedPtr ip_tags1 = - ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), - IpTaggingFilterConfigPeer::ipTagsLoader(*config1)); - LcTrieSharedPtr ip_tags2 = - ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), - IpTaggingFilterConfigPeer::ipTagsLoader(*config2)); - EXPECT_EQ(ip_tags1.get(), ip_tags2.get()); -} - -TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { - envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; - TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), - proto_config1); - auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, - validation_visitor_); - const std::string config2_string = R"EOF( - request_type: external - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" - )EOF"; - envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; - TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); - auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, + *stats_.rootScope(), runtime_, *api_, dispatcher_, validation_visitor_); auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); - LcTrieSharedPtr ip_tags1 = + auto provider1 = ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), - IpTaggingFilterConfigPeer::ipTagsLoader(*config1)); - LcTrieSharedPtr ip_tags2 = + IpTaggingFilterConfigPeer::ipTagsLoader(*config1), + *api_, dispatcher_, ip_tags_registry1); + auto provider2 = ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), - IpTaggingFilterConfigPeer::ipTagsLoader(*config2)); - EXPECT_NE(ip_tags1.get(), ip_tags2.get()); + IpTaggingFilterConfigPeer::ipTagsLoader(*config2), + *api_, dispatcher_, ip_tags_registry1); + EXPECT_NE(nullptr, provider1); + EXPECT_NE(nullptr, provider2); + EXPECT_EQ(provider1->ipTags(), provider2->ipTags()); } -class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { - const std::string config = GetParam(); - initializeFilter(config); - EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - - // Check external requests don't get a tag. - request_headers = Http::TestRequestHeaderMapImpl{}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} - -INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, - ::testing::ValuesIn({internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); - -class ExternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { - const std::string config = GetParam(); - initializeFilter(config); - EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers; - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - - // Check internal requests don't get a tag. - request_headers = {{"x-envoy-internal", "true"}}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} - -INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, - ::testing::ValuesIn({external_request_config, - external_request_with_json_file_config, - external_request_with_yaml_file_config})); - -class BothRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { - const std::string config = GetParam(); - initializeFilter(config); - EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - request_headers = Http::TestRequestHeaderMapImpl{}; - remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -} - -INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, - ::testing::ValuesIn({both_request_config, - both_request_with_json_file_config, - both_request_with_yaml_file_config})); - -class NoHitsIpTaggingFilterTest : public IpTaggingFilterTest {}; - -TEST_P(NoHitsIpTaggingFilterTest, NoHits) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, - ::testing::ValuesIn({internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); - -class AppendEntryFilterTest : public IpTaggingFilterTest {}; - -TEST_P(AppendEntryFilterTest, AppendEntry) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-ip-tags", "test"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, - ::testing::ValuesIn({internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); - -class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest : public IpTaggingFilterTest {}; - -TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, - ReplaceAlternateHeaderWhenActionIsDefaulted) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}, // foo will be removed - {"x-envoy-optional-header", "bar"}, // bar will be removed - {"x-envoy-optional-header", "baz"}, // baz will be removed - }; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, - ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, - ::testing::ValuesIn({internal_request_with_header_config, - internal_request_with_header_with_json_file_config, - internal_request_with_header_with_yaml_file_config})); - -class ReplaceAlternateHeaderFilterTest : public IpTaggingFilterTest {}; - -TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{ - {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P( - ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, - ::testing::ValuesIn({internal_request_with_replace_header_config, - internal_request_with_replace_header_with_json_file_config, - internal_request_with_replace_header_with_yaml_file_config})); - -class ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest : public IpTaggingFilterTest {}; - -TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, - ClearAlternateHeaderWhenUnmatchedAndSanitized) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}}; // header will - // be removed - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P( - ClearAlternateHeaderWhenUnmatchedAndSanitized, - ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, - ::testing::ValuesIn({internal_request_with_replace_header_config, - internal_request_with_replace_header_with_json_file_config, - internal_request_with_replace_header_with_yaml_file_config})); - -class AppendForwardAlternateHeaderFilterTest : public IpTaggingFilterTest {}; - -TEST_P(AppendForwardAlternateHeaderFilterTest, AppendForwardAlternateHeader) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}}; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("foo,internal_request_with_optional_header", - request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P( - AppendForwardAlternateHeader, AppendForwardAlternateHeaderFilterTest, - ::testing::ValuesIn({internal_request_with_append_or_add_config, - internal_request_with_append_or_add_with_json_file_config, - internal_request_with_append_or_add_with_yaml_file_config})); - -class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest { -}; - -TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, - RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-optional-header", "foo"}}; - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P( - RetainAlternateHeaderWhenUnmatchedAndAppendForwarded, - RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, - ::testing::ValuesIn({internal_request_with_append_or_add_config, - internal_request_with_append_or_add_with_json_file_config, - internal_request_with_append_or_add_with_yaml_file_config})); - -class NestedPrefixesFilterTest : public IpTaggingFilterTest {}; - -TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, - {"x-envoy-ip-tags", "test"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - - // There is no guarantee for the order tags are returned by the LC-Trie. - const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); - EXPECT_NE(std::string::npos, header_tag_data.find("test")); - EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); - EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(NestedPrefixes, NestedPrefixesFilterTest, - ::testing::ValuesIn({duplicate_request_config, - duplicate_request_with_json_file_config, - duplicate_request_with_yaml_file_config})); - -class Ipv6AddressTest : public IpTaggingFilterTest {}; - -TEST_P(Ipv6AddressTest, Ipv6Address) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, - ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, - ipv6_with_yaml_file_config})); - -class RuntimeDisabledTest : public IpTaggingFilterTest {}; - -TEST_P(RuntimeDisabledTest, RuntimeDisabled) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillOnce(Return(false)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -} - -INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, - ::testing::ValuesIn({internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); - -class ClearRouteCacheTest : public IpTaggingFilterTest {}; - -TEST_P(ClearRouteCacheTest, ClearRouteCache) { - const std::string config = GetParam(); - initializeFilter(config); - Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - - EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - - // no tags, no call - EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); - request_headers = Http::TestRequestHeaderMapImpl{}; - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -} - -INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, - ::testing::ValuesIn({internal_request_config, - internal_request_with_json_file_config, - internal_request_with_yaml_file_config})); - -struct IpTagsFileReloadTestCase { - - IpTagsFileReloadTestCase() = default; - IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, - const std::string& remote_address, - const std::string& source_ip_tags_file_path, - const std::string& reloaded_ip_tags_file_path, - const std::string& hit_counter_prefix, - const Http::TestRequestHeaderMapImpl tagged_headers, - const Http::TestRequestHeaderMapImpl not_tagged_headers) - : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), - source_ip_tags_file_path_(source_ip_tags_file_path), - reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), - hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), - not_tagged_headers_(not_tagged_headers) {} - IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; - - std::string yaml_config_; - FilterRequestType request_type_; - std::string remote_address_; - std::string source_ip_tags_file_path_; - std::string reloaded_ip_tags_file_path_; - std::string hit_counter_prefix_; - Http::TestRequestHeaderMapImpl tagged_headers_; - Http::TestRequestHeaderMapImpl not_tagged_headers_; -}; - -class IpTagsFileReloadImplTest : public ::testing::TestWithParam { -public: - IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { - ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillByDefault(Return(true)); - } - - static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } - - void initializeFilter(const std::string& yaml) { - envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); - config_ = std::make_shared(config, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, - validation_visitor_); - filter_ = std::make_unique(config_); - filter_->setDecoderFilterCallbacks(filter_callbacks_); - } - - ~IpTagsFileReloadImplTest() override { - if (filter_) { - filter_->onDestroy(); - } - } - - NiceMock stats_; - IpTaggingFilterConfigSharedPtr config_; - std::unique_ptr filter_; - NiceMock filter_callbacks_; - Buffer::OwnedImpl data_; - NiceMock runtime_; - Api::ApiPtr api_; - NiceMock validation_visitor_; -}; - -TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { - IpTagsFileReloadTestCase test_case = GetParam(); - initializeFilter(test_case.yaml_config_); - EXPECT_EQ(test_case.request_type_, config_->requestType()); - Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); - filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( - remote_address); - EXPECT_CALL(stats_, - counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->decodeHeaders(test_case.tagged_headers_, false)); - EXPECT_EQ(test_case.hit_counter_prefix_, - test_case.tagged_headers_.get_(Http::Headers::get().EnvoyIpTags)); - EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); - Http::TestRequestTrailerMapImpl request_trailers; - EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->decodeHeaders(test_case.not_tagged_headers_, false)); - EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); - std::string source_ip_tags_file_path = TestEnvironment::substitute(test_case.source_ip_tags_file_path_); - std::string reloaded_db_file_path = TestEnvironment::substitute(test_case.reloaded_db_file_path_); - TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); - TestEnvironment::renameFile(reloaded_db_file_path, source_ip_tags_file_path); -} - -struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { - {internal_request_with_yaml_file_config, - FilterRequestType::INTERNAL, - "1.2.3.5", - "", - "", - "internal_request", - {{"x-envoy-internal", "true"}}, - {}}, -}; - -INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, - ::testing::ValuesIn(ip_tags_file_reload_test_cases)); +// TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; +// TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), +// proto_config1); +// auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", +// *stats_.rootScope(), runtime_, *api_, dispatcher_, +// validation_visitor_); +// const std::string config2_string = R"EOF( +// request_type: external +// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" +// )EOF"; +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; +// TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); +// auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", +// *stats_.rootScope(), runtime_, *api_, dispatcher_, +// validation_visitor_); +// auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); +// auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); +// EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); +// auto provider1 = +// ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), +// IpTaggingFilterConfigPeer::ipTagsLoader(*config1), +// *api_, dispatcher_, ip_tags_registry1); +// auto provider2 = +// ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), +// IpTaggingFilterConfigPeer::ipTagsLoader(*config2), +// *api_, dispatcher_, ip_tags_registry1); +// EXPECT_NE(nullptr, provider1); +// EXPECT_NE(nullptr, provider2); +// EXPECT_NE(provider1->ipTags(), provider2->ipTags()); +// } + +// class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { +// const std::string config = GetParam(); +// initializeFilter(config); +// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check external requests don't get a tag. +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, +// ::testing::ValuesIn({internal_request_config, +// internal_request_with_json_file_config, +// internal_request_with_yaml_file_config})); + +// class ExternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { +// const std::string config = GetParam(); +// initializeFilter(config); +// EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers; + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + +// // Check internal requests don't get a tag. +// request_headers = {{"x-envoy-internal", "true"}}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, +// ::testing::ValuesIn({external_request_config, +// external_request_with_json_file_config, +// external_request_with_yaml_file_config})); + +// class BothRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { +// const std::string config = GetParam(); +// initializeFilter(config); +// EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// request_headers = Http::TestRequestHeaderMapImpl{}; +// remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); +// } + +// INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, +// ::testing::ValuesIn({both_request_config, +// both_request_with_json_file_config, +// both_request_with_yaml_file_config})); + +// class NoHitsIpTaggingFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(NoHitsIpTaggingFilterTest, NoHits) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, +// ::testing::ValuesIn({internal_request_config, +// internal_request_with_json_file_config, +// internal_request_with_yaml_file_config})); + +// class AppendEntryFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(AppendEntryFilterTest, AppendEntry) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-ip-tags", "test"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, +// ::testing::ValuesIn({internal_request_config, +// internal_request_with_json_file_config, +// internal_request_with_yaml_file_config})); + +// class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, +// ReplaceAlternateHeaderWhenActionIsDefaulted) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{ +// {"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}, // foo will be removed +// {"x-envoy-optional-header", "bar"}, // bar will be removed +// {"x-envoy-optional-header", "baz"}, // baz will be removed +// }; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, +// ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, +// ::testing::ValuesIn({internal_request_with_header_config, +// internal_request_with_header_with_json_file_config, +// internal_request_with_header_with_yaml_file_config})); + +// class ReplaceAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{ +// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P( +// ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, +// ::testing::ValuesIn({internal_request_with_replace_header_config, +// internal_request_with_replace_header_with_json_file_config, +// internal_request_with_replace_header_with_yaml_file_config})); + +// class ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, +// ClearAlternateHeaderWhenUnmatchedAndSanitized) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}}; // header will +// // be removed +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P( +// ClearAlternateHeaderWhenUnmatchedAndSanitized, +// ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, +// ::testing::ValuesIn({internal_request_with_replace_header_config, +// internal_request_with_replace_header_with_json_file_config, +// internal_request_with_replace_header_with_yaml_file_config})); + +// class AppendForwardAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(AppendForwardAlternateHeaderFilterTest, AppendForwardAlternateHeader) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}}; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("foo,internal_request_with_optional_header", +// request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P( +// AppendForwardAlternateHeader, AppendForwardAlternateHeaderFilterTest, +// ::testing::ValuesIn({internal_request_with_append_or_add_config, +// internal_request_with_append_or_add_with_json_file_config, +// internal_request_with_append_or_add_with_yaml_file_config})); + +// class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest { +// }; + +// TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, +// RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-optional-header", "foo"}}; +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P( +// RetainAlternateHeaderWhenUnmatchedAndAppendForwarded, +// RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, +// ::testing::ValuesIn({internal_request_with_append_or_add_config, +// internal_request_with_append_or_add_with_json_file_config, +// internal_request_with_append_or_add_with_yaml_file_config})); + +// class NestedPrefixesFilterTest : public IpTaggingFilterTest {}; + +// TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, +// {"x-envoy-ip-tags", "test"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); +// EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + +// // There is no guarantee for the order tags are returned by the LC-Trie. +// const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); +// EXPECT_NE(std::string::npos, header_tag_data.find("test")); +// EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); +// EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(NestedPrefixes, NestedPrefixesFilterTest, +// ::testing::ValuesIn({duplicate_request_config, +// duplicate_request_with_json_file_config, +// duplicate_request_with_yaml_file_config})); + +// class Ipv6AddressTest : public IpTaggingFilterTest {}; + +// TEST_P(Ipv6AddressTest, Ipv6Address) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, +// ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, +// ipv6_with_yaml_file_config})); + +// class RuntimeDisabledTest : public IpTaggingFilterTest {}; + +// TEST_P(RuntimeDisabledTest, RuntimeDisabled) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) +// .WillOnce(Return(false)); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// } + +// INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, +// ::testing::ValuesIn({internal_request_config, +// internal_request_with_json_file_config, +// internal_request_with_yaml_file_config})); + +// class ClearRouteCacheTest : public IpTaggingFilterTest {}; + +// TEST_P(ClearRouteCacheTest, ClearRouteCache) { +// const std::string config = GetParam(); +// initializeFilter(config); +// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); + +// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + +// // no tags, no call +// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); +// request_headers = Http::TestRequestHeaderMapImpl{}; +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); +// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +// } + +// INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, +// ::testing::ValuesIn({internal_request_config, +// internal_request_with_json_file_config, +// internal_request_with_yaml_file_config})); + +// struct IpTagsFileReloadTestCase { + +// IpTagsFileReloadTestCase() = default; +// IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, +// const std::string& remote_address, +// const std::string& source_ip_tags_file_path, +// const std::string& reloaded_ip_tags_file_path, +// const std::string& hit_counter_prefix, +// const Http::TestRequestHeaderMapImpl tagged_headers, +// const Http::TestRequestHeaderMapImpl not_tagged_headers) +// : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), +// source_ip_tags_file_path_(source_ip_tags_file_path), +// reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), +// hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), +// not_tagged_headers_(not_tagged_headers) {} +// IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; + +// std::string yaml_config_; +// FilterRequestType request_type_; +// std::string remote_address_; +// std::string source_ip_tags_file_path_; +// std::string reloaded_ip_tags_file_path_; +// std::string hit_counter_prefix_; +// Http::TestRequestHeaderMapImpl tagged_headers_; +// Http::TestRequestHeaderMapImpl not_tagged_headers_; +// }; + +// class IpTagsFileReloadImplTest : public ::testing::TestWithParam { +// public: +// IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { +// ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) +// .WillByDefault(Return(true)); +// } + +// static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } + +// void initializeFilter(const std::string& yaml, +// absl::optional& conditional) { +// EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) +// .WillRepeatedly(Invoke([this, &conditional] { +// Filesystem::MockWatcher* mock_watcher = new NiceMock(); +// EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) +// .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, +// Filesystem::Watcher::OnChangedCb cb) { +// { +// absl::WriterMutexLock lock(&mutex_); +// on_changed_cbs_.reserve(1); +// on_changed_cbs_.emplace_back(std::move(cb)); +// } +// if (conditional.has_value()) { +// conditional->setReady(); +// } +// return absl::OkStatus(); +// })); +// return mock_watcher; +// })); +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; +// TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); +// config_ = std::make_shared(config, ip_tags_registry, "prefix.", +// *stats_.rootScope(), runtime_, *api_, dispatcher_, +// validation_visitor_); +// filter_ = std::make_unique(config_); +// filter_->setDecoderFilterCallbacks(filter_callbacks_); +// } + +// ~IpTagsFileReloadImplTest() override { +// if (filter_) { +// filter_->onDestroy(); +// } +// } + +// Event::MockDispatcher dispatcher_; +// NiceMock stats_; +// IpTaggingFilterConfigSharedPtr config_; +// std::unique_ptr filter_; +// NiceMock filter_callbacks_; +// Buffer::OwnedImpl data_; +// NiceMock runtime_; +// Api::ApiPtr api_; +// NiceMock validation_visitor_; +// absl::Mutex mutex_; +// std::vector on_changed_cbs_ ABSL_GUARDED_BY(mutex_); +// absl::optional cb_added_nullopt = absl::nullopt; +// }; + +// TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { +// IpTagsFileReloadTestCase test_case = GetParam(); +// initializeFilter(test_case.yaml_config_, cb_added_nullopt); +// EXPECT_EQ(test_case.request_type_, config_->requestType()); +// Network::Address::InstanceConstSharedPtr remote_address = +// Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); +// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( +// remote_address); +// EXPECT_CALL(stats_, +// counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); +// EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, +// filter_->decodeHeaders(test_case.tagged_headers_, false)); +// EXPECT_EQ(test_case.hit_counter_prefix_, +// test_case.tagged_headers_.get_(Http::Headers::get().EnvoyIpTags)); +// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); +// Http::TestRequestTrailerMapImpl request_trailers; +// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +// EXPECT_EQ(Http::FilterHeadersStatus::Continue, +// filter_->decodeHeaders(test_case.not_tagged_headers_, false)); +// EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); +// std::string source_ip_tags_file_path = TestEnvironment::substitute(test_case.source_ip_tags_file_path_); +// std::string reloaded_ip_tags_file_path = TestEnvironment::substitute(test_case.reloaded_ip_tags_file_path_); +// std::cerr << "source_ip_tags_file_path: " << source_ip_tags_file_path << std::endl; +// std::cerr << "reloaded_ip_tags_file_path: " << reloaded_ip_tags_file_path << std::endl; +// TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); +// TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); +// // Clean up modifications to ip tags file names. +// TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); +// TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); +// } + +// struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { +// {internal_request_with_yaml_file_config, +// FilterRequestType::INTERNAL, +// "1.2.3.5", +// internal_request_file_path, +// updated_internal_request_file_path, +// "internal_request", +// {{"x-envoy-internal", "true"}}, +// {}}, +// }; + +// INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, +// ::testing::ValuesIn(ip_tags_file_reload_test_cases)); } // namespace } // namespace IpTagging From dedeca0026e4786354dd451f7af0808463b4cc14 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 23 Apr 2025 15:28:24 +0200 Subject: [PATCH 11/76] Add base reload test Signed-off-by: Kateryna Nezdolii --- source/common/event/dispatcher_impl.cc | 6 +- source/common/runtime/runtime_features.cc | 2 +- .../http/ip_tagging/ip_tagging_filter.cc | 111 ++-- .../http/ip_tagging/ip_tagging_filter.h | 102 ++-- .../http/ip_tagging/ip_tagging_filter_test.cc | 556 +++++++++--------- .../ip_tags_updated_external_request.json | 22 + .../ip_tags_updated_internal_request.yaml | 4 +- 7 files changed, 454 insertions(+), 349 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 5295f7eb67f8d..9a29e50cd77dc 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -90,6 +90,7 @@ DispatcherImpl::DispatcherImpl(const std::string& name, Thread::ThreadFactory& t DispatcherImpl::~DispatcherImpl() { ENVOY_LOG(debug, "destroying dispatcher {}", name_); + std::cerr << "deleting dispatcher" << std::endl; FatalErrorHandler::removeFatalErrorHandler(*this); // TODO(lambdai): Resolve https://github.com/envoyproxy/envoy/issues/15072 and enable // ASSERT(deletable_in_dispatcher_thread_.empty()) @@ -237,7 +238,10 @@ void DispatcherImpl::deferredDelete(DeferredDeletablePtr&& to_delete) { } } -void DispatcherImpl::exit() { base_scheduler_.loopExit(); } +void DispatcherImpl::exit() { + std::cerr << "*****Exiting dispatcher" << std::endl; + base_scheduler_.loopExit(); +} SignalEventPtr DispatcherImpl::listenForSignal(signal_t signal_num, SignalCb cb) { ASSERT(isThreadSafe()); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index a719a94d26160..dcd02cfbedbab 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -107,7 +107,7 @@ RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); RUNTIME_GUARD(envoy_reloadable_features_wait_for_first_byte_before_balsa_msg_done); RUNTIME_GUARD(envoy_reloadable_features_xds_failover_to_primary_enabled); RUNTIME_GUARD(envoy_reloadable_features_xds_prevent_resource_copy); -RUNTIME_GUARD(envoy_restart_features_fix_dispatcher_approximate_now); +FALSE_RUNTIME_GUARD(envoy_restart_features_fix_dispatcher_approximate_now); RUNTIME_GUARD(envoy_restart_features_skip_backing_cluster_check_for_sds); RUNTIME_GUARD(envoy_restart_features_use_eds_cache_for_ads); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index dd5998672b7ff..03b753eee54ac 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -13,44 +13,64 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -/** - * A singleton for file based loading of ip tags and looking up parsed trie data structures with Ip - * tags. When given equivalent file paths to the Ip tags, the singleton returns pointers to the same - * trie structure. - */ -class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { -public: - IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } - IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Api::Api& api, Event::Dispatcher& dispatcher, - std::shared_ptr singleton); - -IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Api::Api& api, Event::Dispatcher& dispatcher, - std::shared_ptr singleton) { - std::shared_ptr ip_tags_provider; - const uint64_t key = std::hash()(ip_tags_path); - absl::MutexLock lock(&mu_); - auto it = ip_tags_registry_.find(key); - if (it != ip_tags_registry_.end()) { - ip_tags_provider = it->second.lock(); - } else { - ip_tags_provider = - std::make_shared(ip_tags_path, tags_loader, dispatcher, api, singleton); - ip_tags_registry_[key] = ip_tags_provider; +IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + IpTagsReloadSuccessCb reload_success_cb, + IpTagsReloadErrorCb reload_error_cb, Event::Dispatcher& dispatcher, + Api::Api& api, Singleton::InstanceSharedPtr owner) + : ip_tags_path_(ip_tags_path), tags_loader_(tags_loader), reload_success_cb_(reload_success_cb), + reload_error_cb_(reload_error_cb), tags_(tags_loader_.loadTags(ip_tags_path_)), + ip_tags_reload_dispatcher_(api.allocateDispatcher("ip_tags_reload_routine")), + ip_tags_file_watcher_(dispatcher.createFilesystemWatcher()), owner_(owner) { + if (ip_tags_path.empty()) { + throw EnvoyException("Cannot load tags from empty file path."); } - return ip_tags_provider; + ip_tags_reload_thread_ = api.threadFactory().createThread( + [this]() -> void { + ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); + THROW_IF_NOT_OK( + ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, + [this](uint32_t) { return onIpTagsFileUpdate(); })); + ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); + }, + Thread::Options{std::string("ip_tags_reload_routine")}); } -private: - absl::Mutex mu_; - // We keep weak_ptr here so the trie structures can be destroyed if the config is updated to stop - // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton - // from being destroyed unless it's no longer keeping track of any providers. (The singleton - // shared_ptr is *only* held by driver instances.) - absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); +IpTagsProvider::~IpTagsProvider() { + ENVOY_LOG(debug, "Shutting down ip tags provider"); + ip_tags_reload_dispatcher_->exit(); + if (ip_tags_reload_thread_) { + ip_tags_reload_thread_->join(); + ip_tags_reload_thread_.reset(); + } +}; + +LcTrieSharedPtr IpTagsProvider::ipTags() const ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { + absl::ReaderMutexLock lock(&ip_tags_mutex_); + return tags_; }; +absl::Status IpTagsProvider::onIpTagsFileUpdate() { + LcTrieSharedPtr reloaded_tags = tags_loader_.loadTags(ip_tags_path_); + return ipTagsReload(reloaded_tags); +} + +absl::Status IpTagsProvider::ipTagsReload(const LcTrieSharedPtr reloaded_tags) { + if (reloaded_tags) { + updateIpTags(reloaded_tags); + reload_success_cb_(); + } else { + reload_error_cb_(); + } + return absl::OkStatus(); +} + +void IpTagsProvider::updateIpTags(const LcTrieSharedPtr reloaded_tags) + ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { + absl::MutexLock lock(&ip_tags_mutex_); + tags_ = reloaded_tags; + std::cerr << "***Finishes reloading ip tags" << std::endl; +} + IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} @@ -92,15 +112,13 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( entry.address_prefix(), entry.prefix_len().value())); } } + std::cerr << "***PArsing ip tag: " << ip_tag.ip_tag_name() << std::endl; tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } return std::make_shared>(tag_data); } - -SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); - IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, std::shared_ptr ip_tags_registry, const std::string& stat_prefix, @@ -135,14 +153,17 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { - std::cerr << "***Resolving provider for: " << ip_tags_path_ <get(ip_tags_path_, tags_loader_, api, dispatcher, ip_tags_registry_); - if (provider && provider->ipTags()) { - trie_ = provider->ipTags(); + std::cerr << "***Resolving provider for: " << ip_tags_path_ << std::endl; + provider_ = ip_tags_registry_->get( + ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, + [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_); + if (provider_ && provider_->ipTags()) { + trie_ = provider_->ipTags(); } else { throw EnvoyException("Failed to get ip tags from provider"); } + stat_name_set_->rememberBuiltin("ip_tags_reload_success"); + stat_name_set_->rememberBuiltin("ip_tags_reload_error"); } } @@ -151,13 +172,14 @@ void IpTaggingFilterConfig::incCounter(Stats::StatName name) { scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } -IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {} +IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config){}; IpTaggingFilter::~IpTaggingFilter() = default; void IpTaggingFilter::onDestroy() {} Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { + std::cerr << "***Decode headers called: " << std::endl; const bool is_internal_request = headers.EnvoyInternalRequest() && (headers.EnvoyInternalRequest()->value() == Http::Headers::get().EnvoyInternalRequestValues.True.c_str()); @@ -170,7 +192,10 @@ Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& std::vector tags = config_->trie().getData(callbacks_->streamInfo().downstreamAddressProvider().remoteAddress()); - + std::cerr << "***Tags from decodeHeaders: " << std::endl; + for (auto t : tags) + std::cerr << t << ' '; + std::cerr << std::endl; applyTags(headers, tags); if (!tags.empty()) { // For a large number(ex > 1000) of tags, stats cardinality will be an issue. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 8f35646f786f7..cda2399424e46 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -26,7 +26,6 @@ using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; using LcTrieSharedPtr = std::shared_ptr>; // TODO supports stats for ip tags -// Add tests for singleton // Support async reload of tags file class IpTagsLoader { public: @@ -44,48 +43,30 @@ class IpTagsLoader { Stats::StatNameSetPtr& stat_name_set_; }; -// todo nezdolik move to impl file +using IpTagsReloadSuccessCb = std::function; +using IpTagsReloadErrorCb = std::function; + class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner) - : ip_tags_path_(ip_tags_path), tags_loader_(tags_loader), owner_(owner) { - if (ip_tags_path.empty()) { - throw EnvoyException("Cannot load tags from empty file path."); - } - tags_ = tags_loader_.loadTags(ip_tags_path_); - ip_tags_reload_dispatcher_ = api.allocateDispatcher("ip_tags_reload_routine"); - ip_tags_file_watcher_ = dispatcher.createFilesystemWatcher(); - ip_tags_reload_thread_ = api.threadFactory().createThread( - [this]() -> void { - ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); - THROW_IF_NOT_OK( - ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, - [this](uint32_t) { return onIpTagsFileUpdate(); })); - ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); - }, - Thread::Options{std::string("ip_tags_reload_routine")}); - } + IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, + Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner); - ~IpTagsProvider() { - ENVOY_LOG(debug, "Shutting down ip tags provider"); - if (ip_tags_reload_dispatcher_) { - ip_tags_reload_dispatcher_->exit(); - } - if (ip_tags_reload_thread_) { - ip_tags_reload_thread_->join(); - ip_tags_reload_thread_.reset(); - } - }; + ~IpTagsProvider(); - LcTrieSharedPtr ipTags() { return tags_; }; + LcTrieSharedPtr ipTags() const ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); - absl::Status onIpTagsFileUpdate() { return absl::OkStatus(); } + absl::Status onIpTagsFileUpdate(); + absl::Status ipTagsReload(const LcTrieSharedPtr reloaded_tags); + void updateIpTags(const LcTrieSharedPtr reloaded_tags) ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); private: const std::string ip_tags_path_; IpTagsLoader& tags_loader_; - LcTrieSharedPtr tags_; + IpTagsReloadSuccessCb reload_success_cb_; + IpTagsReloadErrorCb reload_error_cb_; + mutable absl::Mutex ip_tags_mutex_; + LcTrieSharedPtr tags_ ABSL_GUARDED_BY(ip_tags_mutex_); Thread::ThreadPtr ip_tags_reload_thread_; Event::DispatcherPtr ip_tags_reload_dispatcher_; Filesystem::WatcherPtr ip_tags_file_watcher_; @@ -103,14 +84,40 @@ using IpTagsProviderSharedPtr = std::shared_ptr; class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } - IpTagsProviderSharedPtr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - Api::Api& api, Event::Dispatcher& dispatcher, - std::shared_ptr singleton); + + std::shared_ptr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + IpTagsReloadSuccessCb reload_success_cb, + IpTagsReloadErrorCb reload_error_cb, Api::Api& api, + Event::Dispatcher& dispatcher, + std::shared_ptr singleton) { + std::shared_ptr ip_tags_provider; + const uint64_t key = std::hash()(ip_tags_path); + absl::MutexLock lock(&mu_); + auto it = ip_tags_registry_.find(key); + std::cerr << "****Found provider " << std::endl; + if (it != ip_tags_registry_.end()) { + if (std::shared_ptr provider = it->second.lock()) { + std::cerr << "****Returning existing provider " << std::endl; + ip_tags_provider = provider; + } else { + ip_tags_provider = + std::make_shared(ip_tags_path, tags_loader, reload_success_cb, + reload_error_cb, dispatcher, api, singleton); + ip_tags_registry_[key] = ip_tags_provider; + } + } else { + std::cerr << "****Creating new provider " << std::endl; + ip_tags_provider = + std::make_shared(ip_tags_path, tags_loader, reload_success_cb, + reload_error_cb, dispatcher, api, singleton); + ip_tags_registry_[key] = ip_tags_provider; + } + return ip_tags_provider; + } private: absl::Mutex mu_; - // We keep weak_ptr here so the trie structures can be destroyed if the config is updated to stop - // using that trie. Each provider stores shared_ptrs to this singleton, which keeps the singleton + // Each provider stores shared_ptrs to this singleton, which keeps the singleton // from being destroyed unless it's no longer keeping track of any providers. (The singleton // shared_ptr is *only* held by driver instances.) absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); @@ -139,7 +146,13 @@ class IpTaggingFilterConfig { Runtime::Loader& runtime() { return runtime_; } FilterRequestType requestType() const { return request_type_; } - const Network::LcTrie::LcTrie& trie() const { return *trie_; } + const Network::LcTrie::LcTrie& trie() const { + if (provider_) { + return *(provider_->ipTags()); + } else { + return *trie_; + } + } OptRef ipTagHeader() const { if (ip_tag_header_.get().empty()) { @@ -152,6 +165,14 @@ class IpTaggingFilterConfig { void incHit(absl::string_view tag) { incCounter(stat_name_set_->getBuiltin(absl::StrCat(tag, ".hit"), unknown_tag_)); } + + void incIpTagsReloadSuccess() { + incCounter(stat_name_set_->getBuiltin("ip_tags_reload_success", unknown_tag_)); + } + void incIpTagsReloadError() { + incCounter(stat_name_set_->getBuiltin("ip_tags_reload_error", unknown_tag_)); + } + void incNoHit() { incCounter(no_hit_); } void incTotal() { incCounter(total_); } @@ -181,7 +202,6 @@ class IpTaggingFilterConfig { const Stats::StatName no_hit_; const Stats::StatName total_; const Stats::StatName unknown_tag_; - LcTrieSharedPtr trie_; const Http::LowerCaseString ip_tag_header_; // An empty string indicates that no ip_tag_header is set. const HeaderAction ip_tag_header_action_; @@ -190,6 +210,8 @@ class IpTaggingFilterConfig { // are in use. const std::shared_ptr ip_tags_registry_; IpTagsLoader tags_loader_; + LcTrieSharedPtr trie_; + std::shared_ptr provider_; }; using IpTaggingFilterConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 62b51438a8d25..26dcf86580e50 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -19,8 +19,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::Return; using testing::InvokeWithoutArgs; +using testing::Return; namespace Envoy { namespace Extensions { @@ -32,6 +32,9 @@ class IpTaggingFilterConfigPeer { static IpTagsLoader& ipTagsLoader(IpTaggingFilterConfig& filter_config) { return filter_config.tags_loader_; } + static std::shared_ptr ipTagsProvider(IpTaggingFilterConfig& filter_config) { + return filter_config.provider_; + } static const std::shared_ptr& ipTagsRegistry(const IpTaggingFilterConfig& filter_config) { return filter_config.ip_tags_registry_; @@ -49,12 +52,6 @@ namespace { const std::string ip_tagging_prefix = "prefix.ip_tagging."; -const std::string internal_request_file_path = - "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml"; - -const std::string updated_internal_request_file_path = - "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml"; - const std::string internal_request_config = R"EOF( request_type: internal ip_tags: @@ -228,154 +225,142 @@ ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_da } // namespace -class IpTaggingFilterTest : public ::testing::TestWithParam { -public: - IpTaggingFilterTest() : api_(Api::createApiForTest()) { - ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillByDefault(Return(true)); - } +// class IpTaggingFilterTest : public ::testing::TestWithParam { +// public: +// IpTaggingFilterTest() : api_(Api::createApiForTest()) { +// ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) +// .WillByDefault(Return(true)); +// EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { +// Filesystem::MockWatcher* mock_watcher = new NiceMock(); +// EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) +// .WillRepeatedly(Return(absl::OkStatus())); +// return mock_watcher; +// })); +// } - static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } +// static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); +// } - void initializeFilter(const std::string& yaml) { - EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { - Filesystem::MockWatcher* mock_watcher = new NiceMock(); - EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly(Return(absl::OkStatus())); - return mock_watcher; - })); - envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); - config_ = std::make_shared(config, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, dispatcher_, - validation_visitor_); - filter_ = std::make_unique(config_); - filter_->setDecoderFilterCallbacks(filter_callbacks_); - } +// void initializeFilter(const std::string& yaml) { +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; +// TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); +// config_ = std::make_shared(config, ip_tags_registry, "prefix.", +// *stats_.rootScope(), runtime_, *api_, +// dispatcher_, validation_visitor_); +// filter_ = std::make_unique(config_); +// filter_->setDecoderFilterCallbacks(filter_callbacks_); +// } - ~IpTaggingFilterTest() override { - if (filter_) { - filter_->onDestroy(); - } - } +// ~IpTaggingFilterTest() override { +// if (filter_) { +// filter_->onDestroy(); +// } +// } - Event::MockDispatcher dispatcher_; - NiceMock stats_; - IpTaggingFilterConfigSharedPtr config_; - std::unique_ptr filter_; - NiceMock filter_callbacks_; - Buffer::OwnedImpl data_; - NiceMock runtime_; - Api::ApiPtr api_; - NiceMock validation_visitor_; -}; +// Event::MockDispatcher dispatcher_; +// NiceMock stats_; +// IpTaggingFilterConfigSharedPtr config_; +// std::unique_ptr filter_; +// NiceMock filter_callbacks_; +// Buffer::OwnedImpl data_; +// NiceMock runtime_; +// Api::ApiPtr api_; +// NiceMock validation_visitor_; +// }; -// TODO nezdolik split config tests into separate test file -TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { - const std::string config_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: SANITIZE -)EOF"; - EXPECT_THROW_WITH_MESSAGE( - initializeFilter(config_yaml), Envoy::EnvoyException, - "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); -} +// // TODO nezdolik split config tests into separate test file +// TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE( +// initializeFilter(config_yaml), Envoy::EnvoyException, +// "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); +// } -TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { - const std::string config_yaml = R"EOF( -request_type: internal -ip_tag_header: - header: x-envoy-optional-header - action: SANITIZE -ip_tags: - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} -ip_tags_path: /test/tags.yaml -)EOF"; - EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, - "Only one of ip_tags or ip_tags_path can be configured."); -} +// TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tag_header: +// header: x-envoy-optional-header +// action: SANITIZE +// ip_tags: +// - ip_tag_name: internal_request +// ip_list: +// - {address_prefix: 1.2.3.5, prefix_len: 32} +// ip_tags_path: /test/tags.yaml +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, +// "Only one of ip_tags or ip_tags_path can be configured."); +// } -TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { - const std::string config_yaml = R"EOF( -request_type: internal -ip_tags_path: /test/tags.csv -)EOF"; - EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, - "Unsupported file format, unable to parse ip tags from file."); -} +// TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { +// const std::string config_yaml = R"EOF( +// request_type: internal +// ip_tags_path: /test/tags.csv +// )EOF"; +// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, +// "Unsupported file format, unable to parse ip tags from file."); +// } -TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { - EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { - Filesystem::MockWatcher* mock_watcher = new NiceMock(); - EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly(Return(absl::OkStatus())); - return mock_watcher; - })); - envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; - TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), - proto_config1); - auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, dispatcher_, - validation_visitor_); - const std::string config2_string = R"EOF( - request_type: internal - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" - )EOF"; - envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; - TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); - auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", - *stats_.rootScope(), runtime_, *api_, dispatcher_, - validation_visitor_); - auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); - auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); - EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); - auto provider1 = - ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), - IpTaggingFilterConfigPeer::ipTagsLoader(*config1), - *api_, dispatcher_, ip_tags_registry1); - auto provider2 = - ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), - IpTaggingFilterConfigPeer::ipTagsLoader(*config2), - *api_, dispatcher_, ip_tags_registry1); - EXPECT_NE(nullptr, provider1); - EXPECT_NE(nullptr, provider2); - EXPECT_EQ(provider1->ipTags(), provider2->ipTags()); -} +// TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; +// TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), +// proto_config1); +// auto config1 = std::make_shared(proto_config1, ip_tags_registry, +// "prefix.", +// *stats_.rootScope(), runtime_, *api_, +// dispatcher_, validation_visitor_); +// const std::string config2_string = R"EOF( +// request_type: internal +// ip_tags_path: "{{ test_rundir +// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; +// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; +// TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); +// auto config2 = std::make_shared(proto_config2, ip_tags_registry, +// "prefix.", +// *stats_.rootScope(), runtime_, *api_, +// dispatcher_, validation_visitor_); +// auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); +// auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); +// EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); +// auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); +// auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); +// EXPECT_NE(nullptr, provider1); +// EXPECT_NE(nullptr, provider2); +// EXPECT_EQ(provider1->ipTags(), provider2->ipTags()); +// } // TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { // envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; // TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), // proto_config1); -// auto config1 = std::make_shared(proto_config1, ip_tags_registry, "prefix.", -// *stats_.rootScope(), runtime_, *api_, dispatcher_, -// validation_visitor_); +// auto config1 = std::make_shared(proto_config1, ip_tags_registry, +// "prefix.", +// *stats_.rootScope(), runtime_, *api_, +// dispatcher_, validation_visitor_); // const std::string config2_string = R"EOF( // request_type: external -// ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" -// )EOF"; +// ip_tags_path: "{{ test_rundir +// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" )EOF"; // envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; // TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); -// auto config2 = std::make_shared(proto_config2, ip_tags_registry, "prefix.", -// *stats_.rootScope(), runtime_, *api_, dispatcher_, -// validation_visitor_); +// auto config2 = std::make_shared(proto_config2, ip_tags_registry, +// "prefix.", +// *stats_.rootScope(), runtime_, *api_, +// dispatcher_, validation_visitor_); // auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); // auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); // EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); -// auto provider1 = -// ip_tags_registry1->get(IpTaggingFilterConfigPeer::ipTagsPath(*config1), -// IpTaggingFilterConfigPeer::ipTagsLoader(*config1), -// *api_, dispatcher_, ip_tags_registry1); -// auto provider2 = -// ip_tags_registry2->get(IpTaggingFilterConfigPeer::ipTagsPath(*config2), -// IpTaggingFilterConfigPeer::ipTagsLoader(*config2), -// *api_, dispatcher_, ip_tags_registry1); +// auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); +// auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); // EXPECT_NE(nullptr, provider1); // EXPECT_NE(nullptr, provider2); // EXPECT_NE(provider1->ipTags(), provider2->ipTags()); +// ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); // } // class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; @@ -600,8 +585,10 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { // const std::string config = GetParam(); // initializeFilter(config); // Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; // header will -// // be removed +// {"x-envoy-optional-header", "foo"}}; // header +// will +// // be +// removed // Network::Address::InstanceConstSharedPtr remote_address = // Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); // filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( @@ -649,7 +636,8 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { // internal_request_with_append_or_add_with_json_file_config, // internal_request_with_append_or_add_with_yaml_file_config})); -// class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest { +// class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest +// { // }; // TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, @@ -698,10 +686,11 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { // EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); // // There is no guarantee for the order tags are returned by the LC-Trie. -// const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); -// EXPECT_NE(std::string::npos, header_tag_data.find("test")); -// EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); -// EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); +// const std::string header_tag_data = +// request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); EXPECT_NE(std::string::npos, +// header_tag_data.find("test")); EXPECT_NE(std::string::npos, +// header_tag_data.find("internal_request")); EXPECT_NE(std::string::npos, +// header_tag_data.find("duplicate_request")); // EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); // Http::TestRequestTrailerMapImpl request_trailers; @@ -786,136 +775,179 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { // internal_request_with_json_file_config, // internal_request_with_yaml_file_config})); -// struct IpTagsFileReloadTestCase { - -// IpTagsFileReloadTestCase() = default; -// IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, -// const std::string& remote_address, -// const std::string& source_ip_tags_file_path, -// const std::string& reloaded_ip_tags_file_path, -// const std::string& hit_counter_prefix, -// const Http::TestRequestHeaderMapImpl tagged_headers, -// const Http::TestRequestHeaderMapImpl not_tagged_headers) -// : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), -// source_ip_tags_file_path_(source_ip_tags_file_path), -// reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), -// hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), -// not_tagged_headers_(not_tagged_headers) {} -// IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; - -// std::string yaml_config_; -// FilterRequestType request_type_; -// std::string remote_address_; -// std::string source_ip_tags_file_path_; -// std::string reloaded_ip_tags_file_path_; -// std::string hit_counter_prefix_; -// Http::TestRequestHeaderMapImpl tagged_headers_; -// Http::TestRequestHeaderMapImpl not_tagged_headers_; -// }; +struct IpTagsFileReloadTestCase { + + IpTagsFileReloadTestCase() = default; + IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, + const std::string& remote_address, + const std::string& source_ip_tags_file_path, + const std::string& reloaded_ip_tags_file_path, + const std::string& hit_counter_prefix, + const std::vector& hit_counter_updated_prefixes, + const Http::TestRequestHeaderMapImpl tagged_headers, + const Http::TestRequestHeaderMapImpl not_tagged_headers) + : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), + source_ip_tags_file_path_(source_ip_tags_file_path), + reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), + hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), + not_tagged_headers_(not_tagged_headers) { + hit_counter_updated_prefixes_.reserve(hit_counter_updated_prefixes.size()); + for (auto prefix : hit_counter_updated_prefixes) { + hit_counter_updated_prefixes_.push_back(prefix); + } + } + IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; + + std::string yaml_config_; + FilterRequestType request_type_; + std::string remote_address_; + std::string source_ip_tags_file_path_; + std::string reloaded_ip_tags_file_path_; + std::string hit_counter_prefix_; + std::vector hit_counter_updated_prefixes_; + Http::TestRequestHeaderMapImpl tagged_headers_; + Http::TestRequestHeaderMapImpl not_tagged_headers_; +}; -// class IpTagsFileReloadImplTest : public ::testing::TestWithParam { -// public: -// IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { -// ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) -// .WillByDefault(Return(true)); -// } +class IpTagsFileReloadImplTest : public ::testing::TestWithParam { +public: + IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { + ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillByDefault(Return(true)); + } -// static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } - -// void initializeFilter(const std::string& yaml, -// absl::optional& conditional) { -// EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) -// .WillRepeatedly(Invoke([this, &conditional] { -// Filesystem::MockWatcher* mock_watcher = new NiceMock(); -// EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) -// .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, -// Filesystem::Watcher::OnChangedCb cb) { -// { -// absl::WriterMutexLock lock(&mutex_); -// on_changed_cbs_.reserve(1); -// on_changed_cbs_.emplace_back(std::move(cb)); -// } -// if (conditional.has_value()) { -// conditional->setReady(); -// } -// return absl::OkStatus(); -// })); -// return mock_watcher; -// })); -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; -// TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); -// config_ = std::make_shared(config, ip_tags_registry, "prefix.", -// *stats_.rootScope(), runtime_, *api_, dispatcher_, -// validation_visitor_); -// filter_ = std::make_unique(config_); -// filter_->setDecoderFilterCallbacks(filter_callbacks_); -// } + static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); } -// ~IpTagsFileReloadImplTest() override { -// if (filter_) { -// filter_->onDestroy(); -// } -// } + void initializeFilter(const std::string& yaml, + absl::optional& conditional) { + EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) + .WillRepeatedly(Invoke([this, &conditional] { + Filesystem::MockWatcher* mock_watcher = new NiceMock(); + EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) + .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, + Filesystem::Watcher::OnChangedCb cb) { + { + absl::WriterMutexLock lock(&mutex_); + on_changed_cbs_.reserve(1); + on_changed_cbs_.emplace_back(std::move(cb)); + } + if (conditional.has_value()) { + conditional->setReady(); + } + return absl::OkStatus(); + })); + return mock_watcher; + })); + envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + config_ = std::make_shared(config, ip_tags_registry, "prefix.", + *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + filter_ = std::make_unique(config_); + filter_->setDecoderFilterCallbacks(filter_callbacks_); + } -// Event::MockDispatcher dispatcher_; -// NiceMock stats_; -// IpTaggingFilterConfigSharedPtr config_; -// std::unique_ptr filter_; -// NiceMock filter_callbacks_; -// Buffer::OwnedImpl data_; -// NiceMock runtime_; -// Api::ApiPtr api_; -// NiceMock validation_visitor_; -// absl::Mutex mutex_; -// std::vector on_changed_cbs_ ABSL_GUARDED_BY(mutex_); -// absl::optional cb_added_nullopt = absl::nullopt; -// }; + ~IpTagsFileReloadImplTest() override { + { + absl::WriterMutexLock lock(&mutex_); + on_changed_cbs_.clear(); + } + if (filter_) { + filter_->onDestroy(); + } + } -// TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { -// IpTagsFileReloadTestCase test_case = GetParam(); -// initializeFilter(test_case.yaml_config_, cb_added_nullopt); -// EXPECT_EQ(test_case.request_type_, config_->requestType()); -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); -// EXPECT_CALL(stats_, -// counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); -// EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, -// filter_->decodeHeaders(test_case.tagged_headers_, false)); -// EXPECT_EQ(test_case.hit_counter_prefix_, -// test_case.tagged_headers_.get_(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, -// filter_->decodeHeaders(test_case.not_tagged_headers_, false)); -// EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); -// std::string source_ip_tags_file_path = TestEnvironment::substitute(test_case.source_ip_tags_file_path_); -// std::string reloaded_ip_tags_file_path = TestEnvironment::substitute(test_case.reloaded_ip_tags_file_path_); -// std::cerr << "source_ip_tags_file_path: " << source_ip_tags_file_path << std::endl; -// std::cerr << "reloaded_ip_tags_file_path: " << reloaded_ip_tags_file_path << std::endl; -// TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); -// TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); -// // Clean up modifications to ip tags file names. -// TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); -// TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); -// } + Event::MockDispatcher dispatcher_; + NiceMock stats_; + IpTaggingFilterConfigSharedPtr config_; + std::unique_ptr filter_; + NiceMock filter_callbacks_; + Buffer::OwnedImpl data_; + NiceMock runtime_; + Api::ApiPtr api_; + NiceMock validation_visitor_; + absl::Mutex mutex_; + std::vector on_changed_cbs_ ABSL_GUARDED_BY(mutex_); + absl::optional cb_added_nullopt = absl::nullopt; +}; -// struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { -// {internal_request_with_yaml_file_config, -// FilterRequestType::INTERNAL, -// "1.2.3.5", -// internal_request_file_path, -// updated_internal_request_file_path, -// "internal_request", -// {{"x-envoy-internal", "true"}}, -// {}}, -// }; +TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { + IpTagsFileReloadTestCase test_case = GetParam(); + auto cb_added_opt = absl::make_optional(); + initializeFilter(test_case.yaml_config_, cb_added_opt); + EXPECT_EQ(test_case.request_type_, config_->requestType()); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + auto request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(test_case.not_tagged_headers_, false)); + EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); + std::string source_ip_tags_file_path = + TestEnvironment::substitute(test_case.source_ip_tags_file_path_); + std::string reloaded_ip_tags_file_path = + TestEnvironment::substitute(test_case.reloaded_ip_tags_file_path_); + TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); + TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_success"))); + cb_added_opt.value().waitReady(); + { + absl::ReaderMutexLock guard(&mutex_); + EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); + } + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + for (auto updated_prefix : test_case.hit_counter_updated_prefixes_) { + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, updated_prefix, ".hit"))); + } + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + for (auto updated_prefix : test_case.hit_counter_updated_prefixes_) { + EXPECT_TRUE(request_headers.get_(Http::Headers::get().EnvoyIpTags).find(updated_prefix) != + std::string::npos); + } + // Clean up modifications to ip tags file names. + TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); + TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); +} + +struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { + {internal_request_with_yaml_file_config, + FilterRequestType::INTERNAL, + "1.2.3.5", + "{{ test_rundir " + "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml", + "{{ test_rundir " + "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml", + "internal_request", + {"internal_updated_request"}, + {{"x-envoy-internal", "true"}}, + {}}, + {external_request_with_json_file_config, + FilterRequestType::EXTERNAL, + "1.2.3.4", + "{{ test_rundir " + "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json", + "{{ test_rundir " + "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json", + "external_request", + {"external_updated_request", "gcp_zone_a_request"}, + {}, + {{"x-envoy-internal", "true"}}}, +}; -// INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, -// ::testing::ValuesIn(ip_tags_file_reload_test_cases)); +INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, + ::testing::ValuesIn(ip_tags_file_reload_test_cases)); } // namespace } // namespace IpTagging diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json new file mode 100644 index 0000000000000..cbe852f4fc078 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json @@ -0,0 +1,22 @@ +{ +"ip_tags": [ + { + "ip_tag_name": "external_updated_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + }, + { + "ip_tag_name": "gcp_zone_a_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + } +] +} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml index 10c43f7144c29..9cc955bf6dd67 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: internal_request + - ip_tag_name: internal_updated_request ip_list: - - {address_prefix: 1.2.3.7, prefix_len: 32} + - {address_prefix: 1.2.3.5, prefix_len: 32} From 76ce0ea2654cced0c85bf98b3b71cbf01476ad08 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 23 Apr 2025 16:58:59 +0200 Subject: [PATCH 12/76] Test reload error Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 31 +++++++----- .../http/ip_tagging/ip_tagging_filter_test.cc | 48 +++++++++++++++++++ .../ip_tagging/test_data/invalid_tags.yaml | 1 + 3 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 03b753eee54ac..de8d543b76c43 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -68,7 +68,6 @@ void IpTagsProvider::updateIpTags(const LcTrieSharedPtr reloaded_tags) ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { absl::MutexLock lock(&ip_tags_mutex_); tags_ = reloaded_tags; - std::cerr << "***Finishes reloading ip tags" << std::endl; } IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, @@ -82,14 +81,27 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path) { throw EnvoyException("Unsupported file format, unable to parse ip tags from file."); } auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); - THROW_IF_NOT_OK_REF(file_or_error.status()); - IpTagFileProto ip_tags_proto; - if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { - MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); - } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { - MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); + if (file_or_error.status().ok()) { + IpTagFileProto ip_tags_proto; + if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { + try { + MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(warn, "failed to parse ip tags file: {}", e.what()); + return nullptr; + } + } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { + try { + MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(warn, "failed to parse ip tags file: {}", e.what()); + return nullptr; + } + } + return parseIpTags(ip_tags_proto.ip_tags()); + } else { + return nullptr; } - return parseIpTags(ip_tags_proto.ip_tags()); } return nullptr; } @@ -112,7 +124,6 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( entry.address_prefix(), entry.prefix_len().value())); } } - std::cerr << "***PArsing ip tag: " << ip_tag.ip_tag_name() << std::endl; tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } @@ -153,7 +164,6 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { - std::cerr << "***Resolving provider for: " << ip_tags_path_ << std::endl; provider_ = ip_tags_registry_->get( ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_); @@ -179,7 +189,6 @@ IpTaggingFilter::~IpTaggingFilter() = default; void IpTaggingFilter::onDestroy() {} Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { - std::cerr << "***Decode headers called: " << std::endl; const bool is_internal_request = headers.EnvoyInternalRequest() && (headers.EnvoyInternalRequest()->value() == Http::Headers::get().EnvoyInternalRequestValues.True.c_str()); diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 26dcf86580e50..33d552f9cafb0 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -921,6 +921,54 @@ TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); } +TEST_P(IpTagsFileReloadImplTest, IpTagsFileNotReloaded) { + IpTagsFileReloadTestCase test_case = GetParam(); + auto cb_added_opt = absl::make_optional(); + initializeFilter(test_case.yaml_config_, cb_added_opt); + EXPECT_EQ(test_case.request_type_, config_->requestType()); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + auto request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(test_case.not_tagged_headers_, false)); + EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); + std::string source_ip_tags_file_path = + TestEnvironment::substitute(test_case.source_ip_tags_file_path_); + std::string reloaded_ip_tags_file_path = TestEnvironment::substitute( + "{{ test_rundir " + "}}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml"); + TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); + TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); + cb_added_opt.value().waitReady(); + { + absl::ReaderMutexLock guard(&mutex_); + EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); + } + // Current ip tags should be used if reload failed. + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + // Clean up modifications to ip tags file names. + TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); + TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); +} + struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { {internal_request_with_yaml_file_config, FilterRequestType::INTERNAL, diff --git a/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml new file mode 100644 index 0000000000000..e3dd56b59bbfe --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml @@ -0,0 +1 @@ +ip_tags \ No newline at end of file From 6974702ede4ef8ba587809f48c574017d95ff259 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 29 Apr 2025 10:11:14 +0200 Subject: [PATCH 13/76] Fix singleton registraton issue Signed-off-by: Kateryna Nezdolii --- envoy/registry/registry.h | 2 +- .../extensions/filters/http/ip_tagging/config.cc | 7 +------ .../filters/http/ip_tagging/ip_tagging_filter.cc | 15 ++++++++++++--- .../filters/http/ip_tagging/ip_tagging_filter.h | 13 ++++--------- .../http/ip_tagging/ip_tagging_filter_test.cc | 6 ++++-- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/envoy/registry/registry.h b/envoy/registry/registry.h index 18480cf6583cc..b13aef7c1c56c 100644 --- a/envoy/registry/registry.h +++ b/envoy/registry/registry.h @@ -220,7 +220,7 @@ template class FactoryRegistry : public Logger::Loggable ip_tags_registry = - context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), - [] { return std::make_shared(); }); - IpTaggingFilterConfigSharedPtr config(new IpTaggingFilterConfig( - proto_config, ip_tags_registry, stat_prefix, context.scope(), + proto_config, stat_prefix, context.serverFactoryContext().singletonManager(), context.scope(), context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), context.serverFactoryContext().mainThreadDispatcher(), context.messageValidationVisitor())); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index de8d543b76c43..016f1ee5a517d 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -130,10 +130,12 @@ LcTrieSharedPtr IpTagsLoader::parseIpTags( return std::make_shared>(tag_data); } +SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); + IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - std::shared_ptr ip_tags_registry, const std::string& stat_prefix, - Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, + const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, + Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor) : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), stat_name_set_(scope.symbolTable().makeSet("IpTagging")), @@ -144,7 +146,10 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( ip_tag_header_action_(config.has_ip_tag_header() ? config.ip_tag_header().action() : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE), - ip_tags_path_(config.ip_tags_path()), ip_tags_registry_(ip_tags_registry), + ip_tags_path_(config.ip_tags_path()), + ip_tags_registry_(singleton_manager.getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), + [] { return std::make_shared(); })), tags_loader_(api, validation_visitor, stat_name_set_) { // Once loading IP tags from a file system is supported, the restriction on the size @@ -164,6 +169,10 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { + // ip_tags_registry_ = + // std::make_shared(singleton_manager.getTyped( + // SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), + // [] { return std::make_shared(); })); provider_ = ip_tags_registry_->get( ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index cda2399424e46..791106be174fa 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -83,7 +83,7 @@ using IpTagsProviderSharedPtr = std::shared_ptr; */ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: - IpTagsRegistrySingleton() { std::cerr << "****Create " << std::endl; } + IpTagsRegistrySingleton() {} std::shared_ptr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, IpTagsReloadSuccessCb reload_success_cb, @@ -94,10 +94,8 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { const uint64_t key = std::hash()(ip_tags_path); absl::MutexLock lock(&mu_); auto it = ip_tags_registry_.find(key); - std::cerr << "****Found provider " << std::endl; if (it != ip_tags_registry_.end()) { if (std::shared_ptr provider = it->second.lock()) { - std::cerr << "****Returning existing provider " << std::endl; ip_tags_provider = provider; } else { ip_tags_provider = @@ -106,7 +104,6 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { ip_tags_registry_[key] = ip_tags_provider; } } else { - std::cerr << "****Creating new provider " << std::endl; ip_tags_provider = std::make_shared(ip_tags_path, tags_loader, reload_success_cb, reload_error_cb, dispatcher, api, singleton); @@ -123,8 +120,6 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); }; -SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); - /** * Type of requests the filter should apply to. */ @@ -139,9 +134,9 @@ class IpTaggingFilterConfig { envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IpTagHeader::HeaderAction; IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - std::shared_ptr ip_tags_registry, - const std::string& stat_prefix, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, + const std::string& stat_prefix, Singleton::Manager& singleton_manager, + Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, + Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor); Runtime::Loader& runtime() { return runtime_; } diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 33d552f9cafb0..94c7a51b3665e 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -816,7 +816,7 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam(); } + static void SetUpTestSuite() {} void initializeFilter(const std::string& yaml, absl::optional& conditional) { @@ -840,7 +840,7 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam(config, ip_tags_registry, "prefix.", + config_ = std::make_shared(config, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, dispatcher_, validation_visitor_); filter_ = std::make_unique(config_); @@ -857,6 +857,8 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam singleton_manager_ = + std::make_unique(); Event::MockDispatcher dispatcher_; NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; From b5cd0d18c068f034d7e768fffe5783376882b8c8 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 29 Apr 2025 11:03:44 +0200 Subject: [PATCH 14/76] Add integration test Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/v3/ip_tagging.proto | 4 +- test/extensions/filters/http/ip_tagging/BUILD | 3 + .../http/ip_tagging/ip_tagging_filter_test.cc | 1090 ++++++++--------- .../ip_tagging/ip_tagging_integration_test.cc | 28 +- 4 files changed, 560 insertions(+), 565 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index ad848e8c402fe..8bfe3925a4e18 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -79,7 +79,7 @@ message IPTagging { // Only one of :ref:`ip_tags ` // or :ref:`ip_tags_path ` // can be set for the Ip Tagging filter. - repeated data.ip_tagging.v3.IPTag ip_tags = 4 [(validate.rules).repeated = {min_items: 1}]; + repeated data.ip_tagging.v3.IPTag ip_tags = 4; // Specify to which header the tags will be written. // @@ -91,5 +91,5 @@ message IPTagging { // Only one of :ref:`ip_tags ` // or :ref:`ip_tags_path ` // can be set for the Ip Tagging filter. - string ip_tags_path = 6 [(validate.rules).string = {min_len: 1}]; + string ip_tags_path = 6; } diff --git a/test/extensions/filters/http/ip_tagging/BUILD b/test/extensions/filters/http/ip_tagging/BUILD index e7188b97072a3..50f2e40e5e6de 100644 --- a/test/extensions/filters/http/ip_tagging/BUILD +++ b/test/extensions/filters/http/ip_tagging/BUILD @@ -41,6 +41,9 @@ envoy_extension_cc_test( name = "ip_tagging_integration_test", size = "large", srcs = ["ip_tagging_integration_test.cc"], + data = [ + "//test/extensions/filters/http/ip_tagging/test_data:ip_tagging_files", + ], extension_names = ["envoy.filters.http.ip_tagging"], rbe_pool = "6gig", deps = [ diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 94c7a51b3665e..3e8a75fbc5bfd 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -46,10 +46,6 @@ class IpTaggingFilterConfigPeer { namespace { -std::shared_ptr ip_tags_registry; - -namespace { - const std::string ip_tagging_prefix = "prefix.ip_tagging."; const std::string internal_request_config = R"EOF( @@ -225,555 +221,542 @@ ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_da } // namespace -// class IpTaggingFilterTest : public ::testing::TestWithParam { -// public: -// IpTaggingFilterTest() : api_(Api::createApiForTest()) { -// ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) -// .WillByDefault(Return(true)); -// EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { -// Filesystem::MockWatcher* mock_watcher = new NiceMock(); -// EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) -// .WillRepeatedly(Return(absl::OkStatus())); -// return mock_watcher; -// })); -// } - -// static void SetUpTestSuite() { ip_tags_registry = std::make_shared(); -// } - -// void initializeFilter(const std::string& yaml) { -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; -// TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); -// config_ = std::make_shared(config, ip_tags_registry, "prefix.", -// *stats_.rootScope(), runtime_, *api_, -// dispatcher_, validation_visitor_); -// filter_ = std::make_unique(config_); -// filter_->setDecoderFilterCallbacks(filter_callbacks_); -// } - -// ~IpTaggingFilterTest() override { -// if (filter_) { -// filter_->onDestroy(); -// } -// } - -// Event::MockDispatcher dispatcher_; -// NiceMock stats_; -// IpTaggingFilterConfigSharedPtr config_; -// std::unique_ptr filter_; -// NiceMock filter_callbacks_; -// Buffer::OwnedImpl data_; -// NiceMock runtime_; -// Api::ApiPtr api_; -// NiceMock validation_visitor_; -// }; - -// // TODO nezdolik split config tests into separate test file -// TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE( -// initializeFilter(config_yaml), Envoy::EnvoyException, -// "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); -// } - -// TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tag_header: -// header: x-envoy-optional-header -// action: SANITIZE -// ip_tags: -// - ip_tag_name: internal_request -// ip_list: -// - {address_prefix: 1.2.3.5, prefix_len: 32} -// ip_tags_path: /test/tags.yaml -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, -// "Only one of ip_tags or ip_tags_path can be configured."); -// } - -// TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { -// const std::string config_yaml = R"EOF( -// request_type: internal -// ip_tags_path: /test/tags.csv -// )EOF"; -// EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, -// "Unsupported file format, unable to parse ip tags from file."); -// } - -// TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; -// TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), -// proto_config1); -// auto config1 = std::make_shared(proto_config1, ip_tags_registry, -// "prefix.", -// *stats_.rootScope(), runtime_, *api_, -// dispatcher_, validation_visitor_); -// const std::string config2_string = R"EOF( -// request_type: internal -// ip_tags_path: "{{ test_rundir -// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; -// TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); -// auto config2 = std::make_shared(proto_config2, ip_tags_registry, -// "prefix.", -// *stats_.rootScope(), runtime_, *api_, -// dispatcher_, validation_visitor_); -// auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); -// auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); -// EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); -// auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); -// auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); -// EXPECT_NE(nullptr, provider1); -// EXPECT_NE(nullptr, provider2); -// EXPECT_EQ(provider1->ipTags(), provider2->ipTags()); -// } - -// TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; -// TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), -// proto_config1); -// auto config1 = std::make_shared(proto_config1, ip_tags_registry, -// "prefix.", -// *stats_.rootScope(), runtime_, *api_, -// dispatcher_, validation_visitor_); -// const std::string config2_string = R"EOF( -// request_type: external -// ip_tags_path: "{{ test_rundir -// }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" )EOF"; -// envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; -// TestUtility::loadFromYaml(TestEnvironment::substitute(config2_string), proto_config2); -// auto config2 = std::make_shared(proto_config2, ip_tags_registry, -// "prefix.", -// *stats_.rootScope(), runtime_, *api_, -// dispatcher_, validation_visitor_); -// auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); -// auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); -// EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); -// auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); -// auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); -// EXPECT_NE(nullptr, provider1); -// EXPECT_NE(nullptr, provider2); -// EXPECT_NE(provider1->ipTags(), provider2->ipTags()); -// ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); -// } - -// class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { -// const std::string config = GetParam(); -// initializeFilter(config); -// EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - -// // Check external requests don't get a tag. -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } - -// INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); - -// class ExternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { -// const std::string config = GetParam(); -// initializeFilter(config); -// EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers; - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - -// // Check internal requests don't get a tag. -// request_headers = {{"x-envoy-internal", "true"}}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } - -// INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, -// ::testing::ValuesIn({external_request_config, -// external_request_with_json_file_config, -// external_request_with_yaml_file_config})); - -// class BothRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { -// const std::string config = GetParam(); -// initializeFilter(config); -// EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// request_headers = Http::TestRequestHeaderMapImpl{}; -// remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// } - -// INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, -// ::testing::ValuesIn({both_request_config, -// both_request_with_json_file_config, -// both_request_with_yaml_file_config})); - -// class NoHitsIpTaggingFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(NoHitsIpTaggingFilterTest, NoHits) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); - -// class AppendEntryFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(AppendEntryFilterTest, AppendEntry) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-ip-tags", "test"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); - -// class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, -// ReplaceAlternateHeaderWhenActionIsDefaulted) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{ -// {"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}, // foo will be removed -// {"x-envoy-optional-header", "bar"}, // bar will be removed -// {"x-envoy-optional-header", "baz"}, // baz will be removed -// }; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, -// ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, -// ::testing::ValuesIn({internal_request_with_header_config, -// internal_request_with_header_with_json_file_config, -// internal_request_with_header_with_yaml_file_config})); - -// class ReplaceAlternateHeaderFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{ -// {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P( -// ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, -// ::testing::ValuesIn({internal_request_with_replace_header_config, -// internal_request_with_replace_header_with_json_file_config, -// internal_request_with_replace_header_with_yaml_file_config})); - -// class ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, -// ClearAlternateHeaderWhenUnmatchedAndSanitized) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; // header -// will -// // be -// removed -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P( -// ClearAlternateHeaderWhenUnmatchedAndSanitized, -// ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, -// ::testing::ValuesIn({internal_request_with_replace_header_config, -// internal_request_with_replace_header_with_json_file_config, -// internal_request_with_replace_header_with_yaml_file_config})); - -// class AppendForwardAlternateHeaderFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(AppendForwardAlternateHeaderFilterTest, AppendForwardAlternateHeader) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("foo,internal_request_with_optional_header", -// request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P( -// AppendForwardAlternateHeader, AppendForwardAlternateHeaderFilterTest, -// ::testing::ValuesIn({internal_request_with_append_or_add_config, -// internal_request_with_append_or_add_with_json_file_config, -// internal_request_with_append_or_add_with_yaml_file_config})); - -// class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest -// { -// }; - -// TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, -// RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-optional-header", "foo"}}; -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P( -// RetainAlternateHeaderWhenUnmatchedAndAppendForwarded, -// RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, -// ::testing::ValuesIn({internal_request_with_append_or_add_config, -// internal_request_with_append_or_add_with_json_file_config, -// internal_request_with_append_or_add_with_yaml_file_config})); - -// class NestedPrefixesFilterTest : public IpTaggingFilterTest {}; - -// TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, -// {"x-envoy-ip-tags", "test"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); -// EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - -// // There is no guarantee for the order tags are returned by the LC-Trie. -// const std::string header_tag_data = -// request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); EXPECT_NE(std::string::npos, -// header_tag_data.find("test")); EXPECT_NE(std::string::npos, -// header_tag_data.find("internal_request")); EXPECT_NE(std::string::npos, -// header_tag_data.find("duplicate_request")); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(NestedPrefixes, NestedPrefixesFilterTest, -// ::testing::ValuesIn({duplicate_request_config, -// duplicate_request_with_json_file_config, -// duplicate_request_with_yaml_file_config})); - -// class Ipv6AddressTest : public IpTaggingFilterTest {}; - -// TEST_P(Ipv6AddressTest, Ipv6Address) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, -// ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, -// ipv6_with_yaml_file_config})); - -// class RuntimeDisabledTest : public IpTaggingFilterTest {}; - -// TEST_P(RuntimeDisabledTest, RuntimeDisabled) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) -// .WillOnce(Return(false)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); - -// class ClearRouteCacheTest : public IpTaggingFilterTest {}; - -// TEST_P(ClearRouteCacheTest, ClearRouteCache) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); - -// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); - -// // no tags, no call -// EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); -// request_headers = Http::TestRequestHeaderMapImpl{}; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// } - -// INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); +class IpTaggingFilterTest : public ::testing::TestWithParam { +public: + IpTaggingFilterTest() : api_(Api::createApiForTest()) { + ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillByDefault(Return(true)); + EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { + Filesystem::MockWatcher* mock_watcher = new NiceMock(); + EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) + .WillRepeatedly(Return(absl::OkStatus())); + return mock_watcher; + })); + } + + void initializeFilter(const std::string& yaml) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); + config_ = std::make_shared(config, "prefix.", *singleton_manager_, + *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + filter_ = std::make_unique(config_); + filter_->setDecoderFilterCallbacks(filter_callbacks_); + } + + ~IpTaggingFilterTest() override { + if (filter_) { + filter_->onDestroy(); + } + } + + std::unique_ptr singleton_manager_ = + std::make_unique(); + Event::MockDispatcher dispatcher_; + NiceMock stats_; + IpTaggingFilterConfigSharedPtr config_; + std::unique_ptr filter_; + NiceMock filter_callbacks_; + Buffer::OwnedImpl data_; + NiceMock runtime_; + Api::ApiPtr api_; + NiceMock validation_visitor_; +}; + +// TODO nezdolik split config tests into separate test file +TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +)EOF"; + EXPECT_THROW_WITH_MESSAGE( + initializeFilter(config_yaml), Envoy::EnvoyException, + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); +} + +TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tag_header: + header: x-envoy-optional-header + action: SANITIZE +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} +ip_tags_path: /test/tags.yaml +)EOF"; + EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, + "Only one of ip_tags or ip_tags_path can be configured."); +} + +TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_path: /test/tags.csv +)EOF"; + EXPECT_THROW_WITH_MESSAGE(initializeFilter(config_yaml), Envoy::EnvoyException, + "Unsupported file format, unable to parse ip tags from file."); +} + +TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; + TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), + proto_config1); + auto config1 = std::make_shared( + proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; + TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), + proto_config2); + auto config2 = std::make_shared( + proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); + auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); + EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); + auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); + auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); + EXPECT_NE(nullptr, provider1); + EXPECT_NE(nullptr, provider2); + EXPECT_EQ(provider1->ipTags(), provider2->ipTags()); +} + +TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) { + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; + TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), + proto_config1); + auto config1 = std::make_shared( + proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; + TestUtility::loadFromYaml(TestEnvironment::substitute(external_request_with_json_file_config), + proto_config2); + auto config2 = std::make_shared( + proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, + dispatcher_, validation_visitor_); + auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); + auto ip_tags_registry2 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config2); + EXPECT_EQ(ip_tags_registry1.get(), ip_tags_registry2.get()); + auto provider1 = IpTaggingFilterConfigPeer::ipTagsProvider(*config1); + auto provider2 = IpTaggingFilterConfigPeer::ipTagsProvider(*config2); + EXPECT_NE(nullptr, provider1); + EXPECT_NE(nullptr, provider2); + EXPECT_NE(provider1->ipTags(), provider2->ipTags()); + ::testing::Mock::VerifyAndClearExpectations(&filter_callbacks_); +} + +class InternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { + const std::string config = GetParam(); + initializeFilter(config); + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + // Check external requests don't get a tag. + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +} + +INSTANTIATE_TEST_CASE_P(InternalRequest, InternalRequestIpTaggingFilterTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class ExternalRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { + const std::string config = GetParam(); + initializeFilter(config); + EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers; + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + // Check internal requests don't get a tag. + request_headers = {{"x-envoy-internal", "true"}}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +} + +INSTANTIATE_TEST_CASE_P(ExternalRequest, ExternalRequestIpTaggingFilterTest, + ::testing::ValuesIn({external_request_config, + external_request_with_json_file_config, + external_request_with_yaml_file_config})); + +class BothRequestIpTaggingFilterTest : public IpTaggingFilterTest {}; + +TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { + const std::string config = GetParam(); + initializeFilter(config); + EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + request_headers = Http::TestRequestHeaderMapImpl{}; + remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); +} + +INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, + ::testing::ValuesIn({both_request_config, + both_request_with_json_file_config, + both_request_with_yaml_file_config})); + +class NoHitsIpTaggingFilterTest : public IpTaggingFilterTest {}; + +TEST_P(NoHitsIpTaggingFilterTest, NoHits) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("10.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(NoHits, NoHitsIpTaggingFilterTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class AppendEntryFilterTest : public IpTaggingFilterTest {}; + +TEST_P(AppendEntryFilterTest, AppendEntry) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-ip-tags", "test"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("test,internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(AppendEntry, AppendEntryFilterTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, + ReplaceAlternateHeaderWhenActionIsDefaulted) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{ + {"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}, // foo will be removed + {"x-envoy-optional-header", "bar"}, // bar will be removed + {"x-envoy-optional-header", "baz"}, // baz will be removed + }; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(ReplaceAlternateHeaderWhenActionIsDefaulted, + ReplaceAlternateHeaderWhenActionIsDefaultedFilterTest, + ::testing::ValuesIn({internal_request_with_header_config, + internal_request_with_header_with_json_file_config, + internal_request_with_header_with_yaml_file_config})); + +class ReplaceAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ReplaceAlternateHeaderFilterTest, ReplaceAlternateHeader) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{ + {"x-envoy-internal", "true"}, {"x-envoy-optional-header", "foo"}}; // foo will be removed + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + ReplaceAlternateHeader, ReplaceAlternateHeaderFilterTest, + ::testing::ValuesIn({internal_request_with_replace_header_config, + internal_request_with_replace_header_with_json_file_config, + internal_request_with_replace_header_with_yaml_file_config})); + +class ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest : public IpTaggingFilterTest {}; + +TEST_P(ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, + ClearAlternateHeaderWhenUnmatchedAndSanitized) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; // header + // will + // be + // removed + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + ClearAlternateHeaderWhenUnmatchedAndSanitized, + ClearAlternateHeaderWhenUnmatchedAndSanitizedFilterTest, + ::testing::ValuesIn({internal_request_with_replace_header_config, + internal_request_with_replace_header_with_json_file_config, + internal_request_with_replace_header_with_yaml_file_config})); + +class AppendForwardAlternateHeaderFilterTest : public IpTaggingFilterTest {}; + +TEST_P(AppendForwardAlternateHeaderFilterTest, AppendForwardAlternateHeader) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("foo,internal_request_with_optional_header", + request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + AppendForwardAlternateHeader, AppendForwardAlternateHeaderFilterTest, + ::testing::ValuesIn({internal_request_with_append_or_add_config, + internal_request_with_append_or_add_with_json_file_config, + internal_request_with_append_or_add_with_yaml_file_config})); + +class RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest : public IpTaggingFilterTest { +}; + +TEST_P(RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, + RetainAlternateHeaderWhenUnmatchedAndAppendForwarded) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-optional-header", "foo"}}; + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("foo", request_headers.get_("x-envoy-optional-header")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P( + RetainAlternateHeaderWhenUnmatchedAndAppendForwarded, + RetainAlternateHeaderWhenUnmatchedAndAppendForwardedFilterTest, + ::testing::ValuesIn({internal_request_with_append_or_add_config, + internal_request_with_append_or_add_with_json_file_config, + internal_request_with_append_or_add_with_yaml_file_config})); + +class NestedPrefixesFilterTest : public IpTaggingFilterTest {}; + +TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}, + {"x-envoy-ip-tags", "test"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + + // There is no guarantee for the order tags are returned by the LC-Trie. + const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); + EXPECT_NE(std::string::npos, header_tag_data.find("test")); + EXPECT_NE(std::string::npos, header_tag_data.find("internal_request")); + EXPECT_NE(std::string::npos, header_tag_data.find("duplicate_request")); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(NestedPrefixes, NestedPrefixesFilterTest, + ::testing::ValuesIn({duplicate_request_config, + duplicate_request_with_json_file_config, + duplicate_request_with_yaml_file_config})); + +class Ipv6AddressTest : public IpTaggingFilterTest {}; + +TEST_P(Ipv6AddressTest, Ipv6Address) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("2001:abcd:ef01:2345::1"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("ipv6_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, + ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, + ipv6_with_yaml_file_config})); + +class RuntimeDisabledTest : public IpTaggingFilterTest {}; + +TEST_P(RuntimeDisabledTest, RuntimeDisabled) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillOnce(Return(false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); + +class ClearRouteCacheTest : public IpTaggingFilterTest {}; + +TEST_P(ClearRouteCacheTest, ClearRouteCache) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + // no tags, no call + EXPECT_CALL(filter_callbacks_.downstream_callbacks_, clearRouteCache()).Times(0); + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); +} + +INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); struct IpTagsFileReloadTestCase { @@ -999,7 +982,6 @@ struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, ::testing::ValuesIn(ip_tags_file_reload_test_cases)); -} // namespace } // namespace IpTagging } // namespace HttpFilters } // namespace Extensions diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index 0aef174daafa0..c2d815b3e368c 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -1,4 +1,5 @@ #include "test/integration/http_integration.h" +#include "test/test_common/environment.h" namespace Envoy { namespace { @@ -13,7 +14,10 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, IpTaggingIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); // Just IP tagging for now. -const char ExampleIpTaggingConfig[] = R"EOF( +const std::string ExampleIpTaggingConfig = R"EOF( + name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging request_type: both ip_tags: - ip_tag_name: external_request @@ -21,16 +25,22 @@ const char ExampleIpTaggingConfig[] = R"EOF( - {address_prefix: 1.2.3.4, prefix_len: 32} )EOF"; +const std::string FileBasedIpTaggingConfig = R"EOF( + name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" +)EOF"; + // Make sure that Envoy starts up with an ip tagging filter. TEST_P(IpTaggingIntegrationTest, IpTaggingV3StaticTypedStructConfig) { - config_helper_.prependFilter(absl::StrCat(R"EOF( -name: ip_tagging -typed_config: - "@type": type.googleapis.com/xds.type.v3.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging - value: - )EOF", - ExampleIpTaggingConfig)); + config_helper_.prependFilter(ExampleIpTaggingConfig); + initialize(); +} + +TEST_P(IpTaggingIntegrationTest, FileBasedIpTagging) { + config_helper_.prependFilter(TestEnvironment::substitute(FileBasedIpTaggingConfig)); initialize(); } From b958d1614b21ba9cb3239a36fd3cd4e03f100342 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 29 Apr 2025 11:06:11 +0200 Subject: [PATCH 15/76] Cleanup Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/ip_tagging_filter.cc | 4 ---- source/extensions/filters/http/ip_tagging/ip_tagging_filter.h | 2 -- 2 files changed, 6 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 016f1ee5a517d..76b20bda1f7eb 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -169,10 +169,6 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTags(config.ip_tags()); } else { - // ip_tags_registry_ = - // std::make_shared(singleton_manager.getTyped( - // SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), - // [] { return std::make_shared(); })); provider_ = ip_tags_registry_->get( ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 791106be174fa..8aa1fa2126fb6 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -25,8 +25,6 @@ namespace IpTagging { using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; using LcTrieSharedPtr = std::shared_ptr>; -// TODO supports stats for ip tags -// Support async reload of tags file class IpTagsLoader { public: IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, From 7e62d10dd81bb82aa50809b7a9d37ace7a84c2bf Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 30 Apr 2025 15:30:04 +0200 Subject: [PATCH 16/76] Format Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/config.cc | 2 +- .../http/ip_tagging/ip_tagging_filter.cc | 64 +++++++++++-------- .../http/ip_tagging/ip_tagging_filter.h | 38 ++++++----- .../http/ip_tagging/ip_tagging_filter_test.cc | 24 +++---- 4 files changed, 70 insertions(+), 58 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/config.cc b/source/extensions/filters/http/ip_tagging/config.cc index 16eb1e14436f7..9fa7ed8481e54 100644 --- a/source/extensions/filters/http/ip_tagging/config.cc +++ b/source/extensions/filters/http/ip_tagging/config.cc @@ -17,7 +17,7 @@ absl::StatusOr IpTaggingFilterFactory::createFilterFactor const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { absl::StatusOr config = IpTaggingFilterConfig::create( - proto_config, stat_prefix, context.serverFactoryContext().singletonManager(), context.scope(), + proto_config, stat_prefix, context.serverFactoryContext().singletonManager(), context.scope(), context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), context.serverFactoryContext().mainThreadDispatcher(), context.messageValidationVisitor()); RETURN_IF_NOT_OK_REF(config.status()); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 79197b9ae771f..b50ed60bfbc55 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -16,9 +16,11 @@ namespace IpTagging { IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Event::Dispatcher& dispatcher, - Api::Api& api, Singleton::InstanceSharedPtr owner, absl::Status& creation_status) + Api::Api& api, Singleton::InstanceSharedPtr owner, + absl::Status& creation_status) : ip_tags_path_(ip_tags_path), tags_loader_(tags_loader), reload_success_cb_(reload_success_cb), - reload_error_cb_(reload_error_cb), tags_(tags_loader_.loadTags(ip_tags_path_, creation_status)), + reload_error_cb_(reload_error_cb), + tags_(tags_loader_.loadTags(ip_tags_path_, creation_status)), ip_tags_reload_dispatcher_(api.allocateDispatcher("ip_tags_reload_routine")), ip_tags_file_watcher_(dispatcher.createFilesystemWatcher()), owner_(owner) { if (ip_tags_path.empty()) { @@ -35,7 +37,6 @@ IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& ta Thread::Options{std::string("ip_tags_reload_routine")}); } - IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); ip_tags_reload_dispatcher_->exit(); @@ -56,7 +57,8 @@ absl::Status IpTagsProvider::onIpTagsFileUpdate() { return ipTagsReload(reloaded_tags, reload_status); } -absl::Status IpTagsProvider::ipTagsReload(const LcTrieSharedPtr reloaded_tags, absl::Status& reload_status) { +absl::Status IpTagsProvider::ipTagsReload(const LcTrieSharedPtr reloaded_tags, + absl::Status& reload_status) { if (reload_status.ok()) { updateIpTags(reloaded_tags); reload_success_cb_(); @@ -76,11 +78,13 @@ IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& va Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} -LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, absl::Status& creation_status) { +LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, + absl::Status& creation_status) { if (!ip_tags_path.empty()) { if (!absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml) && !absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { - creation_status = absl::InvalidArgumentError("Unsupported file format, unable to parse ip tags from file."); + creation_status = + absl::InvalidArgumentError("Unsupported file format, unable to parse ip tags from file."); return nullptr; } auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); @@ -110,7 +114,8 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, absl::St } LcTrieSharedPtr IpTagsLoader::parseIpTags( - const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status) { + const Protobuf::RepeatedPtrField& ip_tags, + absl::Status& creation_status) { std::vector>> tag_data; tag_data.reserve(ip_tags.size()); for (const auto& ip_tag : ip_tags) { @@ -138,13 +143,13 @@ SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); absl::StatusOr IpTaggingFilterConfig::create( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - const std::string& stat_prefix, Singleton::Manager& singleton_manager, - Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, - Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor) { + const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, + Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, + ProtobufMessage::ValidationVisitor& validation_visitor) { absl::Status creation_status = absl::OkStatus(); auto config_ptr = std::shared_ptr( - new IpTaggingFilterConfig(config, stat_prefix, singleton_manager, scope, runtime, api, dispatcher, validation_visitor, creation_status)); + new IpTaggingFilterConfig(config, stat_prefix, singleton_manager, scope, runtime, api, + dispatcher, validation_visitor, creation_status)); RETURN_IF_NOT_OK(creation_status); return config_ptr; } @@ -175,30 +180,33 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( // TODO(ccaraman): Remove size check once file system support is implemented. // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. if (config.ip_tags().empty() && config.ip_tags_path().empty()) { - creation_status = absl::InvalidArgumentError("HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); + creation_status = absl::InvalidArgumentError( + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); } if (!config.ip_tags().empty() && !config.ip_tags_path().empty()) { - creation_status = absl::InvalidArgumentError("Only one of ip_tags or ip_tags_path can be configured."); + creation_status = + absl::InvalidArgumentError("Only one of ip_tags or ip_tags_path can be configured."); } if (creation_status.ok()) { - if (!config.ip_tags().empty()) { - trie_ = tags_loader_.parseIpTags(config.ip_tags(), creation_status); - } else { - provider_ = ip_tags_registry_->get( - ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, - [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, creation_status); - if (provider_ && provider_->ipTags()) { - trie_ = provider_->ipTags(); + if (!config.ip_tags().empty()) { + trie_ = tags_loader_.parseIpTags(config.ip_tags(), creation_status); } else { - if (creation_status.ok()) { - creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); + provider_ = ip_tags_registry_->get( + ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, + [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, + creation_status); + if (provider_ && provider_->ipTags()) { + trie_ = provider_->ipTags(); + } else { + if (creation_status.ok()) { + creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); + } } + stat_name_set_->rememberBuiltin("ip_tags_reload_success"); + stat_name_set_->rememberBuiltin("ip_tags_reload_error"); } - stat_name_set_->rememberBuiltin("ip_tags_reload_success"); - stat_name_set_->rememberBuiltin("ip_tags_reload_error"); - } } } @@ -207,7 +215,7 @@ void IpTaggingFilterConfig::incCounter(Stats::StatName name) { scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } -IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config){}; +IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {}; IpTaggingFilter::~IpTaggingFilter() = default; diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index bf3f25534c846..2b2f4434155e3 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -33,7 +33,8 @@ class IpTagsLoader { LcTrieSharedPtr loadTags(const std::string& ip_tags_path, absl::Status& creation_status); LcTrieSharedPtr - parseIpTags(const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status); + parseIpTags(const Protobuf::RepeatedPtrField& ip_tags, + absl::Status& creation_status); private: Api::Api& api_; @@ -48,7 +49,8 @@ class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, - Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner, absl::Status& creation_status); + Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner, + absl::Status& creation_status); ~IpTagsProvider(); @@ -87,7 +89,8 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Api::Api& api, Event::Dispatcher& dispatcher, - std::shared_ptr singleton, absl::Status& creation_status) { + std::shared_ptr singleton, + absl::Status& creation_status) { std::shared_ptr ip_tags_provider; const uint64_t key = std::hash()(ip_tags_path); absl::MutexLock lock(&mu_); @@ -96,15 +99,15 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { if (std::shared_ptr provider = it->second.lock()) { ip_tags_provider = provider; } else { - ip_tags_provider = - std::make_shared(ip_tags_path, tags_loader, reload_success_cb, - reload_error_cb, dispatcher, api, singleton, creation_status); + ip_tags_provider = std::make_shared( + ip_tags_path, tags_loader, reload_success_cb, reload_error_cb, dispatcher, api, + singleton, creation_status); ip_tags_registry_[key] = ip_tags_provider; } } else { - ip_tags_provider = - std::make_shared(ip_tags_path, tags_loader, reload_success_cb, - reload_error_cb, dispatcher, api, singleton, creation_status); + ip_tags_provider = std::make_shared( + ip_tags_path, tags_loader, reload_success_cb, reload_error_cb, dispatcher, api, singleton, + creation_status); ip_tags_registry_[key] = ip_tags_provider; } return ip_tags_provider; @@ -133,10 +136,9 @@ class IpTaggingFilterConfig { static absl::StatusOr> create(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - const std::string& stat_prefix, Singleton::Manager& singleton_manager, - Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, - Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor); + const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, + Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, + ProtobufMessage::ValidationVisitor& validation_visitor); Runtime::Loader& runtime() { return runtime_; } FilterRequestType requestType() const { return request_type_; } @@ -171,10 +173,12 @@ class IpTaggingFilterConfig { void incTotal() { incCounter(total_); } private: - IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, - const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor, absl::Status& creation_status); + IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, + const std::string& stat_prefix, Singleton::Manager& singleton_manager, + Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, + Event::Dispatcher& dispatcher, + ProtobufMessage::ValidationVisitor& validation_visitor, + absl::Status& creation_status); static FilterRequestType requestTypeEnum( envoy::extensions::filters::http::ip_tagging::v3::IPTagging::RequestType request_type) { diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index e9c5c34d73435..03cb603f2f093 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -239,9 +239,8 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); auto config_or = - IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, - *stats_.rootScope(), runtime_, *api_, - dispatcher_, validation_visitor_); + IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, *stats_.rootScope(), + runtime_, *api_, dispatcher_, validation_visitor_); if (expected_error.has_value()) { EXPECT_FALSE(config_or.ok()); EXPECT_EQ(expected_error.value(), absl::StrCat(config_or.status())); @@ -280,8 +279,8 @@ request_type: internal header: x-envoy-optional-header action: SANITIZE )EOF"; - initializeFilter(config_yaml, - "INVALID_ARGUMENT: HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); + initializeFilter(config_yaml, "INVALID_ARGUMENT: HTTP IP Tagging Filter requires either ip_tags " + "or ip_tags_path to be specified."); } TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { @@ -297,7 +296,7 @@ request_type: internal ip_tags_path: /test/tags.yaml )EOF"; initializeFilter(config_yaml, - "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_path can be configured."); + "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_path can be configured."); } TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { @@ -306,7 +305,7 @@ request_type: internal ip_tags_path: /test/tags.csv )EOF"; initializeFilter(config_yaml, - "INVALID_ARGUMENT: Unsupported file format, unable to parse ip tags from file."); + "INVALID_ARGUMENT: Unsupported file format, unable to parse ip tags from file."); } TEST_F(IpTaggingFilterTest, InvalidCidr) { @@ -854,9 +853,8 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam(config_); @@ -953,7 +951,8 @@ TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { // EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); // auto request_headers = test_case.tagged_headers_; // EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); +// EXPECT_EQ(test_case.hit_counter_prefix_, +// request_headers.get_(Http::Headers::get().EnvoyIpTags)); // EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); // Http::TestRequestTrailerMapImpl request_trailers; // EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -980,7 +979,8 @@ TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { // EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); // request_headers = test_case.tagged_headers_; // EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); +// EXPECT_EQ(test_case.hit_counter_prefix_, +// request_headers.get_(Http::Headers::get().EnvoyIpTags)); // // Clean up modifications to ip tags file names. // TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); // TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); From 732003a012056e8550221646730bf828a3795a0a Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 5 May 2025 12:50:07 +0200 Subject: [PATCH 17/76] More fixes Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 45 +++--- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 134 +++++++++--------- 3 files changed, 91 insertions(+), 90 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index b50ed60bfbc55..80d34779b2666 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -23,18 +23,21 @@ IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& ta tags_(tags_loader_.loadTags(ip_tags_path_, creation_status)), ip_tags_reload_dispatcher_(api.allocateDispatcher("ip_tags_reload_routine")), ip_tags_file_watcher_(dispatcher.createFilesystemWatcher()), owner_(owner) { - if (ip_tags_path.empty()) { - creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); - } - ip_tags_reload_thread_ = api.threadFactory().createThread( - [this]() -> void { - ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); - THROW_IF_NOT_OK( - ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, - [this](uint32_t) { return onIpTagsFileUpdate(); })); - ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); - }, - Thread::Options{std::string("ip_tags_reload_routine")}); + RETURN_ONLY_IF_NOT_OK_REF(creation_status); + if (ip_tags_path.empty()) { + creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); + return; + } + ip_tags_reload_thread_ = api.threadFactory().createThread( + [this, &creation_status]() -> void { + ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); + SET_AND_RETURN_IF_NOT_OK(ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, + [this](uint32_t) { return onIpTagsFileUpdate(); }), + creation_status); + ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); + }, + Thread::Options{std::string("ip_tags_reload_routine")}); + } IpTagsProvider::~IpTagsProvider() { @@ -94,18 +97,20 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, try { MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); } catch (const EnvoyException& e) { - ENVOY_LOG_MISC(warn, "failed to parse ip tags file: {}", e.what()); + creation_status = + absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); return nullptr; } } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { try { MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); } catch (const EnvoyException& e) { - ENVOY_LOG_MISC(warn, "failed to parse ip tags file: {}", e.what()); + creation_status = + absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); return nullptr; } } - return parseIpTags(ip_tags_proto.ip_tags(), creation_status); + return parseIpTagsAsProto(ip_tags_proto.ip_tags(), creation_status); } else { return nullptr; } @@ -113,7 +118,7 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, return nullptr; } -LcTrieSharedPtr IpTagsLoader::parseIpTags( +LcTrieSharedPtr IpTagsLoader::parseIpTagsAsProto( const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status) { std::vector>> tag_data; @@ -189,25 +194,23 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( absl::InvalidArgumentError("Only one of ip_tags or ip_tags_path can be configured."); } - if (creation_status.ok()) { + RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (!config.ip_tags().empty()) { - trie_ = tags_loader_.parseIpTags(config.ip_tags(), creation_status); + trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); } else { provider_ = ip_tags_registry_->get( ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, creation_status); + RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (provider_ && provider_->ipTags()) { trie_ = provider_->ipTags(); } else { - if (creation_status.ok()) { creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); - } } stat_name_set_->rememberBuiltin("ip_tags_reload_success"); stat_name_set_->rememberBuiltin("ip_tags_reload_error"); } - } } void IpTaggingFilterConfig::incCounter(Stats::StatName name) { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 2b2f4434155e3..2cfaea117cff5 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -33,7 +33,7 @@ class IpTagsLoader { LcTrieSharedPtr loadTags(const std::string& ip_tags_path, absl::Status& creation_status); LcTrieSharedPtr - parseIpTags(const Protobuf::RepeatedPtrField& ip_tags, + parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status); private: diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 03cb603f2f093..1df03e3f917f0 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -738,26 +738,26 @@ INSTANTIATE_TEST_CASE_P(Ipv6Address, Ipv6AddressTest, ::testing::ValuesIn({ipv6_config, ipv6_with_json_file_config, ipv6_with_yaml_file_config})); -// class RuntimeDisabledTest : public IpTaggingFilterTest {}; - -// TEST_P(RuntimeDisabledTest, RuntimeDisabled) { -// const std::string config = GetParam(); -// initializeFilter(config); -// Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - -// EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) -// .WillOnce(Return(false)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// } - -// INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, -// ::testing::ValuesIn({internal_request_config, -// internal_request_with_json_file_config, -// internal_request_with_yaml_file_config})); +class RuntimeDisabledTest : public IpTaggingFilterTest {}; + +TEST_P(RuntimeDisabledTest, RuntimeDisabled) { + const std::string config = GetParam(); + initializeFilter(config); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) + .WillOnce(Return(false)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +INSTANTIATE_TEST_CASE_P(RuntimeDisabled, RuntimeDisabledTest, + ::testing::ValuesIn({internal_request_config, + internal_request_with_json_file_config, + internal_request_with_yaml_file_config})); class ClearRouteCacheTest : public IpTaggingFilterTest {}; @@ -937,54 +937,52 @@ TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); } -// TEST_P(IpTagsFileReloadImplTest, IpTagsFileNotReloaded) { -// IpTagsFileReloadTestCase test_case = GetParam(); -// auto cb_added_opt = absl::make_optional(); -// initializeFilter(test_case.yaml_config_, cb_added_opt); -// EXPECT_EQ(test_case.request_type_, config_->requestType()); -// Network::Address::InstanceConstSharedPtr remote_address = -// Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); -// EXPECT_CALL(stats_, -// counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); -// EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); -// auto request_headers = test_case.tagged_headers_; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ(test_case.hit_counter_prefix_, -// request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); -// Http::TestRequestTrailerMapImpl request_trailers; -// EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, -// filter_->decodeHeaders(test_case.not_tagged_headers_, false)); -// EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); -// std::string source_ip_tags_file_path = -// TestEnvironment::substitute(test_case.source_ip_tags_file_path_); -// std::string reloaded_ip_tags_file_path = TestEnvironment::substitute( -// "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml"); -// TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); -// TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); -// EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); -// cb_added_opt.value().waitReady(); -// { -// absl::ReaderMutexLock guard(&mutex_); -// EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); -// } -// // Current ip tags should be used if reload failed. -// filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( -// remote_address); -// EXPECT_CALL(stats_, -// counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); -// EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); -// request_headers = test_case.tagged_headers_; -// EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); -// EXPECT_EQ(test_case.hit_counter_prefix_, -// request_headers.get_(Http::Headers::get().EnvoyIpTags)); -// // Clean up modifications to ip tags file names. -// TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); -// TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); -// } +TEST_P(IpTagsFileReloadImplTest, IpTagsFileNotReloaded) { + IpTagsFileReloadTestCase test_case = GetParam(); + auto cb_added_opt = absl::make_optional(); + initializeFilter(test_case.yaml_config_, cb_added_opt); + EXPECT_EQ(test_case.request_type_, config_->requestType()); + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + auto request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->decodeHeaders(test_case.not_tagged_headers_, false)); + EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); + std::string source_ip_tags_file_path = + TestEnvironment::substitute(test_case.source_ip_tags_file_path_); + std::string reloaded_ip_tags_file_path = TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml"); + TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); + TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); + cb_added_opt.value().waitReady(); + { + absl::ReaderMutexLock guard(&mutex_); + EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); + } + // Current ip tags should be used if reload failed. + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + EXPECT_CALL(stats_, + counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); + request_headers = test_case.tagged_headers_; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + // Clean up modifications to ip tags file names. + TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); + TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); +} struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { {internal_request_with_yaml_file_config, From f0fd9f95a7e47810dc8bfb3d1357454b2e7f3974 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 6 May 2025 13:08:55 +0200 Subject: [PATCH 18/76] Fix race condition Signed-off-by: Kateryna Nezdolii --- source/common/event/dispatcher_impl.cc | 6 +- source/common/runtime/runtime_features.cc | 2 +- .../http/ip_tagging/ip_tagging_filter.cc | 65 +++++++++---------- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 5 +- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 9a29e50cd77dc..5295f7eb67f8d 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -90,7 +90,6 @@ DispatcherImpl::DispatcherImpl(const std::string& name, Thread::ThreadFactory& t DispatcherImpl::~DispatcherImpl() { ENVOY_LOG(debug, "destroying dispatcher {}", name_); - std::cerr << "deleting dispatcher" << std::endl; FatalErrorHandler::removeFatalErrorHandler(*this); // TODO(lambdai): Resolve https://github.com/envoyproxy/envoy/issues/15072 and enable // ASSERT(deletable_in_dispatcher_thread_.empty()) @@ -238,10 +237,7 @@ void DispatcherImpl::deferredDelete(DeferredDeletablePtr&& to_delete) { } } -void DispatcherImpl::exit() { - std::cerr << "*****Exiting dispatcher" << std::endl; - base_scheduler_.loopExit(); -} +void DispatcherImpl::exit() { base_scheduler_.loopExit(); } SignalEventPtr DispatcherImpl::listenForSignal(signal_t signal_num, SignalCb cb) { ASSERT(isThreadSafe()); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index f40911f6696a5..d908db5c3b283 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -99,7 +99,7 @@ RUNTIME_GUARD(envoy_reloadable_features_validate_upstream_headers); RUNTIME_GUARD(envoy_reloadable_features_wait_for_first_byte_before_balsa_msg_done); RUNTIME_GUARD(envoy_reloadable_features_xds_failover_to_primary_enabled); RUNTIME_GUARD(envoy_reloadable_features_xds_prevent_resource_copy); -FALSE_RUNTIME_GUARD(envoy_restart_features_fix_dispatcher_approximate_now); +RUNTIME_GUARD(envoy_restart_features_fix_dispatcher_approximate_now); RUNTIME_GUARD(envoy_restart_features_skip_backing_cluster_check_for_sds); RUNTIME_GUARD(envoy_restart_features_use_eds_cache_for_ads); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 80d34779b2666..4907ad85b90c9 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -23,26 +23,28 @@ IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& ta tags_(tags_loader_.loadTags(ip_tags_path_, creation_status)), ip_tags_reload_dispatcher_(api.allocateDispatcher("ip_tags_reload_routine")), ip_tags_file_watcher_(dispatcher.createFilesystemWatcher()), owner_(owner) { - RETURN_ONLY_IF_NOT_OK_REF(creation_status); - if (ip_tags_path.empty()) { - creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); - return; - } - ip_tags_reload_thread_ = api.threadFactory().createThread( - [this, &creation_status]() -> void { - ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); - SET_AND_RETURN_IF_NOT_OK(ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, - [this](uint32_t) { return onIpTagsFileUpdate(); }), - creation_status); - ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); - }, - Thread::Options{std::string("ip_tags_reload_routine")}); - + RETURN_ONLY_IF_NOT_OK_REF(creation_status); + if (ip_tags_path.empty()) { + creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); + return; + } + ip_tags_reload_thread_ = api.threadFactory().createThread( + [this, &creation_status]() -> void { + ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); + SET_AND_RETURN_IF_NOT_OK( + ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, + [this](uint32_t) { return onIpTagsFileUpdate(); }), + creation_status); + ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); + }, + Thread::Options{std::string("ip_tags_reload_routine")}); } IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); - ip_tags_reload_dispatcher_->exit(); + if (ip_tags_reload_dispatcher_) { + ip_tags_reload_dispatcher_->exit(); + } if (ip_tags_reload_thread_) { ip_tags_reload_thread_->join(); ip_tags_reload_thread_.reset(); @@ -195,22 +197,21 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( } RETURN_ONLY_IF_NOT_OK_REF(creation_status); - if (!config.ip_tags().empty()) { - trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); + if (!config.ip_tags().empty()) { + trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); + } else { + provider_ = ip_tags_registry_->get( + ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, + [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, creation_status); + RETURN_ONLY_IF_NOT_OK_REF(creation_status); + if (provider_ && provider_->ipTags()) { + trie_ = provider_->ipTags(); } else { - provider_ = ip_tags_registry_->get( - ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, - [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, - creation_status); - RETURN_ONLY_IF_NOT_OK_REF(creation_status); - if (provider_ && provider_->ipTags()) { - trie_ = provider_->ipTags(); - } else { - creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); - } - stat_name_set_->rememberBuiltin("ip_tags_reload_success"); - stat_name_set_->rememberBuiltin("ip_tags_reload_error"); + creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); } + stat_name_set_->rememberBuiltin("ip_tags_reload_success"); + stat_name_set_->rememberBuiltin("ip_tags_reload_error"); + } } void IpTaggingFilterConfig::incCounter(Stats::StatName name) { @@ -237,10 +238,6 @@ Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& std::vector tags = config_->trie().getData(callbacks_->streamInfo().downstreamAddressProvider().remoteAddress()); - std::cerr << "***Tags from decodeHeaders: " << std::endl; - for (auto t : tags) - std::cerr << t << ' '; - std::cerr << std::endl; applyTags(headers, tags); if (!tags.empty()) { // For a large number(ex > 1000) of tags, stats cardinality will be an issue. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 2cfaea117cff5..9ab6c57d33f57 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -34,7 +34,7 @@ class IpTagsLoader { LcTrieSharedPtr parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags, - absl::Status& creation_status); + absl::Status& creation_status); private: Api::Api& api_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 1df03e3f917f0..254e96bb15f2d 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -260,6 +260,7 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { std::unique_ptr singleton_manager_ = std::make_unique(); + Api::ApiPtr api_; Event::MockDispatcher dispatcher_; NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; @@ -267,11 +268,9 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { NiceMock filter_callbacks_; Buffer::OwnedImpl data_; NiceMock runtime_; - Api::ApiPtr api_; NiceMock validation_visitor_; }; -// TODO nezdolik split config tests into separate test file TEST_F(IpTaggingFilterTest, NoIpTagsConfigured) { const std::string config_yaml = R"EOF( request_type: internal @@ -873,6 +872,7 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam singleton_manager_ = std::make_unique(); + Api::ApiPtr api_; Event::MockDispatcher dispatcher_; NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; @@ -880,7 +880,6 @@ class IpTagsFileReloadImplTest : public ::testing::TestWithParam filter_callbacks_; Buffer::OwnedImpl data_; NiceMock runtime_; - Api::ApiPtr api_; NiceMock validation_visitor_; absl::Mutex mutex_; std::vector on_changed_cbs_ ABSL_GUARDED_BY(mutex_); From 39bc535f8a3ebba9531a4abe2c9006c96ede0e49 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 6 May 2025 13:09:55 +0200 Subject: [PATCH 19/76] Cleanup Signed-off-by: Kateryna Nezdolii --- .bazelrc | 13 +++++-------- bazel/envoy_internal.bzl | 1 - 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.bazelrc b/.bazelrc index a8aaedb20d56c..d8d1c6e4424fc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -67,7 +67,6 @@ build:linux --copt=-fdebug-types-section # (Workaround for https://github.com/bazelbuild/rules_foreign_cc/issues/421) build:linux --copt=-fPIC build:linux --copt=-Wno-deprecated-declarations -build:linux --copt=-Wno-implicit-function-declaration build:linux --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 build:linux --cxxopt=-fsized-deallocation --host_cxxopt=-fsized-deallocation build:linux --conlyopt=-fexceptions @@ -88,8 +87,7 @@ build:sanitizer --linkopt -ldl # Common flags for Clang build:clang --action_env=BAZEL_COMPILER=clang -build:clang --linkopt=-fuse-ld=ld -build:clang --copt=-Wno-implicit-function-declaration +build:clang --linkopt=-fuse-ld=lld build:clang --action_env=CC=clang --host_action_env=CC=clang build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ build:clang --incompatible_enable_cc_toolchain_resolution=false @@ -153,7 +151,7 @@ build:asan --copt -fno-optimize-sibling-calls # Clang ASAN/UBSAN build:clang-asan-common --config=clang build:clang-asan-common --config=asan -build:clang-asan-common --linkopt -fuse-ld=ld +build:clang-asan-common --linkopt -fuse-ld=lld build:clang-asan-common --linkopt --rtlib=compiler-rt build:clang-asan-common --linkopt --unwindlib=libgcc @@ -167,7 +165,6 @@ build:clang-asan --linkopt=-fsanitize=vptr,function # macOS build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 build:macos --copt=-Wno-deprecated-declarations -build:macos --copt=-Wno-implicit-function-declaration build:macos --action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin build:macos --host_action_env=PATH=/opt/homebrew/bin:/opt/local/bin:/usr/local/bin:/usr/bin:/bin build:macos --define tcmalloc=disabled @@ -188,7 +185,7 @@ build:clang-tsan --config=sanitizer build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread -build:clang-tsan --linkopt -fuse-ld=ld +build:clang-tsan --linkopt -fuse-ld=lld build:clang-tsan --copt -DTHREAD_SANITIZER=1 build:clang-tsan --build_tag_filters=-no_san,-no_tsan build:clang-tsan --test_tag_filters=-no_san,-no_tsan @@ -209,7 +206,7 @@ build:clang-msan --test_tag_filters=-no_san build:clang-msan --define ENVOY_CONFIG_MSAN=1 build:clang-msan --copt -fsanitize=memory build:clang-msan --linkopt -fsanitize=memory -build:clang-msan --linkopt -fuse-ld=ld +build:clang-msan --linkopt -fuse-ld=lld build:clang-msan --copt -fsanitize-memory-track-origins=2 build:clang-msan --copt -DMEMORY_SANITIZER=1 build:clang-msan --test_env=MSAN_SYMBOLIZER_PATH @@ -309,7 +306,7 @@ build:rbe-toolchain-arm64-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-asan --config=clang-asan -build:rbe-toolchain-asan --linkopt -fuse-ld=ld +build:rbe-toolchain-asan --linkopt -fuse-ld=lld build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1 build:rbe-toolchain-asan --copt=-fsanitize=vptr,function build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 0cd6793d67f7f..866a00f280748 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -10,7 +10,6 @@ def envoy_copts(repository, test = False): "-Wextra", "-Werror", "-Wnon-virtual-dtor", - "-Wno-implicit-function-declaration", "-Woverloaded-virtual", "-Wold-style-cast", "-Wformat", From e2e459b66d0b73b4f8e9ac005de8cce581202a25 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 6 May 2025 13:23:30 +0200 Subject: [PATCH 20/76] Cleanup Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 20 +++++++------------ .../http/ip_tagging/ip_tagging_filter.h | 3 +-- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 4907ad85b90c9..32b3a74a596ad 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -95,22 +95,16 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); if (file_or_error.status().ok()) { IpTagFileProto ip_tags_proto; - if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { - try { + try { + if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); - } catch (const EnvoyException& e) { - creation_status = - absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); - return nullptr; - } - } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { - try { + } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); - } catch (const EnvoyException& e) { - creation_status = - absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); - return nullptr; } + } catch (const EnvoyException& e) { + creation_status = + absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); + return nullptr; } return parseIpTagsAsProto(ip_tags_proto.ip_tags(), creation_status); } else { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 9ab6c57d33f57..f9e95aa9ea878 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -116,8 +116,7 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { private: absl::Mutex mu_; // Each provider stores shared_ptrs to this singleton, which keeps the singleton - // from being destroyed unless it's no longer keeping track of any providers. (The singleton - // shared_ptr is *only* held by driver instances.) + // from being destroyed unless it's no longer keeping track of any providers. absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); }; From 16aa10f49d272269a0c524f2a266715eae1d5bd5 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 6 May 2025 15:44:07 +0200 Subject: [PATCH 21/76] Get rid of exception Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 32b3a74a596ad..3e461b0191d31 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -95,16 +95,22 @@ LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); if (file_or_error.status().ok()) { IpTagFileProto ip_tags_proto; - try { - if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { + if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { + TRY_NEEDS_AUDIT { MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); - } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { - MessageUtil::loadFromJson(file_or_error.value(), ip_tags_proto, validation_visitor_); } - } catch (const EnvoyException& e) { - creation_status = - absl::InvalidArgumentError(fmt::format("failed to parse ip tags file: {}", e.what())); - return nullptr; + END_TRY catch (EnvoyException& ex) { + creation_status = absl::InvalidArgumentError( + fmt::format("failed to parse ip tags file as yaml: {}", ex.what())); + return nullptr; + } + } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { + bool has_unknown_field; + creation_status = MessageUtil::loadFromJsonNoThrow(file_or_error.value(), ip_tags_proto, + has_unknown_field); + if (!creation_status.ok()) { + return nullptr; + } } return parseIpTagsAsProto(ip_tags_proto.ip_tags(), creation_status); } else { From 4b93e630e44227b3f9b876963dba95755b64db86 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 15 May 2025 17:16:42 +0200 Subject: [PATCH 22/76] Switch to datasource impl Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/v3/BUILD | 1 + .../http/ip_tagging/v3/ip_tagging.proto | 22 +- .../http/http_filters/ip_tagging_filter.rst | 7 + .../extensions/filters/http/ip_tagging/BUILD | 3 + .../filters/http/ip_tagging/config.cc | 2 + .../http/ip_tagging/ip_tagging_filter.cc | 208 ++++---- .../http/ip_tagging/ip_tagging_filter.h | 69 +-- test/extensions/filters/http/ip_tagging/BUILD | 1 + .../http/ip_tagging/ip_tagging_filter_test.cc | 472 +++++++++--------- 9 files changed, 411 insertions(+), 374 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD index 9ebe0ec6dfa52..8c2da79c79be0 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD @@ -6,6 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ + "//envoy/config/core/v3:pkg", "//envoy/data/ip_tagging/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", ], diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 8bfe3925a4e18..0c44c0b4488a6 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.extensions.filters.http.ip_tagging.v3; +import "google/protobuf/duration.proto"; +import "envoy/config/core/v3/base.proto"; import "envoy/data/ip_tagging/v3/ip_tagging.proto"; import "udpa/annotations/status.proto"; @@ -18,7 +20,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // IP tagging :ref:`configuration overview `. // [#extension: envoy.filters.http.ip_tagging] -// [#next-free-field: 7] +// [#next-free-field: 8] message IPTagging { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.ip_tagging.v2.IPTagging"; @@ -86,10 +88,16 @@ message IPTagging { // If left unspecified, the tags will be appended to the ``x-envoy-ip-tags`` header. IpTagHeader ip_tag_header = 5; - // Absolute path to the file with ip tags. - // Supports Yaml and Json file formats for ip tags. - // Only one of :ref:`ip_tags ` - // or :ref:`ip_tags_path ` - // can be set for the Ip Tagging filter. - string ip_tags_path = 6; + // Data source from which to retrieve ip tags. + // Only filename based data source is currently supported for ip tags. + // When using this data source, if a ``watched_directory`` is provided, the ip tags file will be re-read when a file move is detected. + // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. + config.core.v3.DataSource ip_tags_datasource = 6; + + + // When :ref:`ip_tags ` is configured + // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. + // Deafults to 0, in this case no refresh will be attempted. + google.protobuf.Duration ip_tags_refresh_rate = 7 + [(validate.rules).duration = {gt {}}]; } diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index 73e4a87c05dfd..f618ba6ff9521 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -22,6 +22,11 @@ described in the paper `IP-address lookup using LC-tries `_ by S. Nilsson and G. Karlsson. +Ip tags can either be parsed from :ref:`ip_tags ` api field or +can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. +Ip tags will be dynamically realoded if :ref:`ip_tags_datasource.watched_directory ` +is configured and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. + Configuration ------------- @@ -41,6 +46,8 @@ the owning HTTP connection manager. .hit, Counter, Total number of requests that have the ```` applied to it no_hit, Counter, Total number of requests with no applicable IP tags total, Counter, Total number of requests the IP Tagging Filter operated on + reload_success, Counter, Total number of successful reloads of ip tags file + reload_error, Counter, Total number of failed reloads of ip tags file Runtime ------- diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 69480dd03ccd5..9b891385fb689 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -19,7 +19,9 @@ envoy_cc_library( deps = [ "//envoy/http:filter_interface", "//envoy/runtime:runtime_interface", + "//envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", + "//source/common/config:datasource_lib", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/network:lc_trie_lib", @@ -35,6 +37,7 @@ envoy_cc_extension( hdrs = ["config.h"], deps = [ "//envoy/registry", + "//envoy/thread_local:thread_local_interface", "//source/common/network:lc_trie_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", diff --git a/source/extensions/filters/http/ip_tagging/config.cc b/source/extensions/filters/http/ip_tagging/config.cc index 9fa7ed8481e54..db63caede19e8 100644 --- a/source/extensions/filters/http/ip_tagging/config.cc +++ b/source/extensions/filters/http/ip_tagging/config.cc @@ -3,6 +3,7 @@ #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" #include "envoy/registry/registry.h" +#include "envoy/thread_local/thread_local.h" #include "source/common/protobuf/utility.h" #include "source/extensions/filters/http/ip_tagging/ip_tagging_filter.h" @@ -19,6 +20,7 @@ absl::StatusOr IpTaggingFilterFactory::createFilterFactor absl::StatusOr config = IpTaggingFilterConfig::create( proto_config, stat_prefix, context.serverFactoryContext().singletonManager(), context.scope(), context.serverFactoryContext().runtime(), context.serverFactoryContext().api(), + context.serverFactoryContext().threadLocal(), context.serverFactoryContext().mainThreadDispatcher(), context.messageValidationVisitor()); RETURN_IF_NOT_OK_REF(config.status()); return diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 3e461b0191d31..1ae4dcf71ebb8 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -3,6 +3,7 @@ #include "envoy/config/core/v3/address.pb.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.h" +#include "source/common/config/datasource.h" #include "source/common/http/header_map_impl.h" #include "source/common/http/headers.h" @@ -13,113 +14,138 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -IpTagsProvider::IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, +IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, + IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Event::Dispatcher& dispatcher, - Api::Api& api, Singleton::InstanceSharedPtr owner, - absl::Status& creation_status) - : ip_tags_path_(ip_tags_path), tags_loader_(tags_loader), reload_success_cb_(reload_success_cb), - reload_error_cb_(reload_error_cb), - tags_(tags_loader_.loadTags(ip_tags_path_, creation_status)), - ip_tags_reload_dispatcher_(api.allocateDispatcher("ip_tags_reload_routine")), - ip_tags_file_watcher_(dispatcher.createFilesystemWatcher()), owner_(owner) { + Api::Api& api, ThreadLocal::SlotAllocator& tls, + Singleton::InstanceSharedPtr owner, absl::Status& creation_status) + : ip_tags_path_(ip_tags_datasource.filename()), tags_loader_(tags_loader), + time_source_(api.timeSource()), + ip_tags_refresh_interval_ms_(std::chrono::milliseconds(ip_tags_refresh_interval_ms)), + reload_success_cb_(reload_success_cb), reload_error_cb_(reload_error_cb), owner_(owner) { RETURN_ONLY_IF_NOT_OK_REF(creation_status); - if (ip_tags_path.empty()) { + if (ip_tags_datasource.filename().empty()) { creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); return; } - ip_tags_reload_thread_ = api.threadFactory().createThread( - [this, &creation_status]() -> void { - ENVOY_LOG_MISC(debug, "Started ip_tags_reload_routine"); - SET_AND_RETURN_IF_NOT_OK( - ip_tags_file_watcher_->addWatch(ip_tags_path_, Filesystem::Watcher::Events::MovedTo, - [this](uint32_t) { return onIpTagsFileUpdate(); }), - creation_status); - ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); - }, - Thread::Options{std::string("ip_tags_reload_routine")}); + if (ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0) && + ip_tags_datasource.has_watched_directory()) { + needs_refresh_ = true; + } + tags_ = tags_loader_.loadTags(ip_tags_datasource, dispatcher, tls, creation_status); } -IpTagsProvider::~IpTagsProvider() { - ENVOY_LOG(debug, "Shutting down ip tags provider"); - if (ip_tags_reload_dispatcher_) { - ip_tags_reload_dispatcher_->exit(); - } - if (ip_tags_reload_thread_) { - ip_tags_reload_thread_->join(); - ip_tags_reload_thread_.reset(); +IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); }; + +LcTrieSharedPtr IpTagsProvider::ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { + if (needs_refresh_) { + const auto current_time = time_source_.monotonicTime(); + if (std::chrono::duration_cast(current_time - last_reloaded_time_) > + ip_tags_refresh_interval_ms_) { + absl::Status refresh_status = absl::OkStatus(); + auto new_tags = tags_loader_.refreshTags(refresh_status); + if (new_tags) { + absl::MutexLock lock(&ip_tags_mutex_); + tags_ = new_tags; + reload_success_cb_(); + } else { + ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", refresh_status.message()); + reload_error_cb_(); + } + last_reloaded_time_ = current_time; + } } -}; - -LcTrieSharedPtr IpTagsProvider::ipTags() const ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { absl::ReaderMutexLock lock(&ip_tags_mutex_); return tags_; -}; - -absl::Status IpTagsProvider::onIpTagsFileUpdate() { - absl::Status reload_status; - LcTrieSharedPtr reloaded_tags = tags_loader_.loadTags(ip_tags_path_, reload_status); - return ipTagsReload(reloaded_tags, reload_status); } -absl::Status IpTagsProvider::ipTagsReload(const LcTrieSharedPtr reloaded_tags, - absl::Status& reload_status) { - if (reload_status.ok()) { - updateIpTags(reloaded_tags); - reload_success_cb_(); +std::shared_ptr IpTagsRegistrySingleton::get( + const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, + uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, + IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& main_dispatcher, std::shared_ptr singleton, + absl::Status& creation_status) { + std::shared_ptr ip_tags_provider; + const uint64_t key = std::hash()(ip_tags_datasource.filename()); + absl::MutexLock lock(&mu_); + auto it = ip_tags_registry_.find(key); + if (it != ip_tags_registry_.end()) { + if (std::shared_ptr provider = it->second.lock()) { + ip_tags_provider = provider; + } else { + ip_tags_provider = std::make_shared( + ip_tags_datasource, tags_loader, ip_tags_refresh_interval_ms, reload_success_cb, + reload_error_cb, main_dispatcher, api, tls, singleton, creation_status); + ip_tags_registry_[key] = ip_tags_provider; + } } else { - reload_error_cb_(); + ip_tags_provider = std::make_shared( + ip_tags_datasource, tags_loader, ip_tags_refresh_interval_ms, reload_success_cb, + reload_error_cb, main_dispatcher, api, tls, singleton, creation_status); + ip_tags_registry_[key] = ip_tags_provider; } - return absl::OkStatus(); -} - -void IpTagsProvider::updateIpTags(const LcTrieSharedPtr reloaded_tags) - ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { - absl::MutexLock lock(&ip_tags_mutex_); - tags_ = reloaded_tags; + return ip_tags_provider; } IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} -LcTrieSharedPtr IpTagsLoader::loadTags(const std::string& ip_tags_path, - absl::Status& creation_status) { - if (!ip_tags_path.empty()) { - if (!absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml) && - !absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { +LcTrieSharedPtr +IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, + Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, + absl::Status& creation_status) { + if (!ip_tags_datasource.filename().empty()) { + if (!absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Yaml) && + !absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Json)) { creation_status = - absl::InvalidArgumentError("Unsupported file format, unable to parse ip tags from file."); + absl::InvalidArgumentError("Unsupported file format, unable to parse ip tags from file " + + ip_tags_datasource.filename()); return nullptr; } - auto file_or_error = api_.fileSystem().fileReadToEnd(ip_tags_path); - if (file_or_error.status().ok()) { - IpTagFileProto ip_tags_proto; - if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Yaml)) { - TRY_NEEDS_AUDIT { - MessageUtil::loadFromYaml(file_or_error.value(), ip_tags_proto, validation_visitor_); - } - END_TRY catch (EnvoyException& ex) { - creation_status = absl::InvalidArgumentError( - fmt::format("failed to parse ip tags file as yaml: {}", ex.what())); - return nullptr; - } - } else if (absl::EndsWith(ip_tags_path, MessageUtil::FileExtensions::get().Json)) { - bool has_unknown_field; - creation_status = MessageUtil::loadFromJsonNoThrow(file_or_error.value(), ip_tags_proto, - has_unknown_field); - if (!creation_status.ok()) { - return nullptr; - } - } - return parseIpTagsAsProto(ip_tags_proto.ip_tags(), creation_status); - } else { + auto provider_or_error = Config::DataSource::DataSourceProvider::create( + ip_tags_datasource, dispatcher, tls, api_, false, 0); + if (!provider_or_error.ok()) { + creation_status = absl::InvalidArgumentError( + fmt::format("unable to create data source '{}'", provider_or_error.status().message())); return nullptr; } + data_source_provider_ = std::move(provider_or_error.value()); + ip_tags_path_ = ip_tags_datasource.filename(); + return refreshTags(creation_status); } return nullptr; } +LcTrieSharedPtr IpTagsLoader::refreshTags(absl::Status& refresh_status) { + if (data_source_provider_) { + IpTagFileProto ip_tags_proto; + if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { + TRY_NEEDS_AUDIT { + MessageUtil::loadFromYaml(data_source_provider_->data(), ip_tags_proto, + validation_visitor_); + } + END_TRY catch (EnvoyException& ex) { + refresh_status = absl::InvalidArgumentError( + fmt::format("failed to parse ip tags file as yaml: {}", ex.what())); + return nullptr; + } + } else if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Json)) { + bool has_unknown_field; + refresh_status = MessageUtil::loadFromJsonNoThrow(data_source_provider_->data(), + ip_tags_proto, has_unknown_field); + if (!refresh_status.ok()) { + return nullptr; + } + } + return parseIpTagsAsProto(ip_tags_proto.ip_tags(), refresh_status); + } else { + refresh_status = absl::InvalidArgumentError("Unable to load tags from empty datasource"); + return nullptr; + } +} + LcTrieSharedPtr IpTagsLoader::parseIpTagsAsProto( const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status) { @@ -151,11 +177,11 @@ SINGLETON_MANAGER_REGISTRATION(ip_tags_registry); absl::StatusOr IpTaggingFilterConfig::create( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor) { + Runtime::Loader& runtime, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor) { absl::Status creation_status = absl::OkStatus(); auto config_ptr = std::shared_ptr( - new IpTaggingFilterConfig(config, stat_prefix, singleton_manager, scope, runtime, api, + new IpTaggingFilterConfig(config, stat_prefix, singleton_manager, scope, runtime, api, tls, dispatcher, validation_visitor, creation_status)); RETURN_IF_NOT_OK(creation_status); return config_ptr; @@ -164,8 +190,9 @@ absl::StatusOr IpTaggingFilterConfig::create( IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor, absl::Status& creation_status) + Runtime::Loader& runtime, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor, + absl::Status& creation_status) : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), stat_name_set_(scope.symbolTable().makeSet("IpTagging")), stats_prefix_(stat_name_set_->add(stat_prefix + "ip_tagging")), @@ -175,7 +202,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( ip_tag_header_action_(config.has_ip_tag_header() ? config.ip_tag_header().action() : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE), - ip_tags_path_(config.ip_tags_path()), + ip_tags_path_(config.ip_tags_datasource().filename()), ip_tags_registry_(singleton_manager.getTyped( SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), [] { return std::make_shared(); })), @@ -186,23 +213,25 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( // to be implemented. // TODO(ccaraman): Remove size check once file system support is implemented. // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. - if (config.ip_tags().empty() && config.ip_tags_path().empty()) { + if (config.ip_tags().empty() && !config.has_ip_tags_datasource()) { creation_status = absl::InvalidArgumentError( - "HTTP IP Tagging Filter requires either ip_tags or ip_tags_path to be specified."); + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_datasource to be specified."); } - if (!config.ip_tags().empty() && !config.ip_tags_path().empty()) { + if (!config.ip_tags().empty() && config.has_ip_tags_datasource()) { creation_status = - absl::InvalidArgumentError("Only one of ip_tags or ip_tags_path can be configured."); + absl::InvalidArgumentError("Only one of ip_tags or ip_tags_datasource can be configured."); } RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); } else { + auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config, ip_tags_refresh_rate, 0); provider_ = ip_tags_registry_->get( - ip_tags_path_, tags_loader_, [this]() { incIpTagsReloadSuccess(); }, - [this]() { incIpTagsReloadError(); }, api, dispatcher, ip_tags_registry_, creation_status); + config.ip_tags_datasource(), tags_loader_, ip_tags_refresh_interval_ms, + [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, tls, + dispatcher, ip_tags_registry_, creation_status); RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (provider_ && provider_->ipTags()) { trie_ = provider_->ipTags(); @@ -226,6 +255,7 @@ IpTaggingFilter::~IpTaggingFilter() = default; void IpTaggingFilter::onDestroy() {} Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& headers, bool) { + const bool is_internal_request = headers.EnvoyInternalRequest() && (headers.EnvoyInternalRequest()->value() == Http::Headers::get().EnvoyInternalRequestValues.True.c_str()); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index f9e95aa9ea878..06b38c66b9cfb 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -12,7 +12,9 @@ #include "envoy/http/filter.h" #include "envoy/runtime/runtime.h" #include "envoy/stats/scope.h" +#include "envoy/thread_local/thread_local.h" +#include "source/common/config/datasource.h" #include "source/common/network/cidr_range.h" #include "source/common/network/lc_trie.h" #include "source/common/stats/symbol_table.h" @@ -30,7 +32,11 @@ class IpTagsLoader { IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set); - LcTrieSharedPtr loadTags(const std::string& ip_tags_path, absl::Status& creation_status); + LcTrieSharedPtr loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, + Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, + absl::Status& creation_status); + + LcTrieSharedPtr refreshTags(absl::Status& refresh_status); LcTrieSharedPtr parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags, @@ -38,6 +44,8 @@ class IpTagsLoader { private: Api::Api& api_; + Envoy::Config::DataSource::DataSourceProviderPtr data_source_provider_; + std::string ip_tags_path_; ProtobufMessage::ValidationVisitor& validation_visitor_; Stats::StatNameSetPtr& stat_name_set_; }; @@ -47,29 +55,27 @@ using IpTagsReloadErrorCb = std::function; class IpTagsProvider : public Logger::Loggable { public: - IpTagsProvider(const std::string& ip_tags_path, IpTagsLoader& tags_loader, + IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, + IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, - Event::Dispatcher& dispatcher, Api::Api& api, Singleton::InstanceSharedPtr owner, - absl::Status& creation_status); + Event::Dispatcher& dispatcher, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Singleton::InstanceSharedPtr owner, absl::Status& creation_status); ~IpTagsProvider(); - LcTrieSharedPtr ipTags() const ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); - - absl::Status onIpTagsFileUpdate(); - absl::Status ipTagsReload(const LcTrieSharedPtr reloaded_tags, absl::Status& reload_status); - void updateIpTags(const LcTrieSharedPtr reloaded_tags) ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); + LcTrieSharedPtr ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); private: const std::string ip_tags_path_; IpTagsLoader& tags_loader_; + bool needs_refresh_{false}; + TimeSource& time_source_; + MonotonicTime last_reloaded_time_; + const std::chrono::milliseconds ip_tags_refresh_interval_ms_; IpTagsReloadSuccessCb reload_success_cb_; IpTagsReloadErrorCb reload_error_cb_; mutable absl::Mutex ip_tags_mutex_; LcTrieSharedPtr tags_ ABSL_GUARDED_BY(ip_tags_mutex_); - Thread::ThreadPtr ip_tags_reload_thread_; - Event::DispatcherPtr ip_tags_reload_dispatcher_; - Filesystem::WatcherPtr ip_tags_file_watcher_; // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. const Singleton::InstanceSharedPtr owner_; }; @@ -85,33 +91,12 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() {} - std::shared_ptr get(const std::string& ip_tags_path, IpTagsLoader& tags_loader, - IpTagsReloadSuccessCb reload_success_cb, - IpTagsReloadErrorCb reload_error_cb, Api::Api& api, - Event::Dispatcher& dispatcher, - std::shared_ptr singleton, - absl::Status& creation_status) { - std::shared_ptr ip_tags_provider; - const uint64_t key = std::hash()(ip_tags_path); - absl::MutexLock lock(&mu_); - auto it = ip_tags_registry_.find(key); - if (it != ip_tags_registry_.end()) { - if (std::shared_ptr provider = it->second.lock()) { - ip_tags_provider = provider; - } else { - ip_tags_provider = std::make_shared( - ip_tags_path, tags_loader, reload_success_cb, reload_error_cb, dispatcher, api, - singleton, creation_status); - ip_tags_registry_[key] = ip_tags_provider; - } - } else { - ip_tags_provider = std::make_shared( - ip_tags_path, tags_loader, reload_success_cb, reload_error_cb, dispatcher, api, singleton, - creation_status); - ip_tags_registry_[key] = ip_tags_provider; - } - return ip_tags_provider; - } + std::shared_ptr + get(const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, + uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, + IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& main_dispatcher, std::shared_ptr singleton, + absl::Status& creation_status); private: absl::Mutex mu_; @@ -136,8 +121,8 @@ class IpTaggingFilterConfig { static absl::StatusOr> create(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, - Runtime::Loader& runtime, Api::Api& api, Event::Dispatcher& dispatcher, - ProtobufMessage::ValidationVisitor& validation_visitor); + Runtime::Loader& runtime, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor); Runtime::Loader& runtime() { return runtime_; } FilterRequestType requestType() const { return request_type_; } @@ -175,7 +160,7 @@ class IpTaggingFilterConfig { IpTaggingFilterConfig(const envoy::extensions::filters::http::ip_tagging::v3::IPTagging& config, const std::string& stat_prefix, Singleton::Manager& singleton_manager, Stats::Scope& scope, Runtime::Loader& runtime, Api::Api& api, - Event::Dispatcher& dispatcher, + ThreadLocal::SlotAllocator& tls, Event::Dispatcher& dispatcher, ProtobufMessage::ValidationVisitor& validation_visitor, absl::Status& creation_status); diff --git a/test/extensions/filters/http/ip_tagging/BUILD b/test/extensions/filters/http/ip_tagging/BUILD index 50f2e40e5e6de..28b9dc8a78b28 100644 --- a/test/extensions/filters/http/ip_tagging/BUILD +++ b/test/extensions/filters/http/ip_tagging/BUILD @@ -32,6 +32,7 @@ envoy_extension_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/stats:stats_mocks", + "//test/mocks/thread_local:thread_local_mocks", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/filters/http/ip_tagging/v3:pkg_cc_proto", ], diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 254e96bb15f2d..ae6c5c6e15c8c 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -13,6 +13,7 @@ #include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/stats/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" @@ -59,12 +60,22 @@ request_type: internal const std::string internal_request_with_json_file_config = R"EOF( request_type: internal - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" + )EOF"; + +const std::string internal_request_with_yaml_file_with_reload_config = R"EOF( + request_type: internal + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" + watched_directory: + path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" )EOF"; const std::string internal_request_with_yaml_file_config = R"EOF( request_type: internal - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; const std::string external_request_config = R"EOF( @@ -77,12 +88,14 @@ request_type: external const std::string external_request_with_json_file_config = R"EOF( request_type: external - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" )EOF"; const std::string external_request_with_yaml_file_config = R"EOF( request_type: external - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" )EOF"; const std::string both_request_config = R"EOF( @@ -98,12 +111,14 @@ request_type: both const std::string both_request_with_json_file_config = R"EOF( request_type: both -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" )EOF"; const std::string both_request_with_yaml_file_config = R"EOF( request_type: both -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" )EOF"; const std::string internal_request_with_header_config = R"EOF( @@ -120,14 +135,16 @@ const std::string internal_request_with_header_with_json_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_header_with_yaml_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string internal_request_with_replace_header_config = R"EOF( @@ -146,7 +163,8 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: SANITIZE -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_replace_header_with_yaml_file_config = R"EOF( @@ -154,7 +172,8 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: SANITIZE -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string internal_request_with_append_or_add_config = R"EOF( @@ -173,7 +192,8 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: APPEND_IF_EXISTS_OR_ADD -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_append_or_add_with_yaml_file_config = R"EOF( @@ -181,7 +201,8 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: APPEND_IF_EXISTS_OR_ADD -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string duplicate_request_config = R"EOF( @@ -197,12 +218,14 @@ request_type: both const std::string duplicate_request_with_json_file_config = R"EOF( request_type: both -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json" )EOF"; const std::string duplicate_request_with_yaml_file_config = R"EOF( request_type: both -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml" )EOF"; const std::string ipv6_config = R"EOF( @@ -213,34 +236,33 @@ const std::string ipv6_config = R"EOF( )EOF"; const std::string ipv6_with_yaml_file_config = R"EOF( -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml" )EOF"; const std::string ipv6_with_json_file_config = R"EOF( -ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json" +ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json" )EOF"; } // namespace -class IpTaggingFilterTest : public ::testing::TestWithParam { +class IpTaggingFilterTest : public Event::TestUsingSimulatedTime, + public ::testing::TestWithParam { public: - IpTaggingFilterTest() : api_(Api::createApiForTest()) { + IpTaggingFilterTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) { ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) .WillByDefault(Return(true)); - EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([&] { - Filesystem::MockWatcher* mock_watcher = new NiceMock(); - EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly(Return(absl::OkStatus())); - return mock_watcher; - })); } + void initializeFilter(const std::string& yaml, absl::optional expected_error = absl::nullopt) { envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); auto config_or = IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, *stats_.rootScope(), - runtime_, *api_, dispatcher_, validation_visitor_); + runtime_, *api_, tls_, *dispatcher_, validation_visitor_); if (expected_error.has_value()) { EXPECT_FALSE(config_or.ok()); EXPECT_EQ(expected_error.value(), absl::StrCat(config_or.status())); @@ -261,7 +283,8 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { std::unique_ptr singleton_manager_ = std::make_unique(); Api::ApiPtr api_; - Event::MockDispatcher dispatcher_; + Event::DispatcherPtr dispatcher_; + NiceMock tls_; NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; std::unique_ptr filter_; @@ -279,7 +302,7 @@ request_type: internal action: SANITIZE )EOF"; initializeFilter(config_yaml, "INVALID_ARGUMENT: HTTP IP Tagging Filter requires either ip_tags " - "or ip_tags_path to be specified."); + "or ip_tags_datasource to be specified."); } TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { @@ -292,19 +315,22 @@ request_type: internal - ip_tag_name: internal_request ip_list: - {address_prefix: 1.2.3.5, prefix_len: 32} -ip_tags_path: /test/tags.yaml +ip_tags_datasource: + filename: /test/tags.yaml )EOF"; - initializeFilter(config_yaml, - "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_path can be configured."); + initializeFilter( + config_yaml, + "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_datasource can be configured."); } TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { const std::string config_yaml = R"EOF( request_type: internal -ip_tags_path: /test/tags.csv +ip_tags_datasource: + filename: /test/tags.csv )EOF"; - initializeFilter(config_yaml, - "INVALID_ARGUMENT: Unsupported file format, unable to parse ip tags from file."); + initializeFilter(config_yaml, "INVALID_ARGUMENT: Unsupported file format, unable to parse ip " + "tags from file /test/tags.csv"); } TEST_F(IpTaggingFilterTest, InvalidCidr) { @@ -326,16 +352,16 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config1); absl::StatusOr config1_result = IpTaggingFilterConfig::create( - proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, - dispatcher_, validation_visitor_); + proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, + *dispatcher_, validation_visitor_); EXPECT_TRUE(config1_result.ok()); auto config1 = config1_result.value(); envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config2); absl::StatusOr config2_result = IpTaggingFilterConfig::create( - proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, - dispatcher_, validation_visitor_); + proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, + *dispatcher_, validation_visitor_); EXPECT_TRUE(config2_result.ok()); auto config2 = config2_result.value(); auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); @@ -353,16 +379,16 @@ TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config1); absl::StatusOr config1_result = IpTaggingFilterConfig::create( - proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, - dispatcher_, validation_visitor_); + proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, + *dispatcher_, validation_visitor_); EXPECT_TRUE(config1_result.ok()); auto config1 = config1_result.value(); envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; TestUtility::loadFromYaml(TestEnvironment::substitute(external_request_with_json_file_config), proto_config2); absl::StatusOr config2_result = IpTaggingFilterConfig::create( - proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, - dispatcher_, validation_visitor_); + proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, + *dispatcher_, validation_visitor_); EXPECT_TRUE(config2_result.ok()); auto config2 = config2_result.value(); auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); @@ -786,230 +812,204 @@ INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, internal_request_with_json_file_config, internal_request_with_yaml_file_config})); -struct IpTagsFileReloadTestCase { - - IpTagsFileReloadTestCase() = default; - IpTagsFileReloadTestCase(const std::string& yaml_config, FilterRequestType request_type, - const std::string& remote_address, - const std::string& source_ip_tags_file_path, - const std::string& reloaded_ip_tags_file_path, - const std::string& hit_counter_prefix, - const std::vector& hit_counter_updated_prefixes, - const Http::TestRequestHeaderMapImpl tagged_headers, - const Http::TestRequestHeaderMapImpl not_tagged_headers) - : yaml_config_(yaml_config), request_type_(request_type), remote_address_(remote_address), - source_ip_tags_file_path_(source_ip_tags_file_path), - reloaded_ip_tags_file_path_(reloaded_ip_tags_file_path), - hit_counter_prefix_(hit_counter_prefix), tagged_headers_(tagged_headers), - not_tagged_headers_(not_tagged_headers) { - hit_counter_updated_prefixes_.reserve(hit_counter_updated_prefixes.size()); - for (auto prefix : hit_counter_updated_prefixes) { - hit_counter_updated_prefixes_.push_back(prefix); - } - } - IpTagsFileReloadTestCase(const IpTagsFileReloadTestCase& rhs) = default; - - std::string yaml_config_; - FilterRequestType request_type_; - std::string remote_address_; - std::string source_ip_tags_file_path_; - std::string reloaded_ip_tags_file_path_; - std::string hit_counter_prefix_; - std::vector hit_counter_updated_prefixes_; - Http::TestRequestHeaderMapImpl tagged_headers_; - Http::TestRequestHeaderMapImpl not_tagged_headers_; -}; +TEST_F(IpTaggingFilterTest, InternalRequestWithReload) { + simTime().advanceTimeWait(std::chrono::seconds(1)); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); -class IpTagsFileReloadImplTest : public ::testing::TestWithParam { -public: - IpTagsFileReloadImplTest() : api_(Api::createApiForTest()) { - ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) - .WillByDefault(Return(true)); - } + envoy::config::core::v3::DataSource config; + TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); - static void SetUpTestSuite() {} - - void initializeFilter(const std::string& yaml, - absl::optional& conditional) { - EXPECT_CALL(dispatcher_, createFilesystemWatcher_()) - .WillRepeatedly(Invoke([this, &conditional] { - Filesystem::MockWatcher* mock_watcher = new NiceMock(); - EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly(Invoke([this, &conditional](absl::string_view, uint32_t, - Filesystem::Watcher::OnChangedCb cb) { - { - absl::WriterMutexLock lock(&mutex_); - on_changed_cbs_.reserve(1); - on_changed_cbs_.emplace_back(std::move(cb)); - } - if (conditional.has_value()) { - conditional->setReady(); - } - return absl::OkStatus(); - })); - return mock_watcher; - })); - envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; - TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); - auto config_or = - IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, *stats_.rootScope(), - runtime_, *api_, dispatcher_, validation_visitor_); - EXPECT_TRUE(config_or.ok()); - config_ = std::move(config_or.value()); - filter_ = std::make_unique(config_); - filter_->setDecoderFilterCallbacks(filter_callbacks_); - } + const std::string yaml = + fmt::format(R"EOF( +request_type: internal +ip_tags_refresh_rate: 5s +ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" + )EOF", + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test")); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + )EOF", + true); - ~IpTagsFileReloadImplTest() override { - { - absl::WriterMutexLock lock(&mutex_); - on_changed_cbs_.clear(); - } - if (filter_) { - filter_->onDestroy(); - } - } + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: internal_updated_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + )EOF", + true); + initializeFilter(yaml); + simTime().advanceTimeWait(std::chrono::seconds(1)); - std::unique_ptr singleton_manager_ = - std::make_unique(); - Api::ApiPtr api_; - Event::MockDispatcher dispatcher_; - NiceMock stats_; - IpTaggingFilterConfigSharedPtr config_; - std::unique_ptr filter_; - NiceMock filter_callbacks_; - Buffer::OwnedImpl data_; - NiceMock runtime_; - NiceMock validation_visitor_; - absl::Mutex mutex_; - std::vector on_changed_cbs_ ABSL_GUARDED_BY(mutex_); - absl::optional cb_added_nullopt = absl::nullopt; -}; + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; -TEST_P(IpTagsFileReloadImplTest, IpTagsFileReloaded) { - IpTagsFileReloadTestCase test_case = GetParam(); - auto cb_added_opt = absl::make_optional(); - initializeFilter(test_case.yaml_config_, cb_added_opt); - EXPECT_EQ(test_case.request_type_, config_->requestType()); Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, - counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); - auto request_headers = test_case.tagged_headers_; + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->decodeHeaders(test_case.not_tagged_headers_, false)); - EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); - std::string source_ip_tags_file_path = - TestEnvironment::substitute(test_case.source_ip_tags_file_path_); - std::string reloaded_ip_tags_file_path = - TestEnvironment::substitute(test_case.reloaded_ip_tags_file_path_); - TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); - TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_success"))); - cb_added_opt.value().waitReady(); - { - absl::ReaderMutexLock guard(&mutex_); - EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); - } + + // Check external requests don't get a tag. + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + + // Handle the events if any. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + simTime().advanceTimeWait(std::chrono::seconds(1)); + + // Update the symlink to point to the new file. + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml")); + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + + auto contents = TestEnvironment::readFileToStringForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + // Handle the events if any. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + simTime().advanceTimeWait(std::chrono::seconds(6)); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - for (auto updated_prefix : test_case.hit_counter_updated_prefixes_) { - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, updated_prefix, ".hit"))); - } - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); - request_headers = test_case.tagged_headers_; + + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_success"))); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - for (auto updated_prefix : test_case.hit_counter_updated_prefixes_) { - EXPECT_TRUE(request_headers.get_(Http::Headers::get().EnvoyIpTags).find(updated_prefix) != - std::string::npos); - } - // Clean up modifications to ip tags file names. - TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); - TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); + EXPECT_EQ("internal_updated_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + request_trailers = Http::TestRequestTrailerMapImpl{}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + // Remove the file. + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); } -TEST_P(IpTagsFileReloadImplTest, IpTagsFileNotReloaded) { - IpTagsFileReloadTestCase test_case = GetParam(); - auto cb_added_opt = absl::make_optional(); - initializeFilter(test_case.yaml_config_, cb_added_opt); - EXPECT_EQ(test_case.request_type_, config_->requestType()); +TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { + simTime().advanceTimeWait(std::chrono::seconds(1)); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + + envoy::config::core::v3::DataSource config; + TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); + + const std::string yaml = + fmt::format(R"EOF( +request_type: internal +ip_tags_refresh_rate: 5s +ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" + )EOF", + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test")); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + )EOF", + true); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( +ip_tags + )EOF", + true); + initializeFilter(yaml); + simTime().advanceTimeWait(std::chrono::seconds(1)); + + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + Network::Address::InstanceConstSharedPtr remote_address = - Network::Utility::parseInternetAddressNoThrow(test_case.remote_address_); + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, - counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); - auto request_headers = test_case.tagged_headers_; + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, - filter_->decodeHeaders(test_case.not_tagged_headers_, false)); - EXPECT_FALSE(test_case.not_tagged_headers_.has(Http::Headers::get().EnvoyIpTags)); - std::string source_ip_tags_file_path = - TestEnvironment::substitute(test_case.source_ip_tags_file_path_); - std::string reloaded_ip_tags_file_path = TestEnvironment::substitute( - "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml"); - TestEnvironment::renameFile(source_ip_tags_file_path, source_ip_tags_file_path + "1"); - TestEnvironment::renameFile(reloaded_ip_tags_file_path, source_ip_tags_file_path); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); - cb_added_opt.value().waitReady(); - { - absl::ReaderMutexLock guard(&mutex_); - EXPECT_TRUE(on_changed_cbs_[0](Filesystem::Watcher::Events::MovedTo).ok()); - } - // Current ip tags should be used if reload failed. + + // Check external requests don't get a tag. + request_headers = Http::TestRequestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + + // Handle the events if any. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + simTime().advanceTimeWait(std::chrono::seconds(1)); + + // Update the symlink to point to the new file. + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml")); + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + + auto contents = TestEnvironment::readFileToStringForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + // Handle the events if any. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + simTime().advanceTimeWait(std::chrono::seconds(6)); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, - counter(absl::StrCat(ip_tagging_prefix, test_case.hit_counter_prefix_, ".hit"))); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "total"))); - request_headers = test_case.tagged_headers_; + + EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); - EXPECT_EQ(test_case.hit_counter_prefix_, request_headers.get_(Http::Headers::get().EnvoyIpTags)); - // Clean up modifications to ip tags file names. - TestEnvironment::renameFile(source_ip_tags_file_path, reloaded_ip_tags_file_path); - TestEnvironment::renameFile(source_ip_tags_file_path + "1", source_ip_tags_file_path); -} + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); -struct IpTagsFileReloadTestCase ip_tags_file_reload_test_cases[] = { - {internal_request_with_yaml_file_config, - FilterRequestType::INTERNAL, - "1.2.3.5", - "{{ test_rundir " - "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml", - "{{ test_rundir " - "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml", - "internal_request", - {"internal_updated_request"}, - {{"x-envoy-internal", "true"}}, - {}}, - {external_request_with_json_file_config, - FilterRequestType::EXTERNAL, - "1.2.3.4", - "{{ test_rundir " - "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json", - "{{ test_rundir " - "}}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_external_request.json", - "external_request", - {"external_updated_request", "gcp_zone_a_request"}, - {}, - {{"x-envoy-internal", "true"}}}, -}; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + request_trailers = Http::TestRequestTrailerMapImpl{}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); -INSTANTIATE_TEST_SUITE_P(TestName, IpTagsFileReloadImplTest, - ::testing::ValuesIn(ip_tags_file_reload_test_cases)); + // Remove the file. + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); +} } // namespace } // namespace IpTagging From b58693546564be647426ee87c078e35e34b535cc Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 26 May 2025 12:03:49 +0200 Subject: [PATCH 23/76] Fix format Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/v3/ip_tagging.proto | 2 +- .../http/ip_tagging/test_data/invalid_tags.yaml | 2 +- .../http/ip_tagging/test_data/ip_tags_both.yaml | 12 ++++++------ .../test_data/ip_tags_external_request.yaml | 6 +++--- .../test_data/ip_tags_internal_request.yaml | 6 +++--- .../test_data/ip_tags_updated_internal_request.yaml | 6 +++--- .../test_data/ip_tags_with_duplicate_request.yaml | 12 ++++++------ .../ip_tagging/test_data/ip_tags_with_header.yaml | 6 +++--- .../http/ip_tagging/test_data/ipv6_request.yaml | 6 +++--- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 0c44c0b4488a6..ccb849876a7f6 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -97,7 +97,7 @@ message IPTagging { // When :ref:`ip_tags ` is configured // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. - // Deafults to 0, in this case no refresh will be attempted. + // Defaults to 0, in this case no refresh will be attempted. google.protobuf.Duration ip_tags_refresh_rate = 7 [(validate.rules).duration = {gt {}}]; } diff --git a/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml index e3dd56b59bbfe..3b12f20bec6c0 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml @@ -1 +1 @@ -ip_tags \ No newline at end of file +ip_tags diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml index 002388277147e..639fce3146c6d 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml @@ -1,7 +1,7 @@ ip_tags: - - ip_tag_name: external_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} +- ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +- ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml index 741868809deee..fbea0dc37e732 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: external_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} +- ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml index 5ae7d7868ab96..f535ae03a6465 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} +- ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml index 9cc955bf6dd67..178179d3675c1 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_updated_internal_request.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: internal_updated_request - ip_list: - - {address_prefix: 1.2.3.5, prefix_len: 32} +- ip_tag_name: internal_updated_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml index c3679cd06da19..340c59fc6af0f 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml @@ -1,7 +1,7 @@ ip_tags: - - ip_tag_name: duplicate_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} - - ip_tag_name: internal_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} +- ip_tag_name: duplicate_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} +- ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml index cba803e6dd72b..9d0df491dc1ea 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: internal_request_with_optional_header - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} +- ip_tag_name: internal_request_with_optional_header + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} diff --git a/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml index d74fdb1ccc60d..9eb84ff19abde 100644 --- a/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml +++ b/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml @@ -1,4 +1,4 @@ ip_tags: - - ip_tag_name: ipv6_request - ip_list: - - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} +- ip_tag_name: ipv6_request + ip_list: + - {address_prefix: 2001:abcd:ef01:2345:6789:abcd:ef01:234, prefix_len: 64} From 617ff51cd7b289429a18af72866ba8ebb7d4ded0 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 26 May 2025 12:59:02 +0200 Subject: [PATCH 24/76] Fix api check Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index ccb849876a7f6..14f6034030afa 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -2,10 +2,11 @@ syntax = "proto3"; package envoy.extensions.filters.http.ip_tagging.v3; -import "google/protobuf/duration.proto"; import "envoy/config/core/v3/base.proto"; import "envoy/data/ip_tagging/v3/ip_tagging.proto"; +import "google/protobuf/duration.proto"; + import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; import "validate/validate.proto"; @@ -94,10 +95,8 @@ message IPTagging { // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. config.core.v3.DataSource ip_tags_datasource = 6; - // When :ref:`ip_tags ` is configured // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. // Defaults to 0, in this case no refresh will be attempted. - google.protobuf.Duration ip_tags_refresh_rate = 7 - [(validate.rules).duration = {gt {}}]; + google.protobuf.Duration ip_tags_refresh_rate = 7 [(validate.rules).duration = {gt {}}]; } From f28f7a51f5a01d93f946f70e4eb97551cf3b80fe Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 27 May 2025 12:27:06 +0200 Subject: [PATCH 25/76] Fix docs ci check Signed-off-by: Kateryna Nezdolii --- api/envoy/data/ip_tagging/v3/ip_tagging.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/envoy/data/ip_tagging/v3/ip_tagging.proto b/api/envoy/data/ip_tagging/v3/ip_tagging.proto index 5cc630fe2cf55..6b2d18d834e64 100644 --- a/api/envoy/data/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/data/ip_tagging/v3/ip_tagging.proto @@ -6,7 +6,6 @@ import "envoy/config/core/v3/address.proto"; import "udpa/annotations/status.proto"; import "udpa/annotations/versioning.proto"; -import "validate/validate.proto"; option java_package = "io.envoyproxy.envoy.data.ip_tagging.v3"; option java_outer_classname = "IpTaggingProto"; @@ -14,6 +13,8 @@ option java_multiple_files = true; option go_package = "github.com/envoyproxy/go-control-plane/envoy/data/ip_tagging/v3;ip_taggingv3"; option (udpa.annotations.file_status).package_version_status = ACTIVE; +// [#protodoc-title: Ip Tagging Data] + // Supplies the IP tag name and the IP address subnets. message IPTag { option (udpa.annotations.versioning).previous_message_type = From cf1c047893a41ec48fe00c7ccc21001f37e8a9b8 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 27 May 2025 15:48:18 +0200 Subject: [PATCH 26/76] Fix docs Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 2 +- .../root/configuration/http/http_filters/ip_tagging_filter.rst | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 14f6034030afa..e75f3beac55ce 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -80,7 +80,7 @@ message IPTagging { // The set of IP tags for the filter. // Only one of :ref:`ip_tags ` - // or :ref:`ip_tags_path ` + // or :ref:`ip_tags_datasource ` // can be set for the Ip Tagging filter. repeated data.ip_tagging.v3.IPTag ip_tags = 4; diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index f618ba6ff9521..746aecd7665fb 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -24,10 +24,9 @@ G. Karlsson. Ip tags can either be parsed from :ref:`ip_tags ` api field or can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. -Ip tags will be dynamically realoded if :ref:`ip_tags_datasource.watched_directory ` +Ip tags will be dynamically realoded if *watched_directory* is configured for :ref:`ip_tags_datasource ` is configured and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. - Configuration ------------- * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging``. From 0ad588daaece4bad893e260e23efbd24b97f604f Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 27 May 2025 23:12:00 +0200 Subject: [PATCH 27/76] Fix docs Signed-off-by: Kateryna Nezdolii --- api/envoy/data/ip_tagging/v3/ip_tagging.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/api/envoy/data/ip_tagging/v3/ip_tagging.proto b/api/envoy/data/ip_tagging/v3/ip_tagging.proto index 6b2d18d834e64..6b68b2db4fcd7 100644 --- a/api/envoy/data/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/data/ip_tagging/v3/ip_tagging.proto @@ -14,6 +14,7 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/data/ip_taggin option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Ip Tagging Data] +// :ref:`Ip Tagging Filter config overview <_config_http_filters_ip_tagging>`. // Supplies the IP tag name and the IP address subnets. message IPTag { From ef2e170ca135ec0de15b95dc724f1365271949d6 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 28 May 2025 10:47:44 +0200 Subject: [PATCH 28/76] Fix docs Signed-off-by: Kateryna Nezdolii --- api/envoy/data/ip_tagging/v3/ip_tagging.proto | 2 +- docs/root/api-v3/data/ip_tagging/ip_tagging.rst | 8 ++++++++ .../configuration/http/http_filters/ip_tagging_filter.rst | 6 ++++-- 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 docs/root/api-v3/data/ip_tagging/ip_tagging.rst diff --git a/api/envoy/data/ip_tagging/v3/ip_tagging.proto b/api/envoy/data/ip_tagging/v3/ip_tagging.proto index 6b68b2db4fcd7..471eed4b00e08 100644 --- a/api/envoy/data/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/data/ip_tagging/v3/ip_tagging.proto @@ -14,7 +14,7 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/data/ip_taggin option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Ip Tagging Data] -// :ref:`Ip Tagging Filter config overview <_config_http_filters_ip_tagging>`. +// :ref:`Ip Tagging Filter config overview `. // Supplies the IP tag name and the IP address subnets. message IPTag { diff --git a/docs/root/api-v3/data/ip_tagging/ip_tagging.rst b/docs/root/api-v3/data/ip_tagging/ip_tagging.rst new file mode 100644 index 0000000000000..8fffc1bb3fefe --- /dev/null +++ b/docs/root/api-v3/data/ip_tagging/ip_tagging.rst @@ -0,0 +1,8 @@ +Ip Tagging +=== + +.. toctree:: + :glob: + :maxdepth: 2 + + v3/* diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index 746aecd7665fb..d0aa98dc57c81 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -24,8 +24,10 @@ G. Karlsson. Ip tags can either be parsed from :ref:`ip_tags ` api field or can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. -Ip tags will be dynamically realoded if *watched_directory* is configured for :ref:`ip_tags_datasource ` -is configured and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. +Both :ref:`ip_tags ` and file with *filename* from :ref:`ip_tags_datasource ` +use :ref:`IPTag ` proto format for defining individual ip tags. +Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` +and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. Configuration ------------- From 95b89db0bb64f86f295029cb5c65d34a93d5f125 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 28 May 2025 14:17:17 +0200 Subject: [PATCH 29/76] Fix integration test Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_integration_test.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index c2d815b3e368c..8887b4839c4af 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -30,7 +30,10 @@ const std::string FileBasedIpTaggingConfig = R"EOF( typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging request_type: both - ip_tags_path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + watched_directory: + path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" )EOF"; // Make sure that Envoy starts up with an ip tagging filter. From ac4f9df14927f32015e5f33841228fee39953aeb Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 28 May 2025 15:00:00 +0200 Subject: [PATCH 30/76] Fix toctree inclusion Signed-off-by: Kateryna Nezdolii --- docs/root/api-v3/data/data.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/api-v3/data/data.rst b/docs/root/api-v3/data/data.rst index 2a4a2fea05e17..a2a30425ee6d4 100644 --- a/docs/root/api-v3/data/data.rst +++ b/docs/root/api-v3/data/data.rst @@ -10,3 +10,4 @@ Envoy data core/core dns/dns tap/tap + ip_tagging/ip_tagging From c6ec5eea11f767a0f69427be6e770217f0c4728c Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 2 Jun 2025 12:02:04 +0200 Subject: [PATCH 31/76] Update docs/root/configuration/http/http_filters/ip_tagging_filter.rst Co-authored-by: Adi (Suissa) Peleg Signed-off-by: Kateryna Nezdolii --- docs/root/configuration/http/http_filters/ip_tagging_filter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index d0aa98dc57c81..489646091eddc 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -22,7 +22,7 @@ described in the paper `IP-address lookup using LC-tries `_ by S. Nilsson and G. Karlsson. -Ip tags can either be parsed from :ref:`ip_tags ` api field or +IP tags can either be provided directly using the :ref:`ip_tags ` api field or can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. Both :ref:`ip_tags ` and file with *filename* from :ref:`ip_tags_datasource ` use :ref:`IPTag ` proto format for defining individual ip tags. From 2c788399309a183021eb9f97a9a61fbcb36442b3 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 2 Jun 2025 14:53:08 +0200 Subject: [PATCH 32/76] Address some of the feedback Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/v3/ip_tagging.proto | 25 ++-- source/extensions/extensions_build_config.bzl | 52 +++---- .../http/ip_tagging/ip_tagging_filter.cc | 26 ++-- .../http/ip_tagging/ip_tagging_filter.h | 35 ++++- .../http/ip_tagging/ip_tagging_filter_test.cc | 128 ++++++++++-------- .../ip_tagging/ip_tagging_integration_test.cc | 9 +- 6 files changed, 169 insertions(+), 106 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index e75f3beac55ce..82dd38c4479e7 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -75,6 +75,20 @@ message IPTagging { HeaderAction action = 2; } + // Common configuration for file based ip tags. + message IpTagsFileProvider { + // Data source from which to retrieve ip tags. + // Only filename based data source is currently supported for ip tags. + // When using this data source, if a ``watched_directory`` is provided, the ip tags file will be re-read when a file move is detected. + // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. + config.core.v3.DataSource ip_tags_datasource = 1; + + // When :ref:`ip_tags ` is configured + // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. + // Defaults to 0, in this case no refresh will be attempted. + google.protobuf.Duration ip_tags_refresh_rate = 2 [(validate.rules).duration = {gt {}}]; + } + // The type of request the filter should apply to. RequestType request_type = 1 [(validate.rules).enum = {defined_only: true}]; @@ -89,14 +103,5 @@ message IPTagging { // If left unspecified, the tags will be appended to the ``x-envoy-ip-tags`` header. IpTagHeader ip_tag_header = 5; - // Data source from which to retrieve ip tags. - // Only filename based data source is currently supported for ip tags. - // When using this data source, if a ``watched_directory`` is provided, the ip tags file will be re-read when a file move is detected. - // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. - config.core.v3.DataSource ip_tags_datasource = 6; - - // When :ref:`ip_tags ` is configured - // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. - // Defaults to 0, in this case no refresh will be attempted. - google.protobuf.Duration ip_tags_refresh_rate = 7 [(validate.rules).duration = {gt {}}]; + IpTagsFileProvider ip_tags_file_provider = 6; } diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 2266b6f5d0316..c16d6942a3dad 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -32,12 +32,12 @@ EXTENSIONS = { # Compression # - "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", - "envoy.compression.gzip.decompressor": "//source/extensions/compression/gzip/decompressor:config", - "envoy.compression.brotli.compressor": "//source/extensions/compression/brotli/compressor:config", - "envoy.compression.brotli.decompressor": "//source/extensions/compression/brotli/decompressor:config", - "envoy.compression.zstd.compressor": "//source/extensions/compression/zstd/compressor:config", - "envoy.compression.zstd.decompressor": "//source/extensions/compression/zstd/decompressor:config", + # "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", + # "envoy.compression.gzip.decompressor": "//source/extensions/compression/gzip/decompressor:config", + # "envoy.compression.brotli.compressor": "//source/extensions/compression/brotli/compressor:config", + # "envoy.compression.brotli.decompressor": "//source/extensions/compression/brotli/decompressor:config", + # "envoy.compression.zstd.compressor": "//source/extensions/compression/zstd/compressor:config", + # "envoy.compression.zstd.decompressor": "//source/extensions/compression/zstd/decompressor:config", # # Config validators @@ -305,15 +305,15 @@ EXTENSIONS = { # Transport sockets # - "envoy.transport_sockets.alts": "//source/extensions/transport_sockets/alts:config", - "envoy.transport_sockets.http_11_proxy": "//source/extensions/transport_sockets/http_11_proxy:upstream_config", - "envoy.transport_sockets.upstream_proxy_protocol": "//source/extensions/transport_sockets/proxy_protocol:upstream_config", - "envoy.transport_sockets.raw_buffer": "//source/extensions/transport_sockets/raw_buffer:config", - "envoy.transport_sockets.tap": "//source/extensions/transport_sockets/tap:config", - "envoy.transport_sockets.starttls": "//source/extensions/transport_sockets/starttls:config", - "envoy.transport_sockets.tcp_stats": "//source/extensions/transport_sockets/tcp_stats:config", - "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", - "envoy.transport_sockets.internal_upstream": "//source/extensions/transport_sockets/internal_upstream:config", + # "envoy.transport_sockets.alts": "//source/extensions/transport_sockets/alts:config", + # "envoy.transport_sockets.http_11_proxy": "//source/extensions/transport_sockets/http_11_proxy:upstream_config", + # "envoy.transport_sockets.upstream_proxy_protocol": "//source/extensions/transport_sockets/proxy_protocol:upstream_config", + # "envoy.transport_sockets.raw_buffer": "//source/extensions/transport_sockets/raw_buffer:config", + # "envoy.transport_sockets.tap": "//source/extensions/transport_sockets/tap:config", + # "envoy.transport_sockets.starttls": "//source/extensions/transport_sockets/starttls:config", + # "envoy.transport_sockets.tcp_stats": "//source/extensions/transport_sockets/tcp_stats:config", + # "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", + # "envoy.transport_sockets.internal_upstream": "//source/extensions/transport_sockets/internal_upstream:config", # # Retry host predicates @@ -376,14 +376,14 @@ EXTENSIONS = { # IO socket # - "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", - "envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", + #"envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", + #"envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", # # TLS peer certification validators # - "envoy.tls.cert_validator.spiffe": "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config", + #"envoy.tls.cert_validator.spiffe": "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config", # # HTTP header formatters @@ -428,14 +428,14 @@ EXTENSIONS = { # QUIC extensions # - "envoy.quic.deterministic_connection_id_generator": "//source/extensions/quic/connection_id_generator/deterministic:envoy_deterministic_connection_id_generator_config", - "envoy.quic.connection_id_generator.quic_lb": "//source/extensions/quic/connection_id_generator/quic_lb:quic_lb_config", - "envoy.quic.crypto_stream.server.quiche": "//source/extensions/quic/crypto_stream:envoy_quic_default_crypto_server_stream", - "envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source", - "envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config", - "envoy.quic.server_preferred_address.datasource": "//source/extensions/quic/server_preferred_address:datasource_server_preferred_address_config_factory_config", - "envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor/basic:envoy_quic_connection_debug_visitor_basic", - "envoy.quic.connection_debug_visitor.quic_stats": "//source/extensions/quic/connection_debug_visitor/quic_stats:config", + # "envoy.quic.deterministic_connection_id_generator": "//source/extensions/quic/connection_id_generator/deterministic:envoy_deterministic_connection_id_generator_config", + # "envoy.quic.connection_id_generator.quic_lb": "//source/extensions/quic/connection_id_generator/quic_lb:quic_lb_config", + # "envoy.quic.crypto_stream.server.quiche": "//source/extensions/quic/crypto_stream:envoy_quic_default_crypto_server_stream", + # "envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source", + # "envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config", + # "envoy.quic.server_preferred_address.datasource": "//source/extensions/quic/server_preferred_address:datasource_server_preferred_address_config_factory_config", + # "envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor/basic:envoy_quic_connection_debug_visitor_basic", + # "envoy.quic.connection_debug_visitor.quic_stats": "//source/extensions/quic/connection_debug_visitor/quic_stats:config", # # UDP packet writers diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 1ae4dcf71ebb8..84ea51cdb8fcb 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -202,7 +202,6 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( ip_tag_header_action_(config.has_ip_tag_header() ? config.ip_tag_header().action() : HeaderAction::IPTagging_IpTagHeader_HeaderAction_SANITIZE), - ip_tags_path_(config.ip_tags_datasource().filename()), ip_tags_registry_(singleton_manager.getTyped( SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), [] { return std::make_shared(); })), @@ -213,25 +212,32 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( // to be implemented. // TODO(ccaraman): Remove size check once file system support is implemented. // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. - if (config.ip_tags().empty() && !config.has_ip_tags_datasource()) { + if (config.ip_tags().empty() && !config.has_ip_tags_file_provider()) { creation_status = absl::InvalidArgumentError( - "HTTP IP Tagging Filter requires either ip_tags or ip_tags_datasource to be specified."); + "HTTP IP Tagging Filter requires either ip_tags or ip_tags_file_provider to be specified."); } - if (!config.ip_tags().empty() && config.has_ip_tags_datasource()) { - creation_status = - absl::InvalidArgumentError("Only one of ip_tags or ip_tags_datasource can be configured."); + if (!config.ip_tags().empty() && config.has_ip_tags_file_provider()) { + creation_status = absl::InvalidArgumentError( + "Only one of ip_tags or ip_tags_file_provider can be configured."); } RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (!config.ip_tags().empty()) { trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); } else { - auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config, ip_tags_refresh_rate, 0); + if (!config.ip_tags_file_provider().has_ip_tags_datasource()) { + creation_status = absl::InvalidArgumentError( + "ip_tags_file_provider requires a valid ip_tags_datasource to be configured."); + return; + } + auto ip_tags_refresh_interval_ms = + PROTOBUF_GET_MS_OR_DEFAULT(config.ip_tags_file_provider(), ip_tags_refresh_rate, 0); provider_ = ip_tags_registry_->get( - config.ip_tags_datasource(), tags_loader_, ip_tags_refresh_interval_ms, - [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, tls, - dispatcher, ip_tags_registry_, creation_status); + config.ip_tags_file_provider().ip_tags_datasource(), tags_loader_, + ip_tags_refresh_interval_ms, [this]() { incIpTagsReloadSuccess(); }, + [this]() { incIpTagsReloadError(); }, api, tls, dispatcher, ip_tags_registry_, + creation_status); RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (provider_ && provider_->ipTags()) { trie_ = provider_->ipTags(); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 06b38c66b9cfb..51ea8f84b06ca 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -27,17 +27,40 @@ namespace IpTagging { using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; using LcTrieSharedPtr = std::shared_ptr>; +/** + * This class is responsible for loading and parsing of ip tags (both inline and file based) + * as well as for periodic refresh of ip tags (file based). + */ class IpTagsLoader { public: IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set); + /** + * Loads file based ip tags from a datasource and parses them into a trie structure. + * @param ip_tags_datasource file based datasource to load ip tags from. + * @param dispatcher The dispatcher for the thread used by a datasource provider. + * @param tls The thread local slot allocator used by a datasource provider. + * @param creation_status This status will be populated with error if loading fails. + * @return Valid LcTrieSharedPtr if loading succeeded or nullptr otherwise. + */ LcTrieSharedPtr loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, absl::Status& creation_status); + /** + * Performs periodic refresh of file based ip tags via datasource. + * @param refresh_status This status will be populated with error if refresh fails. + * @return Valid LcTrieSharedPtr if loading succeeded or nullptr otherwise. + */ LcTrieSharedPtr refreshTags(absl::Status& refresh_status); + /** + * Parses ip tags in a proto format into a trie structure. + * @param ip_tags Collection of ip tags in proto format. + * @param creation_status This status will be populated with error if parsing fails. + * @return Valid LcTrieSharedPtr if parsing succeeded or nullptr otherwise. + */ LcTrieSharedPtr parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags, absl::Status& creation_status); @@ -53,6 +76,10 @@ class IpTagsLoader { using IpTagsReloadSuccessCb = std::function; using IpTagsReloadErrorCb = std::function; +/** + * This class owns ip tags trie structure for a configured abosulte file path and provides access to + * the ip tags data. It also performs periodic refresh of ip tags data. + */ class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, @@ -63,6 +90,11 @@ class IpTagsProvider : public Logger::Loggable { ~IpTagsProvider(); + /** + * Getter method for ip tags trie structure which either returns the current version of ip tags + * data or performs a reload of data and returns the most recent version of ip tags data. + * @return Valid LcTrieSharedPtr or nullptr if reload has failed. + */ LcTrieSharedPtr ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); private: @@ -102,6 +134,8 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { absl::Mutex mu_; // Each provider stores shared_ptrs to this singleton, which keeps the singleton // from being destroyed unless it's no longer keeping track of any providers. + // Each entry in this map consists of a key (hash of an absolute file path to ip tags file) + // and and value (instance of `IpTagsProvider` that owns ip tags data). absl::flat_hash_map> ip_tags_registry_ ABSL_GUARDED_BY(mu_); }; @@ -192,7 +226,6 @@ class IpTaggingFilterConfig { const Http::LowerCaseString ip_tag_header_; // An empty string indicates that no ip_tag_header is set. const HeaderAction ip_tag_header_action_; - const std::string ip_tags_path_; // A shared_ptr to keep the ip tags registry singleton alive as long as any of its trie structures // are in use. const std::shared_ptr ip_tags_registry_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index ae6c5c6e15c8c..473ea543908f7 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -40,9 +40,6 @@ class IpTaggingFilterConfigPeer { ipTagsRegistry(const IpTaggingFilterConfig& filter_config) { return filter_config.ip_tags_registry_; } - static const std::string& ipTagsPath(const IpTaggingFilterConfig& filter_config) { - return filter_config.ip_tags_path_; - } }; namespace { @@ -60,22 +57,25 @@ request_type: internal const std::string internal_request_with_json_file_config = R"EOF( request_type: internal - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.json" )EOF"; const std::string internal_request_with_yaml_file_with_reload_config = R"EOF( request_type: internal - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" - watched_directory: - path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" + watched_directory: + path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" )EOF"; const std::string internal_request_with_yaml_file_config = R"EOF( request_type: internal - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" )EOF"; const std::string external_request_config = R"EOF( @@ -88,14 +88,16 @@ request_type: external const std::string external_request_with_json_file_config = R"EOF( request_type: external - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.json" )EOF"; const std::string external_request_with_yaml_file_config = R"EOF( request_type: external - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" )EOF"; const std::string both_request_config = R"EOF( @@ -111,14 +113,16 @@ request_type: both const std::string both_request_with_json_file_config = R"EOF( request_type: both -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.json" )EOF"; const std::string both_request_with_yaml_file_config = R"EOF( request_type: both -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_both.yaml" )EOF"; const std::string internal_request_with_header_config = R"EOF( @@ -135,16 +139,18 @@ const std::string internal_request_with_header_with_json_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_header_with_yaml_file_config = R"EOF( request_type: internal ip_tag_header: header: x-envoy-optional-header -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string internal_request_with_replace_header_config = R"EOF( @@ -163,8 +169,9 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: SANITIZE -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_replace_header_with_yaml_file_config = R"EOF( @@ -172,8 +179,9 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: SANITIZE -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string internal_request_with_append_or_add_config = R"EOF( @@ -192,8 +200,9 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: APPEND_IF_EXISTS_OR_ADD -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.json" )EOF"; const std::string internal_request_with_append_or_add_with_yaml_file_config = R"EOF( @@ -201,8 +210,9 @@ request_type: internal ip_tag_header: header: x-envoy-optional-header action: APPEND_IF_EXISTS_OR_ADD -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_header.yaml" )EOF"; const std::string duplicate_request_config = R"EOF( @@ -218,14 +228,16 @@ request_type: both const std::string duplicate_request_with_json_file_config = R"EOF( request_type: both -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.json" )EOF"; const std::string duplicate_request_with_yaml_file_config = R"EOF( request_type: both -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_with_duplicate_request.yaml" )EOF"; const std::string ipv6_config = R"EOF( @@ -236,13 +248,15 @@ const std::string ipv6_config = R"EOF( )EOF"; const std::string ipv6_with_yaml_file_config = R"EOF( -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.yaml" )EOF"; const std::string ipv6_with_json_file_config = R"EOF( -ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json" +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ipv6_request.json" )EOF"; } // namespace @@ -302,7 +316,7 @@ request_type: internal action: SANITIZE )EOF"; initializeFilter(config_yaml, "INVALID_ARGUMENT: HTTP IP Tagging Filter requires either ip_tags " - "or ip_tags_datasource to be specified."); + "or ip_tags_file_provider to be specified."); } TEST_F(IpTaggingFilterTest, BothIpTagsAndIpTagsFileConfigured) { @@ -315,19 +329,21 @@ request_type: internal - ip_tag_name: internal_request ip_list: - {address_prefix: 1.2.3.5, prefix_len: 32} -ip_tags_datasource: - filename: /test/tags.yaml +ip_tags_file_provider: + ip_tags_datasource: + filename: /test/tags.yaml )EOF"; initializeFilter( config_yaml, - "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_datasource can be configured."); + "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_file_provider can be configured."); } TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { const std::string config_yaml = R"EOF( request_type: internal -ip_tags_datasource: - filename: /test/tags.csv +ip_tags_file_provider: + ip_tags_datasource: + filename: /test/tags.csv )EOF"; initializeFilter(config_yaml, "INVALID_ARGUMENT: Unsupported file format, unable to parse ip " "tags from file /test/tags.csv"); @@ -823,11 +839,12 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithReload) { const std::string yaml = fmt::format(R"EOF( request_type: internal -ip_tags_refresh_rate: 5s -ip_tags_datasource: - filename: "{}" - watched_directory: - path: "{}" +ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" )EOF", TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test")); @@ -923,11 +940,12 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { const std::string yaml = fmt::format(R"EOF( request_type: internal -ip_tags_refresh_rate: 5s -ip_tags_datasource: - filename: "{}" - watched_directory: - path: "{}" +ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" )EOF", TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test")); diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index 8887b4839c4af..182fb9977676c 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -30,10 +30,11 @@ const std::string FileBasedIpTaggingConfig = R"EOF( typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging request_type: both - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" - watched_directory: - path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + watched_directory: + path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" )EOF"; // Make sure that Envoy starts up with an ip tagging filter. From c0d8349a839ac083188b579b14456ff1afea6f04 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 3 Jun 2025 15:39:28 +0200 Subject: [PATCH 33/76] Review comments Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 71 ++++++++++--------- .../http/ip_tagging/ip_tagging_filter.h | 19 +++-- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 84ea51cdb8fcb..74da1ce511e69 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -33,7 +33,11 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag ip_tags_datasource.has_watched_directory()) { needs_refresh_ = true; } - tags_ = tags_loader_.loadTags(ip_tags_datasource, dispatcher, tls, creation_status); + auto tags_or_error = tags_loader_.loadTags(ip_tags_datasource, dispatcher, tls); + creation_status = tags_or_error.status(); + if (tags_or_error.status().ok()) { + tags_ = tags_or_error.value(); + } } IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); }; @@ -43,14 +47,14 @@ LcTrieSharedPtr IpTagsProvider::ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { const auto current_time = time_source_.monotonicTime(); if (std::chrono::duration_cast(current_time - last_reloaded_time_) > ip_tags_refresh_interval_ms_) { - absl::Status refresh_status = absl::OkStatus(); - auto new_tags = tags_loader_.refreshTags(refresh_status); - if (new_tags) { + auto new_tags_or_error = tags_loader_.refreshTags(); + if (new_tags_or_error.status().ok()) { absl::MutexLock lock(&ip_tags_mutex_); - tags_ = new_tags; + tags_ = new_tags_or_error.value(); reload_success_cb_(); } else { - ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", refresh_status.message()); + ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", + new_tags_or_error.status().message()); reload_error_cb_(); } last_reloaded_time_ = current_time; @@ -92,33 +96,30 @@ IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& va Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} -LcTrieSharedPtr +absl::StatusOr IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, - Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, - absl::Status& creation_status) { + Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls) { if (!ip_tags_datasource.filename().empty()) { if (!absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Yaml) && !absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Json)) { - creation_status = - absl::InvalidArgumentError("Unsupported file format, unable to parse ip tags from file " + - ip_tags_datasource.filename()); - return nullptr; + return absl::InvalidArgumentError( + "Unsupported file format, unable to parse ip tags from file " + + ip_tags_datasource.filename()); } auto provider_or_error = Config::DataSource::DataSourceProvider::create( ip_tags_datasource, dispatcher, tls, api_, false, 0); - if (!provider_or_error.ok()) { - creation_status = absl::InvalidArgumentError( + if (!provider_or_error.status().ok()) { + return absl::InvalidArgumentError( fmt::format("unable to create data source '{}'", provider_or_error.status().message())); - return nullptr; } data_source_provider_ = std::move(provider_or_error.value()); ip_tags_path_ = ip_tags_datasource.filename(); - return refreshTags(creation_status); + return refreshTags(); } - return nullptr; + return absl::InvalidArgumentError("Cannot load tags from empty filename in datasource."); } -LcTrieSharedPtr IpTagsLoader::refreshTags(absl::Status& refresh_status) { +absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { IpTagFileProto ip_tags_proto; if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { @@ -127,28 +128,25 @@ LcTrieSharedPtr IpTagsLoader::refreshTags(absl::Status& refresh_status) { validation_visitor_); } END_TRY catch (EnvoyException& ex) { - refresh_status = absl::InvalidArgumentError( + return absl::InvalidArgumentError( fmt::format("failed to parse ip tags file as yaml: {}", ex.what())); - return nullptr; } } else if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Json)) { bool has_unknown_field; - refresh_status = MessageUtil::loadFromJsonNoThrow(data_source_provider_->data(), - ip_tags_proto, has_unknown_field); - if (!refresh_status.ok()) { - return nullptr; + auto load_status = MessageUtil::loadFromJsonNoThrow(data_source_provider_->data(), + ip_tags_proto, has_unknown_field); + if (!load_status.ok()) { + return load_status; } } - return parseIpTagsAsProto(ip_tags_proto.ip_tags(), refresh_status); + return parseIpTagsAsProto(ip_tags_proto.ip_tags()); } else { - refresh_status = absl::InvalidArgumentError("Unable to load tags from empty datasource"); - return nullptr; + return absl::InvalidArgumentError("Unable to load tags from empty datasource"); } } -LcTrieSharedPtr IpTagsLoader::parseIpTagsAsProto( - const Protobuf::RepeatedPtrField& ip_tags, - absl::Status& creation_status) { +absl::StatusOr IpTagsLoader::parseIpTagsAsProto( + const Protobuf::RepeatedPtrField& ip_tags) { std::vector>> tag_data; tag_data.reserve(ip_tags.size()); for (const auto& ip_tag : ip_tags) { @@ -160,10 +158,9 @@ LcTrieSharedPtr IpTagsLoader::parseIpTagsAsProto( if (cidr_or_error.status().ok()) { cidr_set.emplace_back(std::move(cidr_or_error.value())); } else { - creation_status = absl::InvalidArgumentError( + return absl::InvalidArgumentError( fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", entry.address_prefix(), entry.prefix_len().value())); - return nullptr; } } tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); @@ -224,7 +221,13 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (!config.ip_tags().empty()) { - trie_ = tags_loader_.parseIpTagsAsProto(config.ip_tags(), creation_status); + auto trie_or_error = tags_loader_.parseIpTagsAsProto(config.ip_tags()); + if (trie_or_error.status().ok()) { + trie_ = trie_or_error.value(); + } else { + creation_status = trie_or_error.status(); + return; + } } else { if (!config.ip_tags_file_provider().has_ip_tags_datasource()) { creation_status = absl::InvalidArgumentError( diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 51ea8f84b06ca..73f1ae76ca7ce 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -42,28 +42,27 @@ class IpTagsLoader { * @param dispatcher The dispatcher for the thread used by a datasource provider. * @param tls The thread local slot allocator used by a datasource provider. * @param creation_status This status will be populated with error if loading fails. - * @return Valid LcTrieSharedPtr if loading succeeded or nullptr otherwise. + * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise. */ - LcTrieSharedPtr loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, - Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, - absl::Status& creation_status); + absl::StatusOr + loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, + Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls); /** * Performs periodic refresh of file based ip tags via datasource. * @param refresh_status This status will be populated with error if refresh fails. - * @return Valid LcTrieSharedPtr if loading succeeded or nullptr otherwise. + * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise.. */ - LcTrieSharedPtr refreshTags(absl::Status& refresh_status); + absl::StatusOr refreshTags(); /** * Parses ip tags in a proto format into a trie structure. * @param ip_tags Collection of ip tags in proto format. * @param creation_status This status will be populated with error if parsing fails. - * @return Valid LcTrieSharedPtr if parsing succeeded or nullptr otherwise. + * @return Valid LcTrieSharedPtr if parsing succeeded or error status otherwise. */ - LcTrieSharedPtr - parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags, - absl::Status& creation_status); + absl::StatusOr + parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags); private: Api::Api& api_; From e31c2157f02c8470cd9c7cad55494c25e5d38181 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 9 Jun 2025 16:17:05 +0200 Subject: [PATCH 34/76] Add test for reload during inflight requests Signed-off-by: Kateryna Nezdolii --- source/common/protobuf/utility.h | 5 +- source/common/protobuf/yaml_utility.cc | 54 +++++--- .../extensions/filters/http/ip_tagging/BUILD | 1 + .../http/ip_tagging/ip_tagging_filter.cc | 82 ++++++++---- .../http/ip_tagging/ip_tagging_filter.h | 12 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 120 +++++++++++++++++- 6 files changed, 222 insertions(+), 52 deletions(-) diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index fa3297b4932b9..15147cd53fe91 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -268,7 +268,8 @@ class MessageUtil { bool& has_unknown_fileld); static void loadFromJson(absl::string_view json, ProtobufWkt::Struct& message); static void loadFromYaml(const std::string& yaml, Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor); + ProtobufMessage::ValidationVisitor& validation_visitor, + bool is_custom_thread = false); #endif // This function attempts to load Envoy configuration from the specified file @@ -607,7 +608,7 @@ class ValueUtil { /** * Load YAML string into ProtobufWkt::Value. */ - static ProtobufWkt::Value loadFromYaml(const std::string& yaml); + static ProtobufWkt::Value loadFromYaml(const std::string& yaml, bool is_custom_thread = false); #endif /** diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index 741817bc04315..c9687acd25733 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -171,8 +171,9 @@ void MessageUtil::loadFromJson(absl::string_view json, ProtobufWkt::Struct& mess } void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor) { - ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml); + ProtobufMessage::ValidationVisitor& validation_visitor, + bool is_custom_thread) { + ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml, is_custom_thread); if (value.kind_case() == ProtobufWkt::Value::kStructValue || value.kind_case() == ProtobufWkt::Value::kListValue) { jsonConvertInternal(value, validation_visitor, message); @@ -277,21 +278,40 @@ bool MessageUtil::jsonConvertValue(const Protobuf::Message& source, ProtobufWkt: return false; } -ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) { - TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); } - END_TRY - catch (YAML::ParserException& e) { - throw EnvoyException(e.what()); - } - catch (YAML::BadConversion& e) { - throw EnvoyException(e.what()); - } - catch (std::exception& e) { - // There is a potentially wide space of exceptions thrown by the YAML parser, - // and enumerating them all may be difficult. Envoy doesn't work well with - // unhandled exceptions, so we capture them and record the exception name in - // the Envoy Exception text. - throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); +ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml, bool is_custom_thread) { + // If this is a custom (not main, worker or test) thread, skip macto validation. + if (is_custom_thread) { + TRY_NEEDS_AUDIT { return parseYamlNode(YAML::Load(yaml)); } + END_TRY + catch (YAML::ParserException& e) { + throw EnvoyException(e.what()); + } + catch (YAML::BadConversion& e) { + throw EnvoyException(e.what()); + } + catch (std::exception& e) { + // There is a potentially wide space of exceptions thrown by the YAML parser, + // and enumerating them all may be difficult. Envoy doesn't work well with + // unhandled exceptions, so we capture them and record the exception name in + // the Envoy Exception text. + throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); + } + } else { + TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); } + END_TRY + catch (YAML::ParserException& e) { + throw EnvoyException(e.what()); + } + catch (YAML::BadConversion& e) { + throw EnvoyException(e.what()); + } + catch (std::exception& e) { + // There is a potentially wide space of exceptions thrown by the YAML parser, + // and enumerating them all may be difficult. Envoy doesn't work well with + // unhandled exceptions, so we capture them and record the exception name in + // the Envoy Exception text. + throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); + } } } diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 9b891385fb689..13cdf1863c1b3 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -21,6 +21,7 @@ envoy_cc_library( "//envoy/runtime:runtime_interface", "//envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", + "//source/common/common:thread_synchronizer_lib", "//source/common/config:datasource_lib", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 74da1ce511e69..2b0e1070137f3 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -17,9 +17,10 @@ namespace IpTagging { IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, - IpTagsReloadErrorCb reload_error_cb, Event::Dispatcher& dispatcher, - Api::Api& api, ThreadLocal::SlotAllocator& tls, - Singleton::InstanceSharedPtr owner, absl::Status& creation_status) + IpTagsReloadErrorCb reload_error_cb, + Event::Dispatcher& main_dispatcher, Api::Api& api, + ThreadLocal::SlotAllocator& tls, Singleton::InstanceSharedPtr owner, + absl::Status& creation_status) : ip_tags_path_(ip_tags_datasource.filename()), tags_loader_(tags_loader), time_source_(api.timeSource()), ip_tags_refresh_interval_ms_(std::chrono::milliseconds(ip_tags_refresh_interval_ms)), @@ -33,37 +34,58 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag ip_tags_datasource.has_watched_directory()) { needs_refresh_ = true; } - auto tags_or_error = tags_loader_.loadTags(ip_tags_datasource, dispatcher, tls); + auto tags_or_error = tags_loader_.loadTags(ip_tags_datasource, main_dispatcher, tls); creation_status = tags_or_error.status(); if (tags_or_error.status().ok()) { tags_ = tags_or_error.value(); } + ip_tags_reload_dispatcher_ = api.allocateDispatcher("ip_tags_reload_routine"); + ip_tags_reload_timer_ = ip_tags_reload_dispatcher_->createTimer([this]() -> void { + ENVOY_LOG(debug, "Trying to update ip tags in background"); + auto new_tags_or_error = tags_loader_.refreshTags(); + if (new_tags_or_error.status().ok()) { + updateIpTags(new_tags_or_error.value()); + reload_success_cb_(); + } else { + ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", + new_tags_or_error.status().message()); + reload_error_cb_(); + } + ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); + }); + + ip_tags_reload_thread_ = api.threadFactory().createThread( + [this]() -> void { + ENVOY_LOG(debug, "Started ip_tags_reload_routine"); + if (ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0)) { + ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); + } + ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); + }, + Thread::Options{std::string("ip_tags_reload_routine")}); } -IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); }; +IpTagsProvider::~IpTagsProvider() { + ENVOY_LOG(debug, "Shutting down ip tags provider"); + if (ip_tags_reload_dispatcher_) { + ip_tags_reload_dispatcher_->exit(); + } + if (ip_tags_reload_thread_) { + ip_tags_reload_thread_->join(); + ip_tags_reload_thread_.reset(); + } +}; LcTrieSharedPtr IpTagsProvider::ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { - if (needs_refresh_) { - const auto current_time = time_source_.monotonicTime(); - if (std::chrono::duration_cast(current_time - last_reloaded_time_) > - ip_tags_refresh_interval_ms_) { - auto new_tags_or_error = tags_loader_.refreshTags(); - if (new_tags_or_error.status().ok()) { - absl::MutexLock lock(&ip_tags_mutex_); - tags_ = new_tags_or_error.value(); - reload_success_cb_(); - } else { - ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", - new_tags_or_error.status().message()); - reload_error_cb_(); - } - last_reloaded_time_ = current_time; - } - } absl::ReaderMutexLock lock(&ip_tags_mutex_); return tags_; } +void IpTagsProvider::updateIpTags(LcTrieSharedPtr new_tags) ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { + absl::MutexLock lock(&ip_tags_mutex_); + tags_ = new_tags; +} + std::shared_ptr IpTagsRegistrySingleton::get( const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, @@ -98,7 +120,7 @@ IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& va absl::StatusOr IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, - Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls) { + Event::Dispatcher& main_dispatcher, ThreadLocal::SlotAllocator& tls) { if (!ip_tags_datasource.filename().empty()) { if (!absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Yaml) && !absl::EndsWith(ip_tags_datasource.filename(), MessageUtil::FileExtensions::get().Json)) { @@ -107,7 +129,7 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso ip_tags_datasource.filename()); } auto provider_or_error = Config::DataSource::DataSourceProvider::create( - ip_tags_datasource, dispatcher, tls, api_, false, 0); + ip_tags_datasource, main_dispatcher, tls, api_, false, 0); if (!provider_or_error.status().ok()) { return absl::InvalidArgumentError( fmt::format("unable to create data source '{}'", provider_or_error.status().message())); @@ -122,10 +144,11 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { IpTagFileProto ip_tags_proto; + auto new_data = std::move(data_source_provider_->data()); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { TRY_NEEDS_AUDIT { - MessageUtil::loadFromYaml(data_source_provider_->data(), ip_tags_proto, - validation_visitor_); + MessageUtil::loadFromYaml(new_data, ip_tags_proto, validation_visitor_, + true /*is_custom_thread*/); } END_TRY catch (EnvoyException& ex) { return absl::InvalidArgumentError( @@ -133,8 +156,8 @@ absl::StatusOr IpTagsLoader::refreshTags() { } } else if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Json)) { bool has_unknown_field; - auto load_status = MessageUtil::loadFromJsonNoThrow(data_source_provider_->data(), - ip_tags_proto, has_unknown_field); + auto load_status = + MessageUtil::loadFromJsonNoThrow(new_data, ip_tags_proto, has_unknown_field); if (!load_status.ok()) { return load_status; } @@ -277,6 +300,9 @@ Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::RequestHeaderMap& std::vector tags = config_->trie().getData(callbacks_->streamInfo().downstreamAddressProvider().remoteAddress()); + + // Used for testing. + synchronizer_.syncPoint("_trie_lookup_complete"); applyTags(headers, tags); if (!tags.empty()) { // For a large number(ex > 1000) of tags, stats cardinality will be an issue. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 73f1ae76ca7ce..e870019db823d 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -14,6 +14,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" +#include "source/common/common/thread_synchronizer.h" #include "source/common/config/datasource.h" #include "source/common/network/cidr_range.h" #include "source/common/network/lc_trie.h" @@ -84,7 +85,7 @@ class IpTagsProvider : public Logger::Loggable { IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, - Event::Dispatcher& dispatcher, Api::Api& api, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& main_dispatcher, Api::Api& api, ThreadLocal::SlotAllocator& tls, Singleton::InstanceSharedPtr owner, absl::Status& creation_status); ~IpTagsProvider(); @@ -107,8 +108,13 @@ class IpTagsProvider : public Logger::Loggable { IpTagsReloadErrorCb reload_error_cb_; mutable absl::Mutex ip_tags_mutex_; LcTrieSharedPtr tags_ ABSL_GUARDED_BY(ip_tags_mutex_); + Thread::ThreadPtr ip_tags_reload_thread_; + Event::DispatcherPtr ip_tags_reload_dispatcher_; + Event::TimerPtr ip_tags_reload_timer_; // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. const Singleton::InstanceSharedPtr owner_; + + void updateIpTags(LcTrieSharedPtr new_tags) ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); }; using IpTagsProviderSharedPtr = std::shared_ptr; @@ -259,6 +265,10 @@ class IpTaggingFilter : public Http::StreamDecoderFilter { IpTaggingFilterConfigSharedPtr config_; Http::StreamDecoderFilterCallbacks* callbacks_{}; + // Used for testing only. + mutable Thread::ThreadSynchronizer synchronizer_; + // Allow the unit test to have access to private members. + friend class IpTaggingFilterPeer; }; } // namespace IpTagging diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 473ea543908f7..cc16aa4229afc 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -42,6 +42,13 @@ class IpTaggingFilterConfigPeer { } }; +class IpTaggingFilterPeer { +public: + static Thread::ThreadSynchronizer& synchronizer(std::unique_ptr& filter) { + return filter->synchronizer_; + } +}; + namespace { namespace { @@ -877,6 +884,7 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); @@ -912,7 +920,6 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_success"))); EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; @@ -923,7 +930,7 @@ request_type: internal request_trailers = Http::TestRequestTrailerMapImpl{}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - // Remove the file. + // Remove the files. unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -975,6 +982,7 @@ ip_tags filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_error")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); @@ -1012,7 +1020,6 @@ ip_tags filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter(absl::StrCat(ip_tagging_prefix, "ip_tags_reload_error"))); EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; @@ -1023,7 +1030,112 @@ ip_tags request_trailers = Http::TestRequestTrailerMapImpl{}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); - // Remove the file. + // Remove the files. + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); +} + +TEST_F(IpTaggingFilterTest, IpTagsReloadedInFlightRequestsNotAffected) { + simTime().advanceTimeWait(std::chrono::seconds(1)); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + + envoy::config::core::v3::DataSource config; + TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); + + const std::string yaml = + fmt::format(R"EOF( +request_type: internal +ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" + )EOF", + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test")); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + )EOF", + true); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: internal_updated_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + )EOF", + true); + initializeFilter(yaml); + EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); + IpTaggingFilterPeer::synchronizer(filter_).enable(); + std::string sync_point_name = "_trie_lookup_complete"; + + // Start a thread that issues request for ip tagging filter and wait in the worker thread right + // before performing lookup from the trie with ip tags. + IpTaggingFilterPeer::synchronizer(filter_).waitOn(sync_point_name); + std::thread t0([&] { + Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; + + Network::Address::InstanceConstSharedPtr remote_address = + Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + Http::TestRequestTrailerMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + // Second request should get the updated ip tags. + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( + remote_address); + + // EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); + EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); + request_headers = {{"x-envoy-internal", "true"}}; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ("internal_updated_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); + request_trailers = Http::TestRequestTrailerMapImpl{}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + }); + // Wait until the thread is actually waiting. + IpTaggingFilterPeer::synchronizer(filter_).barrierOn(sync_point_name); + + // Update the symlink to point to the new file. + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml")); + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + + auto contents = TestEnvironment::readFileToStringForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + // Handle the events if any. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + + simTime().advanceTimeWait(std::chrono::seconds(6)); + + IpTaggingFilterPeer::synchronizer(filter_).signal(sync_point_name); + t0.join(); + // Remove the files. unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); From 98039e04f06150e3e65ac7a2df179db5a41213ca Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 10 Jun 2025 13:06:46 +0200 Subject: [PATCH 35/76] Fix ci Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/v3/ip_tagging.proto | 2 +- source/common/protobuf/yaml_utility.cc | 2 +- source/extensions/extensions_build_config.bzl | 52 +++++++++---------- .../http/ip_tagging/ip_tagging_filter.h | 10 ++-- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 82dd38c4479e7..66c3d5e1e6f66 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -21,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // IP tagging :ref:`configuration overview `. // [#extension: envoy.filters.http.ip_tagging] -// [#next-free-field: 8] +// [#next-free-field: 7] message IPTagging { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.ip_tagging.v2.IPTagging"; diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index c9687acd25733..7df60021218c4 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -279,7 +279,7 @@ bool MessageUtil::jsonConvertValue(const Protobuf::Message& source, ProtobufWkt: } ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml, bool is_custom_thread) { - // If this is a custom (not main, worker or test) thread, skip macto validation. + // If this is a custom (not main, worker or test) thread, skip macro validation. if (is_custom_thread) { TRY_NEEDS_AUDIT { return parseYamlNode(YAML::Load(yaml)); } END_TRY diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index c16d6942a3dad..2266b6f5d0316 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -32,12 +32,12 @@ EXTENSIONS = { # Compression # - # "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", - # "envoy.compression.gzip.decompressor": "//source/extensions/compression/gzip/decompressor:config", - # "envoy.compression.brotli.compressor": "//source/extensions/compression/brotli/compressor:config", - # "envoy.compression.brotli.decompressor": "//source/extensions/compression/brotli/decompressor:config", - # "envoy.compression.zstd.compressor": "//source/extensions/compression/zstd/compressor:config", - # "envoy.compression.zstd.decompressor": "//source/extensions/compression/zstd/decompressor:config", + "envoy.compression.gzip.compressor": "//source/extensions/compression/gzip/compressor:config", + "envoy.compression.gzip.decompressor": "//source/extensions/compression/gzip/decompressor:config", + "envoy.compression.brotli.compressor": "//source/extensions/compression/brotli/compressor:config", + "envoy.compression.brotli.decompressor": "//source/extensions/compression/brotli/decompressor:config", + "envoy.compression.zstd.compressor": "//source/extensions/compression/zstd/compressor:config", + "envoy.compression.zstd.decompressor": "//source/extensions/compression/zstd/decompressor:config", # # Config validators @@ -305,15 +305,15 @@ EXTENSIONS = { # Transport sockets # - # "envoy.transport_sockets.alts": "//source/extensions/transport_sockets/alts:config", - # "envoy.transport_sockets.http_11_proxy": "//source/extensions/transport_sockets/http_11_proxy:upstream_config", - # "envoy.transport_sockets.upstream_proxy_protocol": "//source/extensions/transport_sockets/proxy_protocol:upstream_config", - # "envoy.transport_sockets.raw_buffer": "//source/extensions/transport_sockets/raw_buffer:config", - # "envoy.transport_sockets.tap": "//source/extensions/transport_sockets/tap:config", - # "envoy.transport_sockets.starttls": "//source/extensions/transport_sockets/starttls:config", - # "envoy.transport_sockets.tcp_stats": "//source/extensions/transport_sockets/tcp_stats:config", - # "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", - # "envoy.transport_sockets.internal_upstream": "//source/extensions/transport_sockets/internal_upstream:config", + "envoy.transport_sockets.alts": "//source/extensions/transport_sockets/alts:config", + "envoy.transport_sockets.http_11_proxy": "//source/extensions/transport_sockets/http_11_proxy:upstream_config", + "envoy.transport_sockets.upstream_proxy_protocol": "//source/extensions/transport_sockets/proxy_protocol:upstream_config", + "envoy.transport_sockets.raw_buffer": "//source/extensions/transport_sockets/raw_buffer:config", + "envoy.transport_sockets.tap": "//source/extensions/transport_sockets/tap:config", + "envoy.transport_sockets.starttls": "//source/extensions/transport_sockets/starttls:config", + "envoy.transport_sockets.tcp_stats": "//source/extensions/transport_sockets/tcp_stats:config", + "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", + "envoy.transport_sockets.internal_upstream": "//source/extensions/transport_sockets/internal_upstream:config", # # Retry host predicates @@ -376,14 +376,14 @@ EXTENSIONS = { # IO socket # - #"envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", - #"envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", + "envoy.io_socket.user_space": "//source/extensions/io_socket/user_space:config", + "envoy.bootstrap.internal_listener": "//source/extensions/bootstrap/internal_listener:config", # # TLS peer certification validators # - #"envoy.tls.cert_validator.spiffe": "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config", + "envoy.tls.cert_validator.spiffe": "//source/extensions/transport_sockets/tls/cert_validator/spiffe:config", # # HTTP header formatters @@ -428,14 +428,14 @@ EXTENSIONS = { # QUIC extensions # - # "envoy.quic.deterministic_connection_id_generator": "//source/extensions/quic/connection_id_generator/deterministic:envoy_deterministic_connection_id_generator_config", - # "envoy.quic.connection_id_generator.quic_lb": "//source/extensions/quic/connection_id_generator/quic_lb:quic_lb_config", - # "envoy.quic.crypto_stream.server.quiche": "//source/extensions/quic/crypto_stream:envoy_quic_default_crypto_server_stream", - # "envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source", - # "envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config", - # "envoy.quic.server_preferred_address.datasource": "//source/extensions/quic/server_preferred_address:datasource_server_preferred_address_config_factory_config", - # "envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor/basic:envoy_quic_connection_debug_visitor_basic", - # "envoy.quic.connection_debug_visitor.quic_stats": "//source/extensions/quic/connection_debug_visitor/quic_stats:config", + "envoy.quic.deterministic_connection_id_generator": "//source/extensions/quic/connection_id_generator/deterministic:envoy_deterministic_connection_id_generator_config", + "envoy.quic.connection_id_generator.quic_lb": "//source/extensions/quic/connection_id_generator/quic_lb:quic_lb_config", + "envoy.quic.crypto_stream.server.quiche": "//source/extensions/quic/crypto_stream:envoy_quic_default_crypto_server_stream", + "envoy.quic.proof_source.filter_chain": "//source/extensions/quic/proof_source:envoy_quic_default_proof_source", + "envoy.quic.server_preferred_address.fixed": "//source/extensions/quic/server_preferred_address:fixed_server_preferred_address_config_factory_config", + "envoy.quic.server_preferred_address.datasource": "//source/extensions/quic/server_preferred_address:datasource_server_preferred_address_config_factory_config", + "envoy.quic.connection_debug_visitor.basic": "//source/extensions/quic/connection_debug_visitor/basic:envoy_quic_connection_debug_visitor_basic", + "envoy.quic.connection_debug_visitor.quic_stats": "//source/extensions/quic/connection_debug_visitor/quic_stats:config", # # UDP packet writers diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index e870019db823d..ccaf88ead8f1c 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -38,10 +38,10 @@ class IpTagsLoader { Stats::StatNameSetPtr& stat_name_set); /** - * Loads file based ip tags from a datasource and parses them into a trie structure. - * @param ip_tags_datasource file based datasource to load ip tags from. - * @param dispatcher The dispatcher for the thread used by a datasource provider. - * @param tls The thread local slot allocator used by a datasource provider. + * Loads file based ip tags from a data source and parses them into a trie structure. + * @param ip_tags_datasource file based data source to load ip tags from. + * @param dispatcher The dispatcher for the thread used by a data source provider. + * @param tls The thread local slot allocator used by a data source provider. * @param creation_status This status will be populated with error if loading fails. * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise. */ @@ -50,7 +50,7 @@ class IpTagsLoader { Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls); /** - * Performs periodic refresh of file based ip tags via datasource. + * Performs periodic refresh of file based ip tags via data source. * @param refresh_status This status will be populated with error if refresh fails. * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise.. */ From 9431d67a4a533cfba1b0501dc631c3df2b524429 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 11 Jun 2025 11:13:20 +0200 Subject: [PATCH 36/76] Cleanup+clarify user docs Signed-off-by: Kateryna Nezdolii --- .../http/http_filters/ip_tagging_filter.rst | 88 ++++++++++++++++++- .../extensions/filters/http/ip_tagging/BUILD | 1 - .../filters/http/ip_tagging/config.h | 1 - 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index 489646091eddc..ef598ad1c5509 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -24,8 +24,7 @@ G. Karlsson. IP tags can either be provided directly using the :ref:`ip_tags ` api field or can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. -Both :ref:`ip_tags ` and file with *filename* from :ref:`ip_tags_datasource ` -use :ref:`IPTag ` proto format for defining individual ip tags. +For file based ip tags *yaml* and *json* file formats are supported. Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. @@ -34,6 +33,91 @@ Configuration * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging``. * :ref:`v3 API reference ` +An example configuration of the filter with inline ip tags may look like the following: + +.. code-block:: yaml + + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + +Below is an example configuration of the filter with the file based ip tags in yaml format: + +.. code-block:: yaml + + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "/geoip/ip-tags.yaml" + watched_directory: + path: "/geoip/" + +Where the *ip-tags.yaml* file would have the following content: + +.. code-block:: yaml + + ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - ip_tag_name: internal_request + ip_list: + - {address_prefix: 1.2.3.5, prefix_len: 32} + +And here is an example configuration of the filter with the file based ip tags in json format: + +.. code-block:: yaml + + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "/geoip/ip-tags.json" + watched_directory: + path: "/geoip/" + +Where the *ip-tags.json* file would have the following content: + +.. code-block:: json + + { + "ip_tags": [ + { + "ip_tag_name": "external_request", + "ip_list": [ + { + "address_prefix": "1.2.3.4", + "prefix_len": 32 + } + ] + }, + { + "ip_tag_name": "internal_request", + "ip_list": [ + { + "address_prefix": "1.2.3.5", + "prefix_len": 32 + } + ] + } + ] + } + Statistics ---------- diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 13cdf1863c1b3..5b48003cc3f6c 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -39,7 +39,6 @@ envoy_cc_extension( deps = [ "//envoy/registry", "//envoy/thread_local:thread_local_interface", - "//source/common/network:lc_trie_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/ip_tagging:ip_tagging_filter_lib", diff --git a/source/extensions/filters/http/ip_tagging/config.h b/source/extensions/filters/http/ip_tagging/config.h index 5e28d045ce1c6..737a5196348a4 100644 --- a/source/extensions/filters/http/ip_tagging/config.h +++ b/source/extensions/filters/http/ip_tagging/config.h @@ -4,7 +4,6 @@ #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" -#include "source/common/network/lc_trie.h" #include "source/extensions/filters/http/common/factory_base.h" namespace Envoy { From cb447f0fbb7d7f356e3c39f5738f5c223e138646 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 11 Jun 2025 15:05:11 +0200 Subject: [PATCH 37/76] Remaining review comments+ci Signed-off-by: Kateryna Nezdolii --- api/BUILD | 1 - api/envoy/data/ip_tagging/v3/BUILD | 12 ------- api/envoy/data/ip_tagging/v3/ip_tagging.proto | 36 ------------------- .../filters/http/ip_tagging/v3/BUILD | 1 - .../http/ip_tagging/v3/ip_tagging.proto | 27 +++++++++++--- api/versioning/BUILD | 1 - .../http/http_filters/ip_tagging_filter.rst | 4 +-- .../extensions/filters/http/ip_tagging/BUILD | 1 - .../filters/http/ip_tagging/config.h | 1 - .../http/ip_tagging/ip_tagging_filter.cc | 9 ++--- .../http/ip_tagging/ip_tagging_filter.h | 11 +++--- .../http/ip_tagging/ip_tagging_filter_test.cc | 3 -- 12 files changed, 36 insertions(+), 71 deletions(-) delete mode 100644 api/envoy/data/ip_tagging/v3/BUILD delete mode 100644 api/envoy/data/ip_tagging/v3/ip_tagging.proto diff --git a/api/BUILD b/api/BUILD index 4610c94c1dcb8..38d2a0f4692d6 100644 --- a/api/BUILD +++ b/api/BUILD @@ -129,7 +129,6 @@ proto_library( "//envoy/data/cluster/v3:pkg", "//envoy/data/core/v3:pkg", "//envoy/data/dns/v3:pkg", - "//envoy/data/ip_tagging/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", "//envoy/extensions/access_loggers/filters/cel/v3:pkg", diff --git a/api/envoy/data/ip_tagging/v3/BUILD b/api/envoy/data/ip_tagging/v3/BUILD deleted file mode 100644 index 09a37ad16b837..0000000000000 --- a/api/envoy/data/ip_tagging/v3/BUILD +++ /dev/null @@ -1,12 +0,0 @@ -# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. - -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = [ - "//envoy/config/core/v3:pkg", - "@com_github_cncf_xds//udpa/annotations:pkg", - ], -) diff --git a/api/envoy/data/ip_tagging/v3/ip_tagging.proto b/api/envoy/data/ip_tagging/v3/ip_tagging.proto deleted file mode 100644 index 471eed4b00e08..0000000000000 --- a/api/envoy/data/ip_tagging/v3/ip_tagging.proto +++ /dev/null @@ -1,36 +0,0 @@ -syntax = "proto3"; - -package envoy.data.ip_tagging.v3; - -import "envoy/config/core/v3/address.proto"; - -import "udpa/annotations/status.proto"; -import "udpa/annotations/versioning.proto"; - -option java_package = "io.envoyproxy.envoy.data.ip_tagging.v3"; -option java_outer_classname = "IpTaggingProto"; -option java_multiple_files = true; -option go_package = "github.com/envoyproxy/go-control-plane/envoy/data/ip_tagging/v3;ip_taggingv3"; -option (udpa.annotations.file_status).package_version_status = ACTIVE; - -// [#protodoc-title: Ip Tagging Data] -// :ref:`Ip Tagging Filter config overview `. - -// Supplies the IP tag name and the IP address subnets. -message IPTag { - option (udpa.annotations.versioning).previous_message_type = - "envoy.config.filter.http.ip_tagging.v2.IPTagging.IPTag"; - - // Specifies the IP tag name to apply. - string ip_tag_name = 1; - - // A list of IP address subnets that will be tagged with - // ip_tag_name. Both IPv4 and IPv6 are supported. - repeated config.core.v3.CidrRange ip_list = 2; -} - -// Specifies the content of the IP tag file. -// Allow the file to be created with no IP tags. -message IPTagFile { - repeated IPTag ip_tags = 1; -} diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD index 8c2da79c79be0..09a37ad16b837 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/BUILD @@ -7,7 +7,6 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ "//envoy/config/core/v3:pkg", - "//envoy/data/ip_tagging/v3:pkg", "@com_github_cncf_xds//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 66c3d5e1e6f66..e5192dbe4ee98 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package envoy.extensions.filters.http.ip_tagging.v3; +import "envoy/config/core/v3/address.proto"; import "envoy/config/core/v3/base.proto"; -import "envoy/data/ip_tagging/v3/ip_tagging.proto"; import "google/protobuf/duration.proto"; @@ -43,6 +43,25 @@ message IPTagging { EXTERNAL = 2; } + // Supplies the IP tag name and the IP address subnets. + message IPTag { + option (udpa.annotations.versioning).previous_message_type = + "envoy.config.filter.http.ip_tagging.v2.IPTagging.IPTag"; + + // Specifies the IP tag name to apply. + string ip_tag_name = 1; + + // A list of IP address subnets that will be tagged with + // ip_tag_name. Both IPv4 and IPv6 are supported. + repeated config.core.v3.CidrRange ip_list = 2; + } + + // Specifies the content of the IP tag file. +// Allow the file to be created with no IP tags. +message IPTagFile { + repeated IPTag ip_tags = 1; +} + // Specify to which header the tags will be written. message IpTagHeader { // Describes how to apply the tags to the headers. @@ -83,7 +102,7 @@ message IPTagging { // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. config.core.v3.DataSource ip_tags_datasource = 1; - // When :ref:`ip_tags ` is configured + // When :ref:`ip_tags ` is configured // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. // Defaults to 0, in this case no refresh will be attempted. google.protobuf.Duration ip_tags_refresh_rate = 2 [(validate.rules).duration = {gt {}}]; @@ -94,9 +113,9 @@ message IPTagging { // The set of IP tags for the filter. // Only one of :ref:`ip_tags ` - // or :ref:`ip_tags_datasource ` + // or :ref:`ip_tags_datasource ` // can be set for the Ip Tagging filter. - repeated data.ip_tagging.v3.IPTag ip_tags = 4; + repeated IPTag ip_tags = 4; // Specify to which header the tags will be written. // diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 922e48a62e9c0..fab4405373885 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -67,7 +67,6 @@ proto_library( "//envoy/data/cluster/v3:pkg", "//envoy/data/core/v3:pkg", "//envoy/data/dns/v3:pkg", - "//envoy/data/ip_tagging/v3:pkg", "//envoy/data/tap/v3:pkg", "//envoy/extensions/access_loggers/file/v3:pkg", "//envoy/extensions/access_loggers/filters/cel/v3:pkg", diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index ef598ad1c5509..24f39a3feee13 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -23,9 +23,9 @@ LC-tries ` api field or -can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. +can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. For file based ip tags *yaml* and *json* file formats are supported. -Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` +Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. Configuration diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 5b48003cc3f6c..8a5b4c3a8793b 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -42,7 +42,6 @@ envoy_cc_extension( "//source/common/protobuf:utility_lib", "//source/extensions/filters/http/common:factory_base_lib", "//source/extensions/filters/http/ip_tagging:ip_tagging_filter_lib", - "@envoy_api//envoy/data/ip_tagging/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/http/ip_tagging/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/filters/http/ip_tagging/config.h b/source/extensions/filters/http/ip_tagging/config.h index 737a5196348a4..0b8ef7165e775 100644 --- a/source/extensions/filters/http/ip_tagging/config.h +++ b/source/extensions/filters/http/ip_tagging/config.h @@ -1,6 +1,5 @@ #pragma once -#include "envoy/data/ip_tagging/v3/ip_tagging.pb.validate.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.h" #include "envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.pb.validate.h" diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 2b0e1070137f3..f24c6f0059e7e 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -86,7 +86,7 @@ void IpTagsProvider::updateIpTags(LcTrieSharedPtr new_tags) ABSL_LOCKS_EXCLUDED( tags_ = new_tags; } -std::shared_ptr IpTagsRegistrySingleton::get( +std::shared_ptr IpTagsRegistrySingleton::getOrCreateProvider( const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, @@ -144,7 +144,7 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { IpTagFileProto ip_tags_proto; - auto new_data = std::move(data_source_provider_->data()); + auto new_data = data_source_provider_->data(); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { TRY_NEEDS_AUDIT { MessageUtil::loadFromYaml(new_data, ip_tags_proto, validation_visitor_, @@ -169,7 +169,8 @@ absl::StatusOr IpTagsLoader::refreshTags() { } absl::StatusOr IpTagsLoader::parseIpTagsAsProto( - const Protobuf::RepeatedPtrField& ip_tags) { + const Protobuf::RepeatedPtrField< + envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags) { std::vector>> tag_data; tag_data.reserve(ip_tags.size()); for (const auto& ip_tag : ip_tags) { @@ -259,7 +260,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( } auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.ip_tags_file_provider(), ip_tags_refresh_rate, 0); - provider_ = ip_tags_registry_->get( + provider_ = ip_tags_registry_->getOrCreateProvider( config.ip_tags_file_provider().ip_tags_datasource(), tags_loader_, ip_tags_refresh_interval_ms, [this]() { incIpTagsReloadSuccess(); }, [this]() { incIpTagsReloadError(); }, api, tls, dispatcher, ip_tags_registry_, diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index ccaf88ead8f1c..92032b9a79606 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -25,7 +25,7 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -using IpTagFileProto = envoy::data::ip_tagging::v3::IPTagFile; +using IpTagFileProto = envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTagFile; using LcTrieSharedPtr = std::shared_ptr>; /** @@ -63,7 +63,8 @@ class IpTagsLoader { * @return Valid LcTrieSharedPtr if parsing succeeded or error status otherwise. */ absl::StatusOr - parseIpTagsAsProto(const Protobuf::RepeatedPtrField& ip_tags); + parseIpTagsAsProto(const Protobuf::RepeatedPtrField< + envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags); private: Api::Api& api_; @@ -77,7 +78,7 @@ using IpTagsReloadSuccessCb = std::function; using IpTagsReloadErrorCb = std::function; /** - * This class owns ip tags trie structure for a configured abosulte file path and provides access to + * This class owns ip tags trie structure for a configured absolute file path and provides access to * the ip tags data. It also performs periodic refresh of ip tags data. */ class IpTagsProvider : public Logger::Loggable { @@ -128,8 +129,8 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() {} - std::shared_ptr - get(const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, + std::shared_ptr getOrCreateProvider( + const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, Event::Dispatcher& main_dispatcher, std::shared_ptr singleton, diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index cc16aa4229afc..b296ba318aafa 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -52,8 +52,6 @@ class IpTaggingFilterPeer { namespace { namespace { -const std::string ip_tagging_prefix = "prefix.ip_tagging."; - const std::string internal_request_config = R"EOF( request_type: internal ip_tags: @@ -1104,7 +1102,6 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - // EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; From 1a5a1e662e48b19fcb233ada3c8c95409e6f6d2a Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 11 Jun 2025 15:15:20 +0200 Subject: [PATCH 38/76] Cleanup docs Signed-off-by: Kateryna Nezdolii --- docs/root/api-v3/data/data.rst | 2 +- docs/root/api-v3/data/ip_tagging/ip_tagging.rst | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 docs/root/api-v3/data/ip_tagging/ip_tagging.rst diff --git a/docs/root/api-v3/data/data.rst b/docs/root/api-v3/data/data.rst index a2a30425ee6d4..de05d89099e86 100644 --- a/docs/root/api-v3/data/data.rst +++ b/docs/root/api-v3/data/data.rst @@ -10,4 +10,4 @@ Envoy data core/core dns/dns tap/tap - ip_tagging/ip_tagging + diff --git a/docs/root/api-v3/data/ip_tagging/ip_tagging.rst b/docs/root/api-v3/data/ip_tagging/ip_tagging.rst deleted file mode 100644 index 8fffc1bb3fefe..0000000000000 --- a/docs/root/api-v3/data/ip_tagging/ip_tagging.rst +++ /dev/null @@ -1,8 +0,0 @@ -Ip Tagging -=== - -.. toctree:: - :glob: - :maxdepth: 2 - - v3/* From 5285554ba5ba6701dc40108e550563e6e4d717d0 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 13 Jun 2025 12:51:28 +0200 Subject: [PATCH 39/76] Fix proto format Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/v3/ip_tagging.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index e5192dbe4ee98..ff95e1c50175d 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -43,7 +43,7 @@ message IPTagging { EXTERNAL = 2; } - // Supplies the IP tag name and the IP address subnets. + // Supplies the IP tag name and the IP address subnets. message IPTag { option (udpa.annotations.versioning).previous_message_type = "envoy.config.filter.http.ip_tagging.v2.IPTagging.IPTag"; @@ -57,10 +57,10 @@ message IPTagging { } // Specifies the content of the IP tag file. -// Allow the file to be created with no IP tags. -message IPTagFile { - repeated IPTag ip_tags = 1; -} + // Allow the file to be created with no IP tags. + message IPTagFile { + repeated IPTag ip_tags = 1; + } // Specify to which header the tags will be written. message IpTagHeader { @@ -123,4 +123,4 @@ message IPTagFile { IpTagHeader ip_tag_header = 5; IpTagsFileProvider ip_tags_file_provider = 6; -} +} \ No newline at end of file From c3da90d616114a97c12f5f45829fa69a1c7f96d5 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 10:52:36 +0200 Subject: [PATCH 40/76] Fix format Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index ff95e1c50175d..bc647d775e72a 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -123,4 +123,4 @@ message IPTagging { IpTagHeader ip_tag_header = 5; IpTagsFileProvider ip_tags_file_provider = 6; -} \ No newline at end of file +} From edf6d5f6d4e0eccd2efd10cd53224126f7ea331d Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 11:09:45 +0200 Subject: [PATCH 41/76] Fix docs Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 4 ++-- .../configuration/http/http_filters/ip_tagging_filter.rst | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index bc647d775e72a..89d513045e6f7 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -102,7 +102,7 @@ message IPTagging { // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. config.core.v3.DataSource ip_tags_datasource = 1; - // When :ref:`ip_tags ` is configured + // When :ref:`ip_tags ` is configured // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds. // Defaults to 0, in this case no refresh will be attempted. google.protobuf.Duration ip_tags_refresh_rate = 2 [(validate.rules).duration = {gt {}}]; @@ -113,7 +113,7 @@ message IPTagging { // The set of IP tags for the filter. // Only one of :ref:`ip_tags ` - // or :ref:`ip_tags_datasource ` + // or :ref:`ip_tags_datasource ` // can be set for the Ip Tagging filter. repeated IPTag ip_tags = 4; diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index 24f39a3feee13..1521a9bf75b41 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -23,10 +23,10 @@ LC-tries ` api field or -can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. +can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. For file based ip tags *yaml* and *json* file formats are supported. -Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` -and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. +Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` +and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. Configuration ------------- From 67f802d114a075172e83f289b6159164902acf06 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 11:58:43 +0200 Subject: [PATCH 42/76] Review comments + more tests Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 8 ++++---- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index f24c6f0059e7e..18e02d8a7a336 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -24,16 +24,16 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag : ip_tags_path_(ip_tags_datasource.filename()), tags_loader_(tags_loader), time_source_(api.timeSource()), ip_tags_refresh_interval_ms_(std::chrono::milliseconds(ip_tags_refresh_interval_ms)), + needs_refresh_(ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0) && + ip_tags_datasource.has_watched_directory() + ? true + : false), reload_success_cb_(reload_success_cb), reload_error_cb_(reload_error_cb), owner_(owner) { RETURN_ONLY_IF_NOT_OK_REF(creation_status); if (ip_tags_datasource.filename().empty()) { creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); return; } - if (ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0) && - ip_tags_datasource.has_watched_directory()) { - needs_refresh_ = true; - } auto tags_or_error = tags_loader_.loadTags(ip_tags_datasource, main_dispatcher, tls); creation_status = tags_or_error.status(); if (tags_or_error.status().ok()) { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 92032b9a79606..75aae91ca7736 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -101,10 +101,10 @@ class IpTagsProvider : public Logger::Loggable { private: const std::string ip_tags_path_; IpTagsLoader& tags_loader_; - bool needs_refresh_{false}; TimeSource& time_source_; MonotonicTime last_reloaded_time_; const std::chrono::milliseconds ip_tags_refresh_interval_ms_; + const bool needs_refresh_; IpTagsReloadSuccessCb reload_success_cb_; IpTagsReloadErrorCb reload_error_cb_; mutable absl::Mutex ip_tags_mutex_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index b296ba318aafa..a2a080a91fbdb 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -343,6 +343,26 @@ request_type: internal "INVALID_ARGUMENT: Only one of ip_tags or ip_tags_file_provider can be configured."); } +TEST_F(IpTaggingFilterTest, EmptyDatasourceConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_file_provider: + ip_tags_datasource: +)EOF"; + initializeFilter(config_yaml, "INVALID_ARGUMENT: ip_tags_file_provider requires a valid " + "ip_tags_datasource to be configured."); +} + +TEST_F(IpTaggingFilterTest, EmptyFilenameInDatasourceConfigured) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_file_provider: + ip_tags_datasource: + filename: +)EOF"; + initializeFilter(config_yaml, "INVALID_ARGUMENT: Cannot load tags from empty file path."); +} + TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { const std::string config_yaml = R"EOF( request_type: internal From 106dc078a58462fb359858cfd71da781a35fa026 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 14:57:40 +0200 Subject: [PATCH 43/76] Temp commit to debug ci failure Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d540eddb0a329..bd799e3a4c090 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -46,6 +46,8 @@ setup_clang_toolchain() { fi CONFIG="$(IFS=- ; echo "${CONFIG_PARTS[*]}")" BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") + BAZEL_BUILD_OPTIONS+=("--test_output=all") + BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}" export BAZEL_BUILD_OPTION_LIST echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}" @@ -264,13 +266,13 @@ if [[ $# -ge 1 ]]; then else # Coverage test will add QUICHE tests by itself. COVERAGE_TEST_TARGETS=("//test/...") - if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then - # We test contrib on release only. - COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") - elif [[ "${CI_TARGET}" == "msan" ]]; then - COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...") - fi - TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests") +# if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then +# # We test contrib on release only. +# #COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") +# elif [[ "${CI_TARGET}" == "msan" ]]; then +# COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...") +# fi + TEST_TARGETS=("test/extensions/filters/http/ip_tagging:ip_tagging_filter_test") fi case $CI_TARGET in From 48f2d0f864e9b2cf22c83a801fdf4804cc673905 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 15:11:30 +0200 Subject: [PATCH 44/76] more verbose output Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index bd799e3a4c090..7adf7a745543c 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -48,6 +48,7 @@ setup_clang_toolchain() { BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") BAZEL_BUILD_OPTIONS+=("--test_output=all") BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") + BAZEL_BUILD_OPTIONS+=("--test_arg='-l trace'") BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}" export BAZEL_BUILD_OPTION_LIST echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}" From 1b1a001cff3b55561fece9c4aa48fdf9eef252ac Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 16 Jun 2025 15:28:00 +0200 Subject: [PATCH 45/76] Revert Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 7adf7a745543c..bd799e3a4c090 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -48,7 +48,6 @@ setup_clang_toolchain() { BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") BAZEL_BUILD_OPTIONS+=("--test_output=all") BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") - BAZEL_BUILD_OPTIONS+=("--test_arg='-l trace'") BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}" export BAZEL_BUILD_OPTION_LIST echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}" From 1cd0b831325debf1afa8db0820e441aeff00d1c9 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 10:36:30 +0200 Subject: [PATCH 46/76] Get verbose ci failure Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index bd799e3a4c090..af3bb2d795a63 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -48,6 +48,7 @@ setup_clang_toolchain() { BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") BAZEL_BUILD_OPTIONS+=("--test_output=all") BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") + BAZEL_BUILD_OPTIONS+=("--verbose_failures") BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}" export BAZEL_BUILD_OPTION_LIST echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}" From 725b5f7f2677bc55451825a5ac6722ffddc99b31 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 10:54:45 +0200 Subject: [PATCH 47/76] Trying to get more output Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index af3bb2d795a63..76758aa5184bc 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -45,7 +45,6 @@ setup_clang_toolchain() { CONFIG_PARTS+=("libc++") fi CONFIG="$(IFS=- ; echo "${CONFIG_PARTS[*]}")" - BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") BAZEL_BUILD_OPTIONS+=("--test_output=all") BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") BAZEL_BUILD_OPTIONS+=("--verbose_failures") From 5ca713f6b7091c0488a73e668b27242a8ed0745a Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 11:09:20 +0200 Subject: [PATCH 48/76] Revert Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 76758aa5184bc..09f291f76bc58 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -45,9 +45,7 @@ setup_clang_toolchain() { CONFIG_PARTS+=("libc++") fi CONFIG="$(IFS=- ; echo "${CONFIG_PARTS[*]}")" - BAZEL_BUILD_OPTIONS+=("--test_output=all") - BAZEL_BUILD_OPTIONS+=("--cache_test_results=no") - BAZEL_BUILD_OPTIONS+=("--verbose_failures") + BAZEL_BUILD_OPTIONS+=("--config=${CONFIG}") BAZEL_BUILD_OPTION_LIST="${BAZEL_BUILD_OPTIONS[*]}" export BAZEL_BUILD_OPTION_LIST echo "clang toolchain with ${ENVOY_STDLIB} configured: ${CONFIG}" From 47e300d2971a45ea65a6f796d743ac6e01d50abf Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 11:24:15 +0200 Subject: [PATCH 49/76] More debug Signed-off-by: Kateryna Nezdolii --- .bazelrc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.bazelrc b/.bazelrc index d8d1c6e4424fc..fb735d421a074 100644 --- a/.bazelrc +++ b/.bazelrc @@ -87,7 +87,7 @@ build:sanitizer --linkopt -ldl # Common flags for Clang build:clang --action_env=BAZEL_COMPILER=clang -build:clang --linkopt=-fuse-ld=lld +build:clang --linkopt=-fuse-ld=ld build:clang --action_env=CC=clang --host_action_env=CC=clang build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ build:clang --incompatible_enable_cc_toolchain_resolution=false @@ -151,7 +151,7 @@ build:asan --copt -fno-optimize-sibling-calls # Clang ASAN/UBSAN build:clang-asan-common --config=clang build:clang-asan-common --config=asan -build:clang-asan-common --linkopt -fuse-ld=lld +build:clang-asan-common --linkopt -fuse-ld=ld build:clang-asan-common --linkopt --rtlib=compiler-rt build:clang-asan-common --linkopt --unwindlib=libgcc @@ -185,7 +185,7 @@ build:clang-tsan --config=sanitizer build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread -build:clang-tsan --linkopt -fuse-ld=lld +build:clang-tsan --linkopt -fuse-ld=ld build:clang-tsan --copt -DTHREAD_SANITIZER=1 build:clang-tsan --build_tag_filters=-no_san,-no_tsan build:clang-tsan --test_tag_filters=-no_san,-no_tsan @@ -206,7 +206,7 @@ build:clang-msan --test_tag_filters=-no_san build:clang-msan --define ENVOY_CONFIG_MSAN=1 build:clang-msan --copt -fsanitize=memory build:clang-msan --linkopt -fsanitize=memory -build:clang-msan --linkopt -fuse-ld=lld +build:clang-msan --linkopt -fuse-ld=ld build:clang-msan --copt -fsanitize-memory-track-origins=2 build:clang-msan --copt -DMEMORY_SANITIZER=1 build:clang-msan --test_env=MSAN_SYMBOLIZER_PATH @@ -304,9 +304,11 @@ build:rbe-toolchain-arm64-clang-libc++ --action_env=CC=clang --action_env=CXX=cl build:rbe-toolchain-arm64-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --define force_libcpp=enabled +build:rbe-toolchain-arm64-clang-libc++ --test_arg="-l trace" +build:rbe-toolchain-arm64-clang-libc++ --test_output=all build:rbe-toolchain-asan --config=clang-asan -build:rbe-toolchain-asan --linkopt -fuse-ld=lld +build:rbe-toolchain-asan --linkopt -fuse-ld=ld build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1 build:rbe-toolchain-asan --copt=-fsanitize=vptr,function build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function From 3988fcb5b8d20c176c0f41be9dfcc8ff90ef5120 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 11:27:22 +0200 Subject: [PATCH 50/76] clean up Signed-off-by: Kateryna Nezdolii --- .bazelrc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.bazelrc b/.bazelrc index fb735d421a074..0a55b4251e308 100644 --- a/.bazelrc +++ b/.bazelrc @@ -87,7 +87,7 @@ build:sanitizer --linkopt -ldl # Common flags for Clang build:clang --action_env=BAZEL_COMPILER=clang -build:clang --linkopt=-fuse-ld=ld +build:clang --linkopt=-fuse-ld=lld build:clang --action_env=CC=clang --host_action_env=CC=clang build:clang --action_env=CXX=clang++ --host_action_env=CXX=clang++ build:clang --incompatible_enable_cc_toolchain_resolution=false @@ -151,7 +151,7 @@ build:asan --copt -fno-optimize-sibling-calls # Clang ASAN/UBSAN build:clang-asan-common --config=clang build:clang-asan-common --config=asan -build:clang-asan-common --linkopt -fuse-ld=ld +build:clang-asan-common --linkopt -fuse-ld=lld build:clang-asan-common --linkopt --rtlib=compiler-rt build:clang-asan-common --linkopt --unwindlib=libgcc @@ -185,7 +185,7 @@ build:clang-tsan --config=sanitizer build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread -build:clang-tsan --linkopt -fuse-ld=ld +build:clang-tsan --linkopt -fuse-ld=lld build:clang-tsan --copt -DTHREAD_SANITIZER=1 build:clang-tsan --build_tag_filters=-no_san,-no_tsan build:clang-tsan --test_tag_filters=-no_san,-no_tsan @@ -206,7 +206,7 @@ build:clang-msan --test_tag_filters=-no_san build:clang-msan --define ENVOY_CONFIG_MSAN=1 build:clang-msan --copt -fsanitize=memory build:clang-msan --linkopt -fsanitize=memory -build:clang-msan --linkopt -fuse-ld=ld +build:clang-msan --linkopt -fuse-ld=lld build:clang-msan --copt -fsanitize-memory-track-origins=2 build:clang-msan --copt -DMEMORY_SANITIZER=1 build:clang-msan --test_env=MSAN_SYMBOLIZER_PATH @@ -217,11 +217,13 @@ build:clang-msan --copt -fno-optimize-sibling-calls # Clang with libc++ build:libc++ --config=clang build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:libc++ --action_env=LDFLAGS=-stdlib=libc++ +build:libc++ --action_env=lldFLAGS=-stdlib=libc++ build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a build:libc++ --action_env=BAZEL_LINKOPTS=-lm:-pthread build:libc++ --define force_libcpp=enabled +build:libc++ --test_arg="-l trace" +build:libc++ --test_output=all build:clang-libc++ --config=libc++ build:clang-libc++ --action_env=ARFLAGS=r build:arm64-clang-libc++ --config=clang-libc++ @@ -292,7 +294,7 @@ build:rbe-toolchain-clang-libc++ --crosstool_top=@envoy//bazel/rbe/toolchains/co build:rbe-toolchain-clang-libc++ --extra_toolchains=@envoy//bazel/rbe/toolchains/configs/linux/clang_libcxx/config:cc-toolchain build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ +build:rbe-toolchain-clang-libc++ --action_env=lldFLAGS=-stdlib=libc++ build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-arm64-clang-libc++ --config=rbe-toolchain @@ -302,13 +304,13 @@ build:rbe-toolchain-arm64-clang-libc++ --crosstool_top=@envoy//bazel/rbe/toolcha build:rbe-toolchain-arm64-clang-libc++ --extra_toolchains=@envoy//bazel/rbe/toolchains/configs/linux/clang_libcxx/config:cc-toolchain-arm64 build:rbe-toolchain-arm64-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ build:rbe-toolchain-arm64-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:rbe-toolchain-arm64-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ +build:rbe-toolchain-arm64-clang-libc++ --action_env=lldFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-arm64-clang-libc++ --test_arg="-l trace" build:rbe-toolchain-arm64-clang-libc++ --test_output=all build:rbe-toolchain-asan --config=clang-asan -build:rbe-toolchain-asan --linkopt -fuse-ld=ld +build:rbe-toolchain-asan --linkopt -fuse-ld=lld build:rbe-toolchain-asan --action_env=ENVOY_UBSAN_VPTR=1 build:rbe-toolchain-asan --copt=-fsanitize=vptr,function build:rbe-toolchain-asan --linkopt=-fsanitize=vptr,function From 51b153f57fbcd9f7ee84c5a5dce5fba76fb60438 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 11:37:57 +0200 Subject: [PATCH 51/76] cleanup Signed-off-by: Kateryna Nezdolii --- .bazelrc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index 0a55b4251e308..4b185b7afc21e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -217,7 +217,7 @@ build:clang-msan --copt -fno-optimize-sibling-calls # Clang with libc++ build:libc++ --config=clang build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:libc++ --action_env=lldFLAGS=-stdlib=libc++ +build:libc++ --action_env=LDFLAGS=-stdlib=libc++ build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a build:libc++ --action_env=BAZEL_LINKOPTS=-lm:-pthread @@ -294,7 +294,7 @@ build:rbe-toolchain-clang-libc++ --crosstool_top=@envoy//bazel/rbe/toolchains/co build:rbe-toolchain-clang-libc++ --extra_toolchains=@envoy//bazel/rbe/toolchains/configs/linux/clang_libcxx/config:cc-toolchain build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ build:rbe-toolchain-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:rbe-toolchain-clang-libc++ --action_env=lldFLAGS=-stdlib=libc++ +build:rbe-toolchain-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-arm64-clang-libc++ --config=rbe-toolchain @@ -304,7 +304,7 @@ build:rbe-toolchain-arm64-clang-libc++ --crosstool_top=@envoy//bazel/rbe/toolcha build:rbe-toolchain-arm64-clang-libc++ --extra_toolchains=@envoy//bazel/rbe/toolchains/configs/linux/clang_libcxx/config:cc-toolchain-arm64 build:rbe-toolchain-arm64-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ build:rbe-toolchain-arm64-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ -build:rbe-toolchain-arm64-clang-libc++ --action_env=lldFLAGS=-stdlib=libc++ +build:rbe-toolchain-arm64-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-arm64-clang-libc++ --test_arg="-l trace" build:rbe-toolchain-arm64-clang-libc++ --test_output=all From 110f1572e860f125cf2e9d8273b3caf311fddc6d Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 12:59:47 +0200 Subject: [PATCH 52/76] Try fix test in ci release mode Signed-off-by: Kateryna Nezdolii --- .bazelrc | 4 - .../http/ip_tagging/ip_tagging_filter_test.cc | 140 ++++++++++-------- 2 files changed, 77 insertions(+), 67 deletions(-) diff --git a/.bazelrc b/.bazelrc index 4b185b7afc21e..d8d1c6e4424fc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -222,8 +222,6 @@ build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a build:libc++ --action_env=BAZEL_LINKOPTS=-lm:-pthread build:libc++ --define force_libcpp=enabled -build:libc++ --test_arg="-l trace" -build:libc++ --test_output=all build:clang-libc++ --config=libc++ build:clang-libc++ --action_env=ARFLAGS=r build:arm64-clang-libc++ --config=clang-libc++ @@ -306,8 +304,6 @@ build:rbe-toolchain-arm64-clang-libc++ --action_env=CC=clang --action_env=CXX=cl build:rbe-toolchain-arm64-clang-libc++ --action_env=CXXFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --action_env=LDFLAGS=-stdlib=libc++ build:rbe-toolchain-arm64-clang-libc++ --define force_libcpp=enabled -build:rbe-toolchain-arm64-clang-libc++ --test_arg="-l trace" -build:rbe-toolchain-arm64-clang-libc++ --test_output=all build:rbe-toolchain-asan --config=clang-asan build:rbe-toolchain-asan --linkopt -fuse-ld=lld diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index a2a080a91fbdb..089c3b15bd86a 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -266,11 +266,11 @@ const std::string ipv6_with_json_file_config = R"EOF( } // namespace -class IpTaggingFilterTest : public Event::TestUsingSimulatedTime, - public ::testing::TestWithParam { +class IpTaggingFilterTest : public ::testing::TestWithParam { public: IpTaggingFilterTest() - : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher("test_thread")) { + : scope_(stats_.rootScope()), api_(Api::createApiForTest(stats_, time_system_)), + dispatcher_(api_->allocateDispatcher("test_thread")) { ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) .WillByDefault(Return(true)); } @@ -280,8 +280,8 @@ class IpTaggingFilterTest : public Event::TestUsingSimulatedTime, envoy::extensions::filters::http::ip_tagging::v3::IPTagging config; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), config); auto config_or = - IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, *stats_.rootScope(), - runtime_, *api_, tls_, *dispatcher_, validation_visitor_); + IpTaggingFilterConfig::create(config, "prefix.", *singleton_manager_, *scope_, runtime_, + *api_, tls_, *dispatcher_, validation_visitor_); if (expected_error.has_value()) { EXPECT_FALSE(config_or.ok()); EXPECT_EQ(expected_error.value(), absl::StrCat(config_or.status())); @@ -301,10 +301,12 @@ class IpTaggingFilterTest : public Event::TestUsingSimulatedTime, std::unique_ptr singleton_manager_ = std::make_unique(); + Envoy::Stats::TestUtil::TestStore stats_; + Stats::ScopeSharedPtr scope_; + Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; NiceMock tls_; - NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; std::unique_ptr filter_; NiceMock filter_callbacks_; @@ -392,17 +394,17 @@ TEST_F(IpTaggingFilterTest, ReusesIpTagsProviderInstanceForSameFilePath) { envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config1); - absl::StatusOr config1_result = IpTaggingFilterConfig::create( - proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, - *dispatcher_, validation_visitor_); + absl::StatusOr config1_result = + IpTaggingFilterConfig::create(proto_config1, "prefix.", *singleton_manager_, *scope_, + runtime_, *api_, tls_, *dispatcher_, validation_visitor_); EXPECT_TRUE(config1_result.ok()); auto config1 = config1_result.value(); envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config2); - absl::StatusOr config2_result = IpTaggingFilterConfig::create( - proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, - *dispatcher_, validation_visitor_); + absl::StatusOr config2_result = + IpTaggingFilterConfig::create(proto_config2, "prefix.", *singleton_manager_, *scope_, + runtime_, *api_, tls_, *dispatcher_, validation_visitor_); EXPECT_TRUE(config2_result.ok()); auto config2 = config2_result.value(); auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); @@ -419,17 +421,17 @@ TEST_F(IpTaggingFilterTest, DifferentIpTagsProviderInstanceForDifferentFilePath) envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config1; TestUtility::loadFromYaml(TestEnvironment::substitute(internal_request_with_json_file_config), proto_config1); - absl::StatusOr config1_result = IpTaggingFilterConfig::create( - proto_config1, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, - *dispatcher_, validation_visitor_); + absl::StatusOr config1_result = + IpTaggingFilterConfig::create(proto_config1, "prefix.", *singleton_manager_, *scope_, + runtime_, *api_, tls_, *dispatcher_, validation_visitor_); EXPECT_TRUE(config1_result.ok()); auto config1 = config1_result.value(); envoy::extensions::filters::http::ip_tagging::v3::IPTagging proto_config2; TestUtility::loadFromYaml(TestEnvironment::substitute(external_request_with_json_file_config), proto_config2); - absl::StatusOr config2_result = IpTaggingFilterConfig::create( - proto_config2, "prefix.", *singleton_manager_, *stats_.rootScope(), runtime_, *api_, tls_, - *dispatcher_, validation_visitor_); + absl::StatusOr config2_result = + IpTaggingFilterConfig::create(proto_config2, "prefix.", *singleton_manager_, *scope_, + runtime_, *api_, tls_, *dispatcher_, validation_visitor_); EXPECT_TRUE(config2_result.ok()); auto config2 = config2_result.value(); auto ip_tags_registry1 = IpTaggingFilterConfigPeer::ipTagsRegistry(*config1); @@ -456,12 +458,12 @@ TEST_P(InternalRequestIpTaggingFilterTest, InternalRequest) { filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -485,9 +487,6 @@ TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { EXPECT_EQ(FilterRequestType::EXTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers; - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.4"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( @@ -496,6 +495,9 @@ TEST_P(ExternalRequestIpTaggingFilterTest, ExternalRequest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.external_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -519,10 +521,6 @@ TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { EXPECT_EQ(FilterRequestType::BOTH, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")).Times(2); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.external_request.hit")); - Network::Address::InstanceConstSharedPtr remote_address = Network::Utility::parseInternetAddressNoThrow("1.2.3.5"); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( @@ -538,6 +536,10 @@ TEST_P(BothRequestIpTaggingFilterTest, BothRequest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("external_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.external_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 2); } INSTANTIATE_TEST_CASE_P(BothRequest, BothRequestIpTaggingFilterTest, @@ -557,12 +559,12 @@ TEST_P(NoHitsIpTaggingFilterTest, NoHits) { filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.no_hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.no_hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -758,12 +760,12 @@ TEST_P(NestedPrefixesFilterTest, NestedPrefixes) { filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.duplicate_request.hit")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.duplicate_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + // There is no guarantee for the order tags are returned by the LC-Trie. const std::string header_tag_data = request_headers.get_(Http::Headers::get().EnvoyIpTags.get()); EXPECT_NE(std::string::npos, header_tag_data.find("test")); @@ -854,7 +856,7 @@ INSTANTIATE_TEST_CASE_P(ClearRouteCache, ClearRouteCacheTest, internal_request_with_yaml_file_config})); TEST_F(IpTaggingFilterTest, InternalRequestWithReload) { - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -892,7 +894,7 @@ request_type: internal )EOF", true); initializeFilter(yaml); - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -902,13 +904,12 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -920,7 +921,7 @@ request_type: internal // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); // Update the symlink to point to the new file. TestEnvironment::renameFile( @@ -934,16 +935,23 @@ request_type: internal TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - simTime().advanceTimeWait(std::chrono::seconds(6)); + time_system_.advanceTimeWait(std::chrono::seconds(6)); + + EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, + time_system_)); + + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.ip_tags_reload_error").value(), 0); + filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_updated_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_updated_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 2); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); request_trailers = Http::TestRequestTrailerMapImpl{}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -955,7 +963,7 @@ request_type: internal } TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -990,7 +998,7 @@ ip_tags )EOF", true); initializeFilter(yaml); - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -1000,13 +1008,12 @@ ip_tags filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_error")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -1018,7 +1025,7 @@ ip_tags // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); // Update the symlink to point to the new file. TestEnvironment::renameFile( @@ -1033,17 +1040,20 @@ ip_tags // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - simTime().advanceTimeWait(std::chrono::seconds(6)); + time_system_.advanceTimeWait(std::chrono::seconds(6)); + EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_error", 1UL, + time_system_)); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 2); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 2); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); request_trailers = Http::TestRequestTrailerMapImpl{}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -1055,7 +1065,7 @@ ip_tags } TEST_F(IpTaggingFilterTest, IpTagsReloadedInFlightRequestsNotAffected) { - simTime().advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeWait(std::chrono::seconds(1)); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -1108,13 +1118,12 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.ip_tags_reload_success")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); - EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_request.hit").value(), 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 1); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); Http::TestRequestTrailerMapImpl request_trailers; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -1122,12 +1131,14 @@ request_type: internal filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.internal_updated_request.hit")); - EXPECT_CALL(stats_, counter("prefix.ip_tagging.total")); request_headers = {{"x-envoy-internal", "true"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_EQ("internal_updated_request", request_headers.get_(Http::Headers::get().EnvoyIpTags)); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.internal_updated_request.hit").value(), + 1); + EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.total").value(), 2); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); request_trailers = Http::TestRequestTrailerMapImpl{}; EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); @@ -1148,7 +1159,10 @@ request_type: internal // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - simTime().advanceTimeWait(std::chrono::seconds(6)); + time_system_.advanceTimeWait(std::chrono::seconds(6)); + + EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, + time_system_)); IpTaggingFilterPeer::synchronizer(filter_).signal(sync_point_name); t0.join(); From c9c6d3e5bcd6bf69e13abbf06a9b8853110011fb Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 13:13:17 +0200 Subject: [PATCH 53/76] Clean up changes to ci script Signed-off-by: Kateryna Nezdolii --- ci/do_ci.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 09f291f76bc58..d540eddb0a329 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -264,13 +264,13 @@ if [[ $# -ge 1 ]]; then else # Coverage test will add QUICHE tests by itself. COVERAGE_TEST_TARGETS=("//test/...") -# if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then -# # We test contrib on release only. -# #COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") -# elif [[ "${CI_TARGET}" == "msan" ]]; then -# COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...") -# fi - TEST_TARGETS=("test/extensions/filters/http/ip_tagging:ip_tagging_filter_test") + if [[ "${CI_TARGET}" == "release" || "${CI_TARGET}" == "release.test_only" ]]; then + # We test contrib on release only. + COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "//contrib/...") + elif [[ "${CI_TARGET}" == "msan" ]]; then + COVERAGE_TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "-//test/extensions/...") + fi + TEST_TARGETS=("${COVERAGE_TEST_TARGETS[@]}" "@com_github_google_quiche//:ci_tests") fi case $CI_TARGET in From c333aa3c874bccb0ed2d66ea442da480ad427af2 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 13:36:54 +0200 Subject: [PATCH 54/76] Add extra debug info for segfault investigation Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_filter.cc | 10 ++++++++++ .../filters/http/ip_tagging/ip_tagging_filter.h | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 18e02d8a7a336..ba9d270c71467 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -43,9 +43,13 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag ip_tags_reload_timer_ = ip_tags_reload_dispatcher_->createTimer([this]() -> void { ENVOY_LOG(debug, "Trying to update ip tags in background"); auto new_tags_or_error = tags_loader_.refreshTags(); + ENVOY_LOG(debug, "Refresh tags done"); if (new_tags_or_error.status().ok()) { + ENVOY_LOG(debug, "Updating ip tags"); updateIpTags(new_tags_or_error.value()); + ENVOY_LOG(debug, "Invoking callbacks"); reload_success_cb_(); + ENVOY_LOG(debug, "After Invoking callbacks"); } else { ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", new_tags_or_error.status().message()); @@ -143,12 +147,16 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { + ENVOY_LOG(debug, "data_source_provider_ ok"); IpTagFileProto ip_tags_proto; auto new_data = data_source_provider_->data(); + ENVOY_LOG(debug, "Got data from data_source_provider"); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { TRY_NEEDS_AUDIT { + ENVOY_LOG(debug, "Loading from yaml"); MessageUtil::loadFromYaml(new_data, ip_tags_proto, validation_visitor_, true /*is_custom_thread*/); + ENVOY_LOG(debug, "Loading from yaml done"); } END_TRY catch (EnvoyException& ex) { return absl::InvalidArgumentError( @@ -171,6 +179,7 @@ absl::StatusOr IpTagsLoader::refreshTags() { absl::StatusOr IpTagsLoader::parseIpTagsAsProto( const Protobuf::RepeatedPtrField< envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags) { + ENVOY_LOG(debug, "Parsing ip tags as proto"); std::vector>> tag_data; tag_data.reserve(ip_tags.size()); for (const auto& ip_tag : ip_tags) { @@ -190,6 +199,7 @@ absl::StatusOr IpTagsLoader::parseIpTagsAsProto( tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } + ENVOY_LOG(debug, "Parsing ip tags as proto done"); return std::make_shared>(tag_data); } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 75aae91ca7736..455bcf5b53bfc 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -32,7 +32,7 @@ using LcTrieSharedPtr = std::shared_ptr>; * This class is responsible for loading and parsing of ip tags (both inline and file based) * as well as for periodic refresh of ip tags (file based). */ -class IpTagsLoader { +class IpTagsLoader : public Logger::Loggable { public: IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set); From 7206d63a8811552d7643bfd62b53b17a79796217 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 23:32:56 +0200 Subject: [PATCH 55/76] Incr coverage + cleanup Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_filter.cc | 10 ---------- .../http/ip_tagging/ip_tagging_filter_test.cc | 14 ++++++++++++++ .../http/ip_tagging/test_data/empty_file.yaml | 0 3 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/empty_file.yaml diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index ba9d270c71467..18e02d8a7a336 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -43,13 +43,9 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag ip_tags_reload_timer_ = ip_tags_reload_dispatcher_->createTimer([this]() -> void { ENVOY_LOG(debug, "Trying to update ip tags in background"); auto new_tags_or_error = tags_loader_.refreshTags(); - ENVOY_LOG(debug, "Refresh tags done"); if (new_tags_or_error.status().ok()) { - ENVOY_LOG(debug, "Updating ip tags"); updateIpTags(new_tags_or_error.value()); - ENVOY_LOG(debug, "Invoking callbacks"); reload_success_cb_(); - ENVOY_LOG(debug, "After Invoking callbacks"); } else { ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", new_tags_or_error.status().message()); @@ -147,16 +143,12 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { - ENVOY_LOG(debug, "data_source_provider_ ok"); IpTagFileProto ip_tags_proto; auto new_data = data_source_provider_->data(); - ENVOY_LOG(debug, "Got data from data_source_provider"); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { TRY_NEEDS_AUDIT { - ENVOY_LOG(debug, "Loading from yaml"); MessageUtil::loadFromYaml(new_data, ip_tags_proto, validation_visitor_, true /*is_custom_thread*/); - ENVOY_LOG(debug, "Loading from yaml done"); } END_TRY catch (EnvoyException& ex) { return absl::InvalidArgumentError( @@ -179,7 +171,6 @@ absl::StatusOr IpTagsLoader::refreshTags() { absl::StatusOr IpTagsLoader::parseIpTagsAsProto( const Protobuf::RepeatedPtrField< envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags) { - ENVOY_LOG(debug, "Parsing ip tags as proto"); std::vector>> tag_data; tag_data.reserve(ip_tags.size()); for (const auto& ip_tag : ip_tags) { @@ -199,7 +190,6 @@ absl::StatusOr IpTagsLoader::parseIpTagsAsProto( tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } - ENVOY_LOG(debug, "Parsing ip tags as proto done"); return std::make_shared>(tag_data); } diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 089c3b15bd86a..185b76a8003d5 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -355,6 +355,20 @@ request_type: internal "ip_tags_datasource to be configured."); } +TEST_F(IpTaggingFilterTest, EmptyIpTagsFile) { + const std::string config_yaml = R"EOF( +request_type: internal +ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/empty_file.yaml" +)EOF"; + std::string file = TestEnvironment::substitute( + "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/empty_file.yaml"); + initializeFilter( + config_yaml, + absl::StrCat("INVALID_ARGUMENT: unable to create data source 'file ", file, " is empty'")); +} + TEST_F(IpTaggingFilterTest, EmptyFilenameInDatasourceConfigured) { const std::string config_yaml = R"EOF( request_type: internal diff --git a/test/extensions/filters/http/ip_tagging/test_data/empty_file.yaml b/test/extensions/filters/http/ip_tagging/test_data/empty_file.yaml new file mode 100644 index 0000000000000..e69de29bb2d1d From b58a0ffca2b0160168d316f0d9cc88dd91e0c0e4 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 23:33:51 +0200 Subject: [PATCH 56/76] Update api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto Co-authored-by: phlax Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 89d513045e6f7..c243f08067e86 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -97,8 +97,8 @@ message IPTagging { // Common configuration for file based ip tags. message IpTagsFileProvider { // Data source from which to retrieve ip tags. - // Only filename based data source is currently supported for ip tags. - // When using this data source, if a ``watched_directory`` is provided, the ip tags file will be re-read when a file move is detected. + // Only filename based data source is currently supported for IP tags. + // When using this data source, if a ``watched_directory`` is provided, the IP tags file will be re-read when a file move is detected. // See :ref:`watched_directory ` for more information about the ``watched_directory`` field. config.core.v3.DataSource ip_tags_datasource = 1; From 587c7a7a20bb918b830722bdcfa97980230f0de1 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 17 Jun 2025 23:34:56 +0200 Subject: [PATCH 57/76] Apply suggestions from code review Co-authored-by: phlax Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/v3/ip_tagging.proto | 2 +- .../http/http_filters/ip_tagging_filter.rst | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index c243f08067e86..0e5e9a6ba9814 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -114,7 +114,7 @@ message IPTagging { // The set of IP tags for the filter. // Only one of :ref:`ip_tags ` // or :ref:`ip_tags_datasource ` - // can be set for the Ip Tagging filter. + // can be set for the IP Tagging filter. repeated IPTag ip_tags = 4; // Specify to which header the tags will be written. diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index 1521a9bf75b41..fc4ec351deec0 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -22,10 +22,10 @@ described in the paper `IP-address lookup using LC-tries `_ by S. Nilsson and G. Karlsson. -IP tags can either be provided directly using the :ref:`ip_tags ` api field or -can be loaded from file if :ref:`ip_tags_datasource ` api field is configured. -For file based ip tags *yaml* and *json* file formats are supported. -Ip tags will be dynamically reloaded if *watched_directory* is configured for :ref:`ip_tags_datasource ` +IP tags can either be provided directly using the :ref:`ip_tags ` API field or +can be loaded from file if :ref:`ip_tags_datasource ` API field is configured. +For file based IP tags YAML and JSON file formats are supported. +IP tags will be dynamically reloaded if ``watched_directory`` is configured for :ref:`ip_tags_datasource ` and :ref:`ip_tags_refresh_rate ` is set to value greater than zero. Configuration @@ -75,7 +75,7 @@ Where the *ip-tags.yaml* file would have the following content: ip_list: - {address_prefix: 1.2.3.5, prefix_len: 32} -And here is an example configuration of the filter with the file based ip tags in json format: +And here is an example configuration of the filter with the file based IP tags in JSON format: .. code-block:: yaml @@ -91,7 +91,7 @@ And here is an example configuration of the filter with the file based ip tags i watched_directory: path: "/geoip/" -Where the *ip-tags.json* file would have the following content: +Where the ``ip-tags.json`` file would have the following content: .. code-block:: json @@ -131,8 +131,8 @@ the owning HTTP connection manager. .hit, Counter, Total number of requests that have the ```` applied to it no_hit, Counter, Total number of requests with no applicable IP tags total, Counter, Total number of requests the IP Tagging Filter operated on - reload_success, Counter, Total number of successful reloads of ip tags file - reload_error, Counter, Total number of failed reloads of ip tags file + reload_success, Counter, Total number of successful reloads of IP tags file + reload_error, Counter, Total number of failed reloads of IP tags file Runtime ------- From dac7a29e22db1cf9abe02f8e4a4d700443dcf68c Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 18 Jun 2025 11:03:46 +0200 Subject: [PATCH 58/76] Increase coverage Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 24 ++++++++++++++++++- .../ip_tagging/test_data/invalid_tags.json | 1 + 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 185b76a8003d5..e21b2b53d292d 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -284,7 +284,7 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { *api_, tls_, *dispatcher_, validation_visitor_); if (expected_error.has_value()) { EXPECT_FALSE(config_or.ok()); - EXPECT_EQ(expected_error.value(), absl::StrCat(config_or.status())); + EXPECT_TRUE(absl::StrContains(absl::StrCat(config_or.status()), expected_error.value())); return; } EXPECT_TRUE(config_or.ok()); @@ -390,6 +390,28 @@ request_type: internal "tags from file /test/tags.csv"); } +TEST_F(IpTaggingFilterTest, InvalidYamlFile) { + const std::string config_yaml = R"EOF( + request_type: internal + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml" + )EOF"; + initializeFilter(config_yaml, "INVALID_ARGUMENT: failed to parse ip tags file as yaml: Unable to " + "convert YAML as JSON: ip_tags"); +} + +TEST_F(IpTaggingFilterTest, InvalidJsonFile) { + const std::string config_yaml = R"EOF( + request_type: internal + ip_tags_file_provider: + ip_tags_datasource: + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json" + )EOF"; + initializeFilter(config_yaml, "INVALID_ARGUMENT: invalid JSON in " + "envoy.extensions.filters.http.ip_tagging.v3.IPTagging.IPTagFile"); +} + TEST_F(IpTaggingFilterTest, InvalidCidr) { const std::string external_request_yaml = R"EOF( request_type: external diff --git a/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json new file mode 100644 index 0000000000000..3b12f20bec6c0 --- /dev/null +++ b/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json @@ -0,0 +1 @@ +ip_tags From 37cc20b979c2a2650834b1f5c94c7b21cab9920b Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 18 Jun 2025 14:25:39 +0200 Subject: [PATCH 59/76] Fix test Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_filter_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index e21b2b53d292d..9bb8df53af522 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -408,8 +408,7 @@ TEST_F(IpTaggingFilterTest, InvalidJsonFile) { ip_tags_datasource: filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.json" )EOF"; - initializeFilter(config_yaml, "INVALID_ARGUMENT: invalid JSON in " - "envoy.extensions.filters.http.ip_tagging.v3.IPTagging.IPTagFile"); + initializeFilter(config_yaml, "INVALID_ARGUMENT: invalid JSON"); } TEST_F(IpTaggingFilterTest, InvalidCidr) { From 05fe014555d0f6fd5cf353bdf51170bfa9bcd3f3 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 18 Jun 2025 15:12:47 +0200 Subject: [PATCH 60/76] Address review comments Signed-off-by: Kateryna Nezdolii --- source/common/protobuf/utility.h | 7 +- source/common/protobuf/yaml_utility.cc | 100 +++++++++++------- .../http/ip_tagging/ip_tagging_filter.cc | 11 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 3 +- 4 files changed, 72 insertions(+), 49 deletions(-) diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 15147cd53fe91..a6751f6ea40bc 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -268,8 +268,9 @@ class MessageUtil { bool& has_unknown_fileld); static void loadFromJson(absl::string_view json, ProtobufWkt::Struct& message); static void loadFromYaml(const std::string& yaml, Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor, - bool is_custom_thread = false); + ProtobufMessage::ValidationVisitor& validation_visitor); + static absl::Status loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validation_visitor); #endif // This function attempts to load Envoy configuration from the specified file @@ -608,7 +609,7 @@ class ValueUtil { /** * Load YAML string into ProtobufWkt::Value. */ - static ProtobufWkt::Value loadFromYaml(const std::string& yaml, bool is_custom_thread = false); + static ProtobufWkt::Value loadFromYaml(const std::string& yaml); #endif /** diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index 7df60021218c4..ac8417dc863c3 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -113,6 +113,20 @@ void jsonConvertInternal(const Protobuf::Message& source, MessageUtil::loadFromJson(json, dest, validation_visitor); } +absl::Status jsonConvertInternalNoThrow(const Protobuf::Message& source, + ProtobufMessage::ValidationVisitor& validation_visitor, + Protobuf::Message& dest) { + absl::Status conversion_status = absl::OkStatus(); + Protobuf::util::JsonPrintOptions json_options; + json_options.preserve_proto_field_names = true; + std::string json; + conversion_status = Protobuf::util::MessageToJsonString(source, &json, json_options); + if (conversion_status.ok()) { + MessageUtil::loadFromJson(json, dest, validation_visitor); + } + return conversion_status; +} + } // namespace void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message, @@ -171,9 +185,8 @@ void MessageUtil::loadFromJson(absl::string_view json, ProtobufWkt::Struct& mess } void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor, - bool is_custom_thread) { - ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml, is_custom_thread); + ProtobufMessage::ValidationVisitor& validation_visitor) { + ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml); if (value.kind_case() == ProtobufWkt::Value::kStructValue || value.kind_case() == ProtobufWkt::Value::kListValue) { jsonConvertInternal(value, validation_visitor, message); @@ -182,6 +195,38 @@ void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& messa throw EnvoyException("Unable to convert YAML as JSON: " + yaml); } +absl::Status +MessageUtil::loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validation_visitor) { + absl::Status load_status = absl::OkStatus(); + ProtobufWkt::Value value; + TRY_NEEDS_AUDIT { value = parseYamlNode(YAML::Load(yaml)); } + END_TRY + catch (YAML::ParserException& e) { + load_status = absl::InvalidArgumentError(fmt::format("Failed to parse yaml: {}", e.what())); + } + catch (YAML::BadConversion& e) { + load_status = + absl::InvalidArgumentError(fmt::format("Failed to convert to yaml: {}", e.what())); + } + catch (std::exception& e) { + // There is a potentially wide space of exceptions thrown by the YAML parser, + // and enumerating them all may be difficult. Envoy doesn't work well with + // unhandled exceptions, so we capture them and record the exception name in + // the status. + load_status = + absl::InvalidArgumentError(fmt::format("Unexpected YAML exception: {}", e.what())); + } + if (value.kind_case() == ProtobufWkt::Value::kStructValue || + value.kind_case() == ProtobufWkt::Value::kListValue) { + load_status = jsonConvertInternalNoThrow(value, validation_visitor, message); + } else { + load_status = + absl::InvalidArgumentError(fmt::format("Unable to convert YAML as JSON: {}", yaml)); + } + return load_status; +} + std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message, const bool block_print, const bool always_print_primitive_fields) { @@ -278,40 +323,21 @@ bool MessageUtil::jsonConvertValue(const Protobuf::Message& source, ProtobufWkt: return false; } -ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml, bool is_custom_thread) { - // If this is a custom (not main, worker or test) thread, skip macro validation. - if (is_custom_thread) { - TRY_NEEDS_AUDIT { return parseYamlNode(YAML::Load(yaml)); } - END_TRY - catch (YAML::ParserException& e) { - throw EnvoyException(e.what()); - } - catch (YAML::BadConversion& e) { - throw EnvoyException(e.what()); - } - catch (std::exception& e) { - // There is a potentially wide space of exceptions thrown by the YAML parser, - // and enumerating them all may be difficult. Envoy doesn't work well with - // unhandled exceptions, so we capture them and record the exception name in - // the Envoy Exception text. - throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); - } - } else { - TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); } - END_TRY - catch (YAML::ParserException& e) { - throw EnvoyException(e.what()); - } - catch (YAML::BadConversion& e) { - throw EnvoyException(e.what()); - } - catch (std::exception& e) { - // There is a potentially wide space of exceptions thrown by the YAML parser, - // and enumerating them all may be difficult. Envoy doesn't work well with - // unhandled exceptions, so we capture them and record the exception name in - // the Envoy Exception text. - throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); - } +ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) { + TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); } + END_TRY + catch (YAML::ParserException& e) { + throw EnvoyException(e.what()); + } + catch (YAML::BadConversion& e) { + throw EnvoyException(e.what()); + } + catch (std::exception& e) { + // There is a potentially wide space of exceptions thrown by the YAML parser, + // and enumerating them all may be difficult. Envoy doesn't work well with + // unhandled exceptions, so we capture them and record the exception name in + // the Envoy Exception text. + throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what())); } } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 18e02d8a7a336..7ba1ccd33c63c 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -146,13 +146,10 @@ absl::StatusOr IpTagsLoader::refreshTags() { IpTagFileProto ip_tags_proto; auto new_data = data_source_provider_->data(); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { - TRY_NEEDS_AUDIT { - MessageUtil::loadFromYaml(new_data, ip_tags_proto, validation_visitor_, - true /*is_custom_thread*/); - } - END_TRY catch (EnvoyException& ex) { - return absl::InvalidArgumentError( - fmt::format("failed to parse ip tags file as yaml: {}", ex.what())); + auto load_status = + MessageUtil::loadFromYamlNoThrow(new_data, ip_tags_proto, validation_visitor_); + if (!load_status.ok()) { + return load_status; } } else if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Json)) { bool has_unknown_field; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 9bb8df53af522..1260ce1a180f3 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -397,8 +397,7 @@ TEST_F(IpTaggingFilterTest, InvalidYamlFile) { ip_tags_datasource: filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/invalid_tags.yaml" )EOF"; - initializeFilter(config_yaml, "INVALID_ARGUMENT: failed to parse ip tags file as yaml: Unable to " - "convert YAML as JSON: ip_tags"); + initializeFilter(config_yaml, "INVALID_ARGUMENT: Unable to convert YAML as JSON: ip_tags"); } TEST_F(IpTaggingFilterTest, InvalidJsonFile) { From ece307abfb21f926313da1c649d062f64498dbb8 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Wed, 18 Jun 2025 15:27:33 +0200 Subject: [PATCH 61/76] More review comments Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 21 ++++++++++++------- .../http/ip_tagging/ip_tagging_filter.h | 5 ++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 7ba1ccd33c63c..e069deb05c4a0 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -86,13 +86,13 @@ void IpTagsProvider::updateIpTags(LcTrieSharedPtr new_tags) ABSL_LOCKS_EXCLUDED( tags_ = new_tags; } -std::shared_ptr IpTagsRegistrySingleton::getOrCreateProvider( +absl::StatusOr> IpTagsRegistrySingleton::getOrCreateProvider( const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, - Event::Dispatcher& main_dispatcher, std::shared_ptr singleton, - absl::Status& creation_status) { + Event::Dispatcher& main_dispatcher, std::shared_ptr singleton) { std::shared_ptr ip_tags_provider; + absl::Status creation_status = absl::OkStatus(); const uint64_t key = std::hash()(ip_tags_datasource.filename()); absl::MutexLock lock(&mu_); auto it = ip_tags_registry_.find(key); @@ -111,6 +111,9 @@ std::shared_ptr IpTagsRegistrySingleton::getOrCreateProvider( reload_error_cb, main_dispatcher, api, tls, singleton, creation_status); ip_tags_registry_[key] = ip_tags_provider; } + if (!creation_status.ok()) { + return creation_status; + } return ip_tags_provider; } @@ -257,12 +260,16 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( } auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.ip_tags_file_provider(), ip_tags_refresh_rate, 0); - provider_ = ip_tags_registry_->getOrCreateProvider( + auto provider_or_error = ip_tags_registry_->getOrCreateProvider( config.ip_tags_file_provider().ip_tags_datasource(), tags_loader_, ip_tags_refresh_interval_ms, [this]() { incIpTagsReloadSuccess(); }, - [this]() { incIpTagsReloadError(); }, api, tls, dispatcher, ip_tags_registry_, - creation_status); - RETURN_ONLY_IF_NOT_OK_REF(creation_status); + [this]() { incIpTagsReloadError(); }, api, tls, dispatcher, ip_tags_registry_); + if (provider_or_error.status().ok()) { + provider_ = provider_or_error.value(); + } else { + creation_status = provider_or_error.status(); + return; + } if (provider_ && provider_->ipTags()) { trie_ = provider_->ipTags(); } else { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 455bcf5b53bfc..54dea029d67f1 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -129,12 +129,11 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() {} - std::shared_ptr getOrCreateProvider( + absl::StatusOr> getOrCreateProvider( const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, - Event::Dispatcher& main_dispatcher, std::shared_ptr singleton, - absl::Status& creation_status); + Event::Dispatcher& main_dispatcher, std::shared_ptr singleton); private: absl::Mutex mu_; From a04c2f8bca9fe00c2d063db030253911386e5fb9 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 19 Jun 2025 10:22:48 +0200 Subject: [PATCH 62/76] Dummy commit with more debug info Signed-off-by: Kateryna Nezdolii --- source/common/config/datasource.cc | 11 +++++++++++ source/common/config/datasource.h | 1 + .../filters/http/ip_tagging/ip_tagging_filter.cc | 2 ++ .../filters/http/ip_tagging/ip_tagging_filter.h | 2 ++ 4 files changed, 16 insertions(+) diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index b486e3bebf5bb..fd85504611e29 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -99,17 +99,28 @@ DynamicData::~DynamicData() { } const std::string& DynamicData::data() const { + ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, + "Accessing TLS data in DataSourceProvider start"); const auto thread_local_data = slot_->get(); + ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, + "TLS data in DataSourceProvider retrieved"); return thread_local_data.has_value() ? *thread_local_data->data_ : EMPTY_STRING; } const std::string& DataSourceProvider::data() const { + ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, + "Accessing TLS data in DataSourceProvider start1"); if (absl::holds_alternative(data_)) { return absl::get(data_); } return absl::get(data_).data(); } +DataSourceProvider::~DataSourceProvider() { + ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, + "Destroying DataSourceProvider"); +}; + absl::StatusOr DataSourceProvider::create(const ProtoDataSource& source, Event::Dispatcher& main_dispatcher, ThreadLocal::SlotAllocator& tls, diff --git a/source/common/config/datasource.h b/source/common/config/datasource.h index 8a515a89eff46..d5eea94eb9576 100644 --- a/source/common/config/datasource.h +++ b/source/common/config/datasource.h @@ -88,6 +88,7 @@ class DynamicData { */ class DataSourceProvider { public: + ~DataSourceProvider(); /** * Create a DataSourceProvider from a DataSource. * @param source data source. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index e069deb05c4a0..1976bf2a9a1c1 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -117,6 +117,8 @@ absl::StatusOr> IpTagsRegistrySingleton::getOrCr return ip_tags_provider; } +IpTagsLoader::~IpTagsLoader() { ENVOY_LOG(warn, "Destroying IpTagsLoader"); }; + IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 54dea029d67f1..7659af867490c 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -37,6 +37,8 @@ class IpTagsLoader : public Logger::Loggable { IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set); + ~IpTagsLoader(); + /** * Loads file based ip tags from a data source and parses them into a trie structure. * @param ip_tags_datasource file based data source to load ip tags from. From 1c22d1e1d2e9c5a68ebedf5f68fa6da4dc10e84b Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 23 Jun 2025 18:43:05 +0200 Subject: [PATCH 63/76] Fix segfault on TLS access Signed-off-by: Kateryna Nezdolii --- source/common/config/datasource.cc | 9 ---- source/common/config/datasource.h | 1 - .../http/ip_tagging/ip_tagging_filter.cc | 43 ++++++------------- .../http/ip_tagging/ip_tagging_filter.h | 16 ++++--- .../http/ip_tagging/ip_tagging_filter_test.cc | 37 ++++++++-------- .../ip_tagging/ip_tagging_integration_test.cc | 6 ++- 6 files changed, 45 insertions(+), 67 deletions(-) diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index fd85504611e29..1b545831c0a92 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -99,8 +99,6 @@ DynamicData::~DynamicData() { } const std::string& DynamicData::data() const { - ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, - "Accessing TLS data in DataSourceProvider start"); const auto thread_local_data = slot_->get(); ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, "TLS data in DataSourceProvider retrieved"); @@ -108,19 +106,12 @@ const std::string& DynamicData::data() const { } const std::string& DataSourceProvider::data() const { - ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, - "Accessing TLS data in DataSourceProvider start1"); if (absl::holds_alternative(data_)) { return absl::get(data_); } return absl::get(data_).data(); } -DataSourceProvider::~DataSourceProvider() { - ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, - "Destroying DataSourceProvider"); -}; - absl::StatusOr DataSourceProvider::create(const ProtoDataSource& source, Event::Dispatcher& main_dispatcher, ThreadLocal::SlotAllocator& tls, diff --git a/source/common/config/datasource.h b/source/common/config/datasource.h index d5eea94eb9576..8a515a89eff46 100644 --- a/source/common/config/datasource.h +++ b/source/common/config/datasource.h @@ -88,7 +88,6 @@ class DynamicData { */ class DataSourceProvider { public: - ~DataSourceProvider(); /** * Create a DataSourceProvider from a DataSource. * @param source data source. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 1976bf2a9a1c1..1724acbd37743 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -39,10 +39,10 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag if (tags_or_error.status().ok()) { tags_ = tags_or_error.value(); } - ip_tags_reload_dispatcher_ = api.allocateDispatcher("ip_tags_reload_routine"); - ip_tags_reload_timer_ = ip_tags_reload_dispatcher_->createTimer([this]() -> void { + ip_tags_reload_timer_ = main_dispatcher.createTimer([this]() -> void { + const auto new_data = tags_loader_.getDataSourceData(); ENVOY_LOG(debug, "Trying to update ip tags in background"); - auto new_tags_or_error = tags_loader_.refreshTags(); + auto new_tags_or_error = tags_loader_.refreshTags(new_data); if (new_tags_or_error.status().ok()) { updateIpTags(new_tags_or_error.value()); reload_success_cb_(); @@ -53,28 +53,10 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag } ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); }); - - ip_tags_reload_thread_ = api.threadFactory().createThread( - [this]() -> void { - ENVOY_LOG(debug, "Started ip_tags_reload_routine"); - if (ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0)) { - ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); - } - ip_tags_reload_dispatcher_->run(Event::Dispatcher::RunType::RunUntilExit); - }, - Thread::Options{std::string("ip_tags_reload_routine")}); + ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); } -IpTagsProvider::~IpTagsProvider() { - ENVOY_LOG(debug, "Shutting down ip tags provider"); - if (ip_tags_reload_dispatcher_) { - ip_tags_reload_dispatcher_->exit(); - } - if (ip_tags_reload_thread_) { - ip_tags_reload_thread_->join(); - ip_tags_reload_thread_.reset(); - } -}; +IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); }; LcTrieSharedPtr IpTagsProvider::ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { absl::ReaderMutexLock lock(&ip_tags_mutex_); @@ -117,7 +99,10 @@ absl::StatusOr> IpTagsRegistrySingleton::getOrCr return ip_tags_provider; } -IpTagsLoader::~IpTagsLoader() { ENVOY_LOG(warn, "Destroying IpTagsLoader"); }; +const std::string& IpTagsLoader::getDataSourceData() { + data_ = data_source_provider_->data(); + return data_; +} IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) @@ -141,15 +126,15 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso } data_source_provider_ = std::move(provider_or_error.value()); ip_tags_path_ = ip_tags_datasource.filename(); - return refreshTags(); + const auto& new_data = getDataSourceData(); + return refreshTags(new_data); } return absl::InvalidArgumentError("Cannot load tags from empty filename in datasource."); } -absl::StatusOr IpTagsLoader::refreshTags() { +absl::StatusOr IpTagsLoader::refreshTags(const std::string& new_data) { if (data_source_provider_) { IpTagFileProto ip_tags_proto; - auto new_data = data_source_provider_->data(); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { auto load_status = MessageUtil::loadFromYamlNoThrow(new_data, ip_tags_proto, validation_visitor_); @@ -260,6 +245,8 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( "ip_tags_file_provider requires a valid ip_tags_datasource to be configured."); return; } + stat_name_set_->rememberBuiltin("ip_tags_reload_success"); + stat_name_set_->rememberBuiltin("ip_tags_reload_error"); auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.ip_tags_file_provider(), ip_tags_refresh_rate, 0); auto provider_or_error = ip_tags_registry_->getOrCreateProvider( @@ -277,8 +264,6 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( } else { creation_status = absl::InvalidArgumentError("Failed to get ip tags from provider"); } - stat_name_set_->rememberBuiltin("ip_tags_reload_success"); - stat_name_set_->rememberBuiltin("ip_tags_reload_error"); } } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 7659af867490c..5c63666f36fc8 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -37,8 +37,6 @@ class IpTagsLoader : public Logger::Loggable { IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set); - ~IpTagsLoader(); - /** * Loads file based ip tags from a data source and parses them into a trie structure. * @param ip_tags_datasource file based data source to load ip tags from. @@ -53,10 +51,10 @@ class IpTagsLoader : public Logger::Loggable { /** * Performs periodic refresh of file based ip tags via data source. - * @param refresh_status This status will be populated with error if refresh fails. - * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise.. + * @param new_data New data from data source which is used to refresh the ip tags structure. + * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise. */ - absl::StatusOr refreshTags(); + absl::StatusOr refreshTags(const std::string& new_data); /** * Parses ip tags in a proto format into a trie structure. @@ -68,9 +66,15 @@ class IpTagsLoader : public Logger::Loggable { parseIpTagsAsProto(const Protobuf::RepeatedPtrField< envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags); + /** + * Fetches latest data source data via TLS slot. + */ + const std::string& getDataSourceData(); + private: Api::Api& api_; Envoy::Config::DataSource::DataSourceProviderPtr data_source_provider_; + std::string data_; std::string ip_tags_path_; ProtobufMessage::ValidationVisitor& validation_visitor_; Stats::StatNameSetPtr& stat_name_set_; @@ -111,8 +115,6 @@ class IpTagsProvider : public Logger::Loggable { IpTagsReloadErrorCb reload_error_cb_; mutable absl::Mutex ip_tags_mutex_; LcTrieSharedPtr tags_ ABSL_GUARDED_BY(ip_tags_mutex_); - Thread::ThreadPtr ip_tags_reload_thread_; - Event::DispatcherPtr ip_tags_reload_dispatcher_; Event::TimerPtr ip_tags_reload_timer_; // A shared_ptr to keep the provider singleton alive as long as any of its providers are in use. const Singleton::InstanceSharedPtr owner_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 1260ce1a180f3..b77d64714b6f0 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -270,7 +270,7 @@ class IpTaggingFilterTest : public ::testing::TestWithParam { public: IpTaggingFilterTest() : scope_(stats_.rootScope()), api_(Api::createApiForTest(stats_, time_system_)), - dispatcher_(api_->allocateDispatcher("test_thread")) { + dispatcher_(api_->allocateDispatcher("test_main_thread")) { ON_CALL(runtime_.snapshot_, featureEnabled("ip_tagging.http_filter_enabled", 100)) .WillByDefault(Return(true)); } @@ -901,7 +901,7 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithReload) { fmt::format(R"EOF( request_type: internal ip_tags_file_provider: - ip_tags_refresh_rate: 5s + ip_tags_refresh_rate: 2s ip_tags_datasource: filename: "{}" watched_directory: @@ -928,7 +928,8 @@ request_type: internal )EOF", true); initializeFilter(yaml); - time_system_.advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -953,10 +954,6 @@ request_type: internal EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); - // Handle the events if any. - dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - time_system_.advanceTimeWait(std::chrono::seconds(1)); - // Update the symlink to point to the new file. TestEnvironment::renameFile( TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), @@ -967,9 +964,10 @@ request_type: internal auto contents = TestEnvironment::readFileToStringForTest( TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - time_system_.advanceTimeWait(std::chrono::seconds(6)); EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, time_system_)); @@ -997,7 +995,7 @@ request_type: internal } TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { - time_system_.advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -1008,7 +1006,7 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { fmt::format(R"EOF( request_type: internal ip_tags_file_provider: - ip_tags_refresh_rate: 5s + ip_tags_refresh_rate: 2s ip_tags_datasource: filename: "{}" watched_directory: @@ -1032,7 +1030,7 @@ ip_tags )EOF", true); initializeFilter(yaml); - time_system_.advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); Http::TestRequestHeaderMapImpl request_headers{{"x-envoy-internal", "true"}}; @@ -1057,9 +1055,9 @@ ip_tags EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); EXPECT_FALSE(request_headers.has(Http::Headers::get().EnvoyIpTags)); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - time_system_.advanceTimeWait(std::chrono::seconds(1)); // Update the symlink to point to the new file. TestEnvironment::renameFile( @@ -1071,10 +1069,9 @@ ip_tags auto contents = TestEnvironment::readFileToStringForTest( TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - - time_system_.advanceTimeWait(std::chrono::seconds(6)); EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_error", 1UL, time_system_)); @@ -1096,10 +1093,11 @@ ip_tags unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + dispatcher_->exit(); } TEST_F(IpTaggingFilterTest, IpTagsReloadedInFlightRequestsNotAffected) { - time_system_.advanceTimeWait(std::chrono::seconds(1)); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); @@ -1110,7 +1108,7 @@ TEST_F(IpTaggingFilterTest, IpTagsReloadedInFlightRequestsNotAffected) { fmt::format(R"EOF( request_type: internal ip_tags_file_provider: - ip_tags_refresh_rate: 5s + ip_tags_refresh_rate: 1s ip_tags_datasource: filename: "{}" watched_directory: @@ -1137,10 +1135,10 @@ request_type: internal )EOF", true); initializeFilter(yaml); + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(1)); EXPECT_EQ(FilterRequestType::INTERNAL, config_->requestType()); IpTaggingFilterPeer::synchronizer(filter_).enable(); std::string sync_point_name = "_trie_lookup_complete"; - // Start a thread that issues request for ip tagging filter and wait in the worker thread right // before performing lookup from the trie with ip tags. IpTaggingFilterPeer::synchronizer(filter_).waitOn(sync_point_name); @@ -1190,11 +1188,11 @@ request_type: internal auto contents = TestEnvironment::readFileToStringForTest( TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + + time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - time_system_.advanceTimeWait(std::chrono::seconds(6)); - EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, time_system_)); @@ -1204,6 +1202,7 @@ request_type: internal unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + dispatcher_->exit(); } } // namespace diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index 182fb9977676c..ed4eded156c39 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -31,8 +31,9 @@ const std::string FileBasedIpTaggingConfig = R"EOF( "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging request_type: both ip_tags_file_provider: + ip_tags_refresh_rate: 1s ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_external_request.yaml" + filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" watched_directory: path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" )EOF"; @@ -43,9 +44,10 @@ TEST_P(IpTaggingIntegrationTest, IpTaggingV3StaticTypedStructConfig) { initialize(); } -TEST_P(IpTaggingIntegrationTest, FileBasedIpTagging) { +TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { config_helper_.prependFilter(TestEnvironment::substitute(FileBasedIpTaggingConfig)); initialize(); + test_server_->waitForCounterEq("http.config_test.ip_tagging.ip_tags_reload_success", 1); } } // namespace From f07531b6cb1387c6e4b06aab8161fa492fd509c5 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 23 Jun 2025 19:10:41 +0200 Subject: [PATCH 64/76] Docs: switch to literal include Signed-off-by: Kateryna Nezdolii --- .../_include/ip-tagging-filter.yaml | 116 ++++++++++++++++++ .../http/http_filters/ip_tagging_filter.rst | 55 +++------ 2 files changed, 134 insertions(+), 37 deletions(-) create mode 100644 docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml diff --git a/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml b/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml new file mode 100644 index 0000000000000..e3ffcf6c50801 --- /dev/null +++ b/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml @@ -0,0 +1,116 @@ +static_resources: + listeners: + - address: + socket_address: + address: 0.0.0.0 + port_value: 8000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress_http + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - domains: + - '*' + name: local_service + routes: + - match: {prefix: "/"} + route: {cluster: default_service} + - address: + socket_address: + address: 0.0.0.0 + port_value: 9000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress1_http + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "/geoip/ip-tags.yaml" + watched_directory: + path: "/geoip/" + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - domains: + - '*' + name: local_service + routes: + - match: {prefix: "/"} + route: {cluster: default_service} + - address: + socket_address: + address: 0.0.0.0 + port_value: 7000 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: ingress2_http + http_filters: + - name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 5s + ip_tags_datasource: + filename: "/geoip/ip-tags.json" + watched_directory: + path: "/geoip/" + - name: envoy.filters.http.router + typed_config: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - domains: + - '*' + name: local_service + routes: + - match: {prefix: "/"} + route: {cluster: default_service} + clusters: + - name: default_service + load_assignment: + cluster_name: default_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 10001 +admin: + address: + socket_address: + address: 0.0.0.0 + port_value: 9901 + diff --git a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst index fc4ec351deec0..59191a59ea404 100644 --- a/docs/root/configuration/http/http_filters/ip_tagging_filter.rst +++ b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst @@ -35,33 +35,21 @@ Configuration An example configuration of the filter with inline ip tags may look like the following: -.. code-block:: yaml - - http_filters: - - name: ip_tagging - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging - request_type: both - ip_tags: - - ip_tag_name: external_request - ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} +.. literalinclude:: _include/ip-tagging-filter.yaml + :language: yaml + :lines: 13-21 + :lineno-start: 13 + :linenos: + :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>` Below is an example configuration of the filter with the file based ip tags in yaml format: -.. code-block:: yaml - - http_filters: - - name: ip_tagging - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging - request_type: both - ip_tags_file_provider: - ip_tags_refresh_rate: 5s - ip_tags_datasource: - filename: "/geoip/ip-tags.yaml" - watched_directory: - path: "/geoip/" +.. literalinclude:: _include/ip-tagging-filter.yaml + :language: yaml + :lines: 44-54 + :lineno-start: 44 + :linenos: + :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>` Where the *ip-tags.yaml* file would have the following content: @@ -77,19 +65,12 @@ Where the *ip-tags.yaml* file would have the following content: And here is an example configuration of the filter with the file based IP tags in JSON format: -.. code-block:: yaml - - http_filters: - - name: ip_tagging - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging - request_type: both - ip_tags_file_provider: - ip_tags_refresh_rate: 5s - ip_tags_datasource: - filename: "/geoip/ip-tags.json" - watched_directory: - path: "/geoip/" +.. literalinclude:: _include/ip-tagging-filter.yaml + :language: yaml + :lines: 77-87 + :lineno-start: 77 + :linenos: + :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>` Where the ``ip-tags.json`` file would have the following content: From 5571734250447b4dad55c13dd9179e020e4a4ab4 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Mon, 23 Jun 2025 19:12:37 +0200 Subject: [PATCH 65/76] Minor cleanup Signed-off-by: Kateryna Nezdolii --- source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 1724acbd37743..9eab8d895e0e9 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -272,7 +272,7 @@ void IpTaggingFilterConfig::incCounter(Stats::StatName name) { scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } -IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {}; +IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {} IpTaggingFilter::~IpTaggingFilter() = default; From fc06cea24924dbae17f39be553894f35c17e07a1 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 24 Jun 2025 14:56:28 +0200 Subject: [PATCH 66/76] cleanup Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_filter.cc | 14 ++++---------- .../filters/http/ip_tagging/ip_tagging_filter.h | 8 +------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 9eab8d895e0e9..ceae6aebb7023 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -40,9 +40,8 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag tags_ = tags_or_error.value(); } ip_tags_reload_timer_ = main_dispatcher.createTimer([this]() -> void { - const auto new_data = tags_loader_.getDataSourceData(); ENVOY_LOG(debug, "Trying to update ip tags in background"); - auto new_tags_or_error = tags_loader_.refreshTags(new_data); + auto new_tags_or_error = tags_loader_.refreshTags(); if (new_tags_or_error.status().ok()) { updateIpTags(new_tags_or_error.value()); reload_success_cb_(); @@ -99,11 +98,6 @@ absl::StatusOr> IpTagsRegistrySingleton::getOrCr return ip_tags_provider; } -const std::string& IpTagsLoader::getDataSourceData() { - data_ = data_source_provider_->data(); - return data_; -} - IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::StatNameSetPtr& stat_name_set) : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} @@ -126,15 +120,15 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso } data_source_provider_ = std::move(provider_or_error.value()); ip_tags_path_ = ip_tags_datasource.filename(); - const auto& new_data = getDataSourceData(); - return refreshTags(new_data); + return refreshTags(); } return absl::InvalidArgumentError("Cannot load tags from empty filename in datasource."); } -absl::StatusOr IpTagsLoader::refreshTags(const std::string& new_data) { +absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { IpTagFileProto ip_tags_proto; + const auto new_data = data_source_provider_->data(); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { auto load_status = MessageUtil::loadFromYamlNoThrow(new_data, ip_tags_proto, validation_visitor_); diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 5c63666f36fc8..11a0cf10585fb 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -54,7 +54,7 @@ class IpTagsLoader : public Logger::Loggable { * @param new_data New data from data source which is used to refresh the ip tags structure. * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise. */ - absl::StatusOr refreshTags(const std::string& new_data); + absl::StatusOr refreshTags(); /** * Parses ip tags in a proto format into a trie structure. @@ -66,15 +66,9 @@ class IpTagsLoader : public Logger::Loggable { parseIpTagsAsProto(const Protobuf::RepeatedPtrField< envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTag>& ip_tags); - /** - * Fetches latest data source data via TLS slot. - */ - const std::string& getDataSourceData(); - private: Api::Api& api_; Envoy::Config::DataSource::DataSourceProviderPtr data_source_provider_; - std::string data_; std::string ip_tags_path_; ProtobufMessage::ValidationVisitor& validation_visitor_; Stats::StatNameSetPtr& stat_name_set_; From b76dbaf9052a26d9472f02e8e9a3c7f55131dd10 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 24 Jun 2025 16:17:19 +0200 Subject: [PATCH 67/76] Filter deserves a real integration test Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter_test.cc | 8 -- .../ip_tagging/ip_tagging_integration_test.cc | 102 +++++++++++++++--- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index b77d64714b6f0..fb658cb8cec0c 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -962,9 +962,6 @@ request_type: internal TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - auto contents = TestEnvironment::readFileToStringForTest( - TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); @@ -1067,8 +1064,6 @@ ip_tags TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - auto contents = TestEnvironment::readFileToStringForTest( - TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); @@ -1186,9 +1181,6 @@ request_type: internal TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - auto contents = TestEnvironment::readFileToStringForTest( - TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index ed4eded156c39..9bbffd53f8b48 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -25,19 +25,6 @@ const std::string ExampleIpTaggingConfig = R"EOF( - {address_prefix: 1.2.3.4, prefix_len: 32} )EOF"; -const std::string FileBasedIpTaggingConfig = R"EOF( - name: ip_tagging - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging - request_type: both - ip_tags_file_provider: - ip_tags_refresh_rate: 1s - ip_tags_datasource: - filename: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data/ip_tags_internal_request.yaml" - watched_directory: - path: "{{ test_rundir }}/test/extensions/filters/http/ip_tagging/test_data" -)EOF"; - // Make sure that Envoy starts up with an ip tagging filter. TEST_P(IpTaggingIntegrationTest, IpTaggingV3StaticTypedStructConfig) { config_helper_.prependFilter(ExampleIpTaggingConfig); @@ -45,9 +32,96 @@ TEST_P(IpTaggingIntegrationTest, IpTaggingV3StaticTypedStructConfig) { } TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { - config_helper_.prependFilter(TestEnvironment::substitute(FileBasedIpTaggingConfig)); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + + envoy::config::core::v3::DataSource config; + TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); + + const std::string yaml = + fmt::format(R"EOF( + name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 1s + ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" + )EOF", + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test")); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + )EOF", + true); + config_helper_.prependFilter(TestEnvironment::substitute(yaml)); initialize(); test_server_->waitForCounterEq("http.config_test.ip_tagging.ip_tags_reload_success", 1); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "1.2.3.4"}}); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().EnvoyIpTags)[0]->value(), + "external_request"); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + timeSystem().advanceTimeWait(std::chrono::seconds(2)); + test_server_->waitForCounterGe("http.config_test.ip_tagging.ip_tags_reload_success", 2); + // Simulate file update. + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: external_updated_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + )EOF", + true); + + // Update the symlink to point to the new file. + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml")); + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + timeSystem().advanceTimeWait(std::chrono::seconds(1)); + + response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "1.2.3.4"}}); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().EnvoyIpTags)[0]->value(), + "external_updated_request"); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + // Remove the files. + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); } } // namespace From f3341c230e4d99d9fe3ea53ec4234ed16600110e Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Tue, 24 Jun 2025 16:22:12 +0200 Subject: [PATCH 68/76] More dead code Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_filter_test.cc | 3 --- .../filters/http/ip_tagging/ip_tagging_integration_test.cc | 1 - 2 files changed, 4 deletions(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index fb658cb8cec0c..465bf68ceb137 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -894,7 +894,6 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithReload) { unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); - envoy::config::core::v3::DataSource config; TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); const std::string yaml = @@ -996,7 +995,6 @@ TEST_F(IpTaggingFilterTest, InternalRequestWithFailedReloadUsesOldData) { unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); - envoy::config::core::v3::DataSource config; TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); const std::string yaml = @@ -1096,7 +1094,6 @@ TEST_F(IpTaggingFilterTest, IpTagsReloadedInFlightRequestsNotAffected) { unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); - envoy::config::core::v3::DataSource config; TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); const std::string yaml = diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index 9bbffd53f8b48..148d4cdf10c8a 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -35,7 +35,6 @@ TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); - envoy::config::core::v3::DataSource config; TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); const std::string yaml = From 6ac056f5278da1c9f0d424f423ce43cc22f66d73 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Thu, 11 Sep 2025 19:07:09 +0200 Subject: [PATCH 69/76] Fix race conditions Signed-off-by: Kateryna Nezdolii --- .../http/ip_tagging/ip_tagging_filter.cc | 71 +++++++------- .../http/ip_tagging/ip_tagging_filter.h | 50 ++++++---- .../http/ip_tagging/ip_tagging_filter_test.cc | 14 +-- .../ip_tagging/ip_tagging_integration_test.cc | 95 ++++++++++++++++++- 4 files changed, 167 insertions(+), 63 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index ceae6aebb7023..476cc33b54fbe 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -15,21 +15,25 @@ namespace HttpFilters { namespace IpTagging { IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, - IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, - IpTagsReloadSuccessCb reload_success_cb, - IpTagsReloadErrorCb reload_error_cb, + uint64_t ip_tags_refresh_interval_ms, Event::Dispatcher& main_dispatcher, Api::Api& api, - ThreadLocal::SlotAllocator& tls, Singleton::InstanceSharedPtr owner, - absl::Status& creation_status) - : ip_tags_path_(ip_tags_datasource.filename()), tags_loader_(tags_loader), + ProtobufMessage::ValidationVisitor& validation_visitor, + ThreadLocal::SlotAllocator& tls, Stats::Scope& scope, + Singleton::InstanceSharedPtr owner, absl::Status& creation_status) + : ip_tags_path_(ip_tags_datasource.filename()), scope_(scope.createScope("ip_tagging_reload.")), + stat_name_set_(scope.symbolTable().makeSet("IpTaggingReload")), + stats_prefix_(stat_name_set_->add("ip_tagging_reload")), + unknown_tag_(stat_name_set_->add("unknown_tag.hit")), tags_loader_(api, validation_visitor), time_source_(api.timeSource()), ip_tags_refresh_interval_ms_(std::chrono::milliseconds(ip_tags_refresh_interval_ms)), needs_refresh_(ip_tags_refresh_interval_ms_ > std::chrono::milliseconds(0) && ip_tags_datasource.has_watched_directory() ? true : false), - reload_success_cb_(reload_success_cb), reload_error_cb_(reload_error_cb), owner_(owner) { + owner_(owner) { RETURN_ONLY_IF_NOT_OK_REF(creation_status); + stat_name_set_->rememberBuiltin("reload_success"); + stat_name_set_->rememberBuiltin("reload_error"); if (ip_tags_datasource.filename().empty()) { creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); return; @@ -44,18 +48,23 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag auto new_tags_or_error = tags_loader_.refreshTags(); if (new_tags_or_error.status().ok()) { updateIpTags(new_tags_or_error.value()); - reload_success_cb_(); + incIpTagsReloadSuccess(); } else { ENVOY_LOG(debug, "Failed to reload ip tags, using old data: {}", new_tags_or_error.status().message()); - reload_error_cb_(); + incIpTagsReloadError(); } ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); }); ip_tags_reload_timer_->enableTimer(ip_tags_refresh_interval_ms_); } -IpTagsProvider::~IpTagsProvider() { ENVOY_LOG(debug, "Shutting down ip tags provider"); }; +IpTagsProvider::~IpTagsProvider() { + if (ip_tags_reload_timer_) { + ip_tags_reload_timer_->disableTimer(); + } + ENVOY_LOG(debug, "Shutting down ip tags provider"); +}; LcTrieSharedPtr IpTagsProvider::ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_) { absl::ReaderMutexLock lock(&ip_tags_mutex_); @@ -67,11 +76,17 @@ void IpTagsProvider::updateIpTags(LcTrieSharedPtr new_tags) ABSL_LOCKS_EXCLUDED( tags_ = new_tags; } +void IpTagsProvider::incCounter(Stats::StatName name) { + Stats::SymbolTable::StoragePtr storage = scope_->symbolTable().join({name}); + scope_->counterFromStatName(Stats::StatName(storage.get())).inc(); +} + absl::StatusOr> IpTagsRegistrySingleton::getOrCreateProvider( - const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, - uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, - IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, - Event::Dispatcher& main_dispatcher, std::shared_ptr singleton) { + const envoy::config::core::v3::DataSource& ip_tags_datasource, + uint64_t ip_tags_refresh_interval_ms, Api::Api& api, + ProtobufMessage::ValidationVisitor& validation_visitor, ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& main_dispatcher, Stats::Scope& scope, + std::shared_ptr singleton) { std::shared_ptr ip_tags_provider; absl::Status creation_status = absl::OkStatus(); const uint64_t key = std::hash()(ip_tags_datasource.filename()); @@ -82,14 +97,14 @@ absl::StatusOr> IpTagsRegistrySingleton::getOrCr ip_tags_provider = provider; } else { ip_tags_provider = std::make_shared( - ip_tags_datasource, tags_loader, ip_tags_refresh_interval_ms, reload_success_cb, - reload_error_cb, main_dispatcher, api, tls, singleton, creation_status); + ip_tags_datasource, ip_tags_refresh_interval_ms, main_dispatcher, api, validation_visitor, + tls, scope, singleton, creation_status); ip_tags_registry_[key] = ip_tags_provider; } } else { ip_tags_provider = std::make_shared( - ip_tags_datasource, tags_loader, ip_tags_refresh_interval_ms, reload_success_cb, - reload_error_cb, main_dispatcher, api, tls, singleton, creation_status); + ip_tags_datasource, ip_tags_refresh_interval_ms, main_dispatcher, api, validation_visitor, + tls, scope, singleton, creation_status); ip_tags_registry_[key] = ip_tags_provider; } if (!creation_status.ok()) { @@ -98,9 +113,8 @@ absl::StatusOr> IpTagsRegistrySingleton::getOrCr return ip_tags_provider; } -IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, - Stats::StatNameSetPtr& stat_name_set) - : api_(api), validation_visitor_(validation_visitor), stat_name_set_(stat_name_set) {} +IpTagsLoader::IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor) + : api_(api), validation_visitor_(validation_visitor) {} absl::StatusOr IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_datasource, @@ -169,7 +183,6 @@ absl::StatusOr IpTagsLoader::parseIpTagsAsProto( } } tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); - stat_name_set_->rememberBuiltin(absl::StrCat(ip_tag.ip_tag_name(), ".hit")); } return std::make_shared>(tag_data); } @@ -207,13 +220,8 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( ip_tags_registry_(singleton_manager.getTyped( SINGLETON_MANAGER_REGISTERED_NAME(ip_tags_registry), [] { return std::make_shared(); })), - tags_loader_(api, validation_visitor, stat_name_set_) { + tags_loader_(api, validation_visitor) { - // Once loading IP tags from a file system is supported, the restriction on the size - // of the set should be removed and observability into what tags are loaded needs - // to be implemented. - // TODO(ccaraman): Remove size check once file system support is implemented. - // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. if (config.ip_tags().empty() && !config.has_ip_tags_file_provider()) { creation_status = absl::InvalidArgumentError( "HTTP IP Tagging Filter requires either ip_tags or ip_tags_file_provider to be specified."); @@ -239,14 +247,11 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( "ip_tags_file_provider requires a valid ip_tags_datasource to be configured."); return; } - stat_name_set_->rememberBuiltin("ip_tags_reload_success"); - stat_name_set_->rememberBuiltin("ip_tags_reload_error"); auto ip_tags_refresh_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.ip_tags_file_provider(), ip_tags_refresh_rate, 0); auto provider_or_error = ip_tags_registry_->getOrCreateProvider( - config.ip_tags_file_provider().ip_tags_datasource(), tags_loader_, - ip_tags_refresh_interval_ms, [this]() { incIpTagsReloadSuccess(); }, - [this]() { incIpTagsReloadError(); }, api, tls, dispatcher, ip_tags_registry_); + config.ip_tags_file_provider().ip_tags_datasource(), ip_tags_refresh_interval_ms, api, + validation_visitor, tls, dispatcher, scope, ip_tags_registry_); if (provider_or_error.status().ok()) { provider_ = provider_or_error.value(); } else { diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 11a0cf10585fb..97aef5c24cea4 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -34,8 +34,7 @@ using LcTrieSharedPtr = std::shared_ptr>; */ class IpTagsLoader : public Logger::Loggable { public: - IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, - Stats::StatNameSetPtr& stat_name_set); + IpTagsLoader(Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor); /** * Loads file based ip tags from a data source and parses them into a trie structure. @@ -71,7 +70,6 @@ class IpTagsLoader : public Logger::Loggable { Envoy::Config::DataSource::DataSourceProviderPtr data_source_provider_; std::string ip_tags_path_; ProtobufMessage::ValidationVisitor& validation_visitor_; - Stats::StatNameSetPtr& stat_name_set_; }; using IpTagsReloadSuccessCb = std::function; @@ -84,9 +82,9 @@ using IpTagsReloadErrorCb = std::function; class IpTagsProvider : public Logger::Loggable { public: IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, - IpTagsLoader& tags_loader, uint64_t ip_tags_refresh_interval_ms, - IpTagsReloadSuccessCb reload_success_cb, IpTagsReloadErrorCb reload_error_cb, - Event::Dispatcher& main_dispatcher, Api::Api& api, ThreadLocal::SlotAllocator& tls, + uint64_t ip_tags_refresh_interval_ms, Event::Dispatcher& main_dispatcher, + Api::Api& api, ProtobufMessage::ValidationVisitor& validation_visitor, + ThreadLocal::SlotAllocator& tls, Stats::Scope& scope, Singleton::InstanceSharedPtr owner, absl::Status& creation_status); ~IpTagsProvider(); @@ -98,15 +96,26 @@ class IpTagsProvider : public Logger::Loggable { */ LcTrieSharedPtr ipTags() ABSL_LOCKS_EXCLUDED(ip_tags_mutex_); + void incIpTagsReloadSuccess() { + incCounter(stat_name_set_->getBuiltin("reload_success", unknown_tag_)); + } + void incIpTagsReloadError() { + incCounter(stat_name_set_->getBuiltin("reload_error", unknown_tag_)); + } + private: + void incCounter(Stats::StatName name); + const std::string ip_tags_path_; - IpTagsLoader& tags_loader_; + Stats::ScopeSharedPtr scope_; + Stats::StatNameSetPtr stat_name_set_; + const Stats::StatName stats_prefix_; + const Stats::StatName unknown_tag_; + IpTagsLoader tags_loader_; TimeSource& time_source_; MonotonicTime last_reloaded_time_; const std::chrono::milliseconds ip_tags_refresh_interval_ms_; const bool needs_refresh_; - IpTagsReloadSuccessCb reload_success_cb_; - IpTagsReloadErrorCb reload_error_cb_; mutable absl::Mutex ip_tags_mutex_; LcTrieSharedPtr tags_ ABSL_GUARDED_BY(ip_tags_mutex_); Event::TimerPtr ip_tags_reload_timer_; @@ -127,11 +136,12 @@ class IpTagsRegistrySingleton : public Envoy::Singleton::Instance { public: IpTagsRegistrySingleton() {} - absl::StatusOr> getOrCreateProvider( - const envoy::config::core::v3::DataSource& ip_tags_datasource, IpTagsLoader& tags_loader, - uint64_t ip_tags_refresh_interval_ms, IpTagsReloadSuccessCb reload_success_cb, - IpTagsReloadErrorCb reload_error_cb, Api::Api& api, ThreadLocal::SlotAllocator& tls, - Event::Dispatcher& main_dispatcher, std::shared_ptr singleton); + absl::StatusOr> + getOrCreateProvider(const envoy::config::core::v3::DataSource& ip_tags_datasource, + uint64_t ip_tags_refresh_interval_ms, Api::Api& api, + ProtobufMessage::ValidationVisitor& validation_visitor, + ThreadLocal::SlotAllocator& tls, Event::Dispatcher& main_dispatcher, + Stats::Scope& scope, std::shared_ptr singleton); private: absl::Mutex mu_; @@ -180,16 +190,14 @@ class IpTaggingFilterConfig { HeaderAction ipTagHeaderAction() const { return ip_tag_header_action_; } void incHit(absl::string_view tag) { + const Stats::StatName tag_counter = + stat_name_set_->getBuiltin(absl::StrCat(tag, ".hit"), unknown_tag_); + if (tag_counter == unknown_tag_) { + stat_name_set_->rememberBuiltin(absl::StrCat(tag, ".hit")); + } incCounter(stat_name_set_->getBuiltin(absl::StrCat(tag, ".hit"), unknown_tag_)); } - void incIpTagsReloadSuccess() { - incCounter(stat_name_set_->getBuiltin("ip_tags_reload_success", unknown_tag_)); - } - void incIpTagsReloadError() { - incCounter(stat_name_set_->getBuiltin("ip_tags_reload_error", unknown_tag_)); - } - void incNoHit() { incCounter(no_hit_); } void incTotal() { incCounter(total_); } diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 465bf68ceb137..2ec5060251459 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -965,10 +965,10 @@ request_type: internal // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, - time_system_)); + EXPECT_TRUE( + TestUtility::waitForCounterEq(stats_, "ip_tagging_reload.reload_success", 1UL, time_system_)); - EXPECT_EQ(stats_.counterFromString("prefix.ip_tagging.ip_tags_reload_error").value(), 0); + EXPECT_EQ(stats_.counterFromString("ip_tagging_reload.reload_error").value(), 0); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); @@ -1065,8 +1065,8 @@ ip_tags time_system_.advanceTimeAsyncImpl(std::chrono::seconds(6)); // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_error", 1UL, - time_system_)); + EXPECT_TRUE( + TestUtility::waitForCounterEq(stats_, "ip_tagging_reload.reload_error", 1UL, time_system_)); filter_callbacks_.stream_info_.downstream_connection_info_provider_->setRemoteAddress( remote_address); @@ -1182,8 +1182,8 @@ request_type: internal // Handle the events if any. dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - EXPECT_TRUE(TestUtility::waitForCounterEq(stats_, "prefix.ip_tagging.ip_tags_reload_success", 1UL, - time_system_)); + EXPECT_TRUE( + TestUtility::waitForCounterEq(stats_, "ip_tagging_reload.reload_success", 1UL, time_system_)); IpTaggingFilterPeer::synchronizer(filter_).signal(sync_point_name); t0.join(); diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index 148d4cdf10c8a..c0871b75979f3 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -63,7 +63,7 @@ TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { true); config_helper_.prependFilter(TestEnvironment::substitute(yaml)); initialize(); - test_server_->waitForCounterEq("http.config_test.ip_tagging.ip_tags_reload_success", 1); + test_server_->waitForCounterEq("ip_tagging_reload.reload_success", 1); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -82,7 +82,7 @@ TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { ASSERT_TRUE(response->complete()); EXPECT_EQ("200", response->headers().getStatusValue()); timeSystem().advanceTimeWait(std::chrono::seconds(2)); - test_server_->waitForCounterGe("http.config_test.ip_tagging.ip_tags_reload_success", 2); + test_server_->waitForCounterGe("ip_tagging_reload.reload_success", 2); // Simulate file update. TestEnvironment::writeStringToFileForTest( TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( @@ -123,5 +123,96 @@ TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); } +TEST_P(IpTaggingIntegrationTest, IptaggingFilterWithReloadNoCrashOnLdsUpdate) { + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); + + TestEnvironment::createPath(TestEnvironment::temporaryPath("ip_tagging_test")); + + const std::string yaml = + fmt::format(R"EOF( + name: ip_tagging + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging + request_type: both + ip_tags_file_provider: + ip_tags_refresh_rate: 1s + ip_tags_datasource: + filename: "{}" + watched_directory: + path: "{}" + )EOF", + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test")); + + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: external_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + )EOF", + true); + config_helper_.prependFilter(TestEnvironment::substitute(yaml)); + initialize(); + test_server_->waitForCounterEq("ip_tagging_reload.reload_success", 1); + + // LDS update to modify the listener and corresponding drain. + { + ConfigHelper new_config_helper(version_, config_helper_.bootstrap()); + new_config_helper.addConfigModifier( + [](envoy::config::bootstrap::v3::Bootstrap& bootstrap) -> void { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->mutable_listener_filters_timeout()->set_seconds(10); + }); + new_config_helper.setLds("1"); + test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 1); + test_server_->waitForCounterEq("listener_manager.lds.update_success", 2); + test_server_->waitForGaugeEq("listener_manager.total_listeners_draining", 0); + } + timeSystem().advanceTimeWait(std::chrono::seconds(2)); + test_server_->waitForCounterGe("ip_tagging_reload.reload_success", 2); + + // Simulate file update. + TestEnvironment::writeStringToFileForTest( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), R"EOF( +ip_tags: + - ip_tag_name: external_updated_request + ip_list: + - {address_prefix: 1.2.3.4, prefix_len: 32} + )EOF", + true); + + // Update the symlink to point to the new file. + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml")); + TestEnvironment::renameFile( + TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), + TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); + timeSystem().advanceTimeWait(std::chrono::seconds(1)); + test_server_->waitForCounterGe("ip_tagging_reload.reload_success", 3); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestRequestHeaderMapImpl{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"x-forwarded-for", "1.2.3.4"}}); + + waitForNextUpstreamRequest(); + EXPECT_EQ(upstream_request_->headers().get(Http::Headers::get().EnvoyIpTags)[0]->value(), + "external_updated_request"); + upstream_request_->encodeHeaders(Http::TestResponseHeaderMapImpl{{":status", "200"}}, true); + ASSERT_TRUE(response->waitForEndStream()); + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().getStatusValue()); + + // Remove the files. + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_old_target.yaml").c_str()); + unlink(TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml").c_str()); +} + } // namespace } // namespace Envoy From a954ca798c1d91dc2fee02dfb439b76a7b8363d8 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 12 Sep 2025 12:12:34 +0200 Subject: [PATCH 70/76] fix prechecks Signed-off-by: Kateryna Nezdolii --- .../http/http_filters/_include/ip-tagging-filter.yaml | 3 +-- source/common/protobuf/yaml_utility.cc | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml b/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml index e3ffcf6c50801..0b47430163ea8 100644 --- a/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml +++ b/docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml @@ -18,7 +18,7 @@ static_resources: ip_tags: - ip_tag_name: external_request ip_list: - - {address_prefix: 1.2.3.4, prefix_len: 32} + - {address_prefix: 1.2.3.4, prefix_len: 32} - name: envoy.filters.http.router typed_config: '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -113,4 +113,3 @@ admin: socket_address: address: 0.0.0.0 port_value: 9901 - diff --git a/source/common/protobuf/yaml_utility.cc b/source/common/protobuf/yaml_utility.cc index 2bff51a4def22..0512bfac09afd 100644 --- a/source/common/protobuf/yaml_utility.cc +++ b/source/common/protobuf/yaml_utility.cc @@ -199,7 +199,7 @@ absl::Status MessageUtil::loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { absl::Status load_status = absl::OkStatus(); - ProtobufWkt::Value value; + Protobuf::Value value; TRY_NEEDS_AUDIT { value = parseYamlNode(YAML::Load(yaml)); } END_TRY catch (YAML::ParserException& e) { @@ -217,8 +217,8 @@ MessageUtil::loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& mes load_status = absl::InvalidArgumentError(fmt::format("Unexpected YAML exception: {}", e.what())); } - if (value.kind_case() == ProtobufWkt::Value::kStructValue || - value.kind_case() == ProtobufWkt::Value::kListValue) { + if (value.kind_case() == Protobuf::Value::kStructValue || + value.kind_case() == Protobuf::Value::kListValue) { load_status = jsonConvertInternalNoThrow(value, validation_visitor, message); } else { load_status = From 3c9dfd109ffe0e64299e07a403077907ff7540b6 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 12 Sep 2025 14:17:08 +0200 Subject: [PATCH 71/76] cleanup Signed-off-by: Kateryna Nezdolii --- docs/root/api-v3/data/data.rst | 1 - source/extensions/filters/http/ip_tagging/ip_tagging_filter.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/docs/root/api-v3/data/data.rst b/docs/root/api-v3/data/data.rst index de05d89099e86..2a4a2fea05e17 100644 --- a/docs/root/api-v3/data/data.rst +++ b/docs/root/api-v3/data/data.rst @@ -10,4 +10,3 @@ Envoy data core/core dns/dns tap/tap - diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 97aef5c24cea4..16e17e5e5b675 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -41,7 +41,6 @@ class IpTagsLoader : public Logger::Loggable { * @param ip_tags_datasource file based data source to load ip tags from. * @param dispatcher The dispatcher for the thread used by a data source provider. * @param tls The thread local slot allocator used by a data source provider. - * @param creation_status This status will be populated with error if loading fails. * @return Valid LcTrieSharedPtr if loading succeeded or error status otherwise. */ absl::StatusOr @@ -58,7 +57,6 @@ class IpTagsLoader : public Logger::Loggable { /** * Parses ip tags in a proto format into a trie structure. * @param ip_tags Collection of ip tags in proto format. - * @param creation_status This status will be populated with error if parsing fails. * @return Valid LcTrieSharedPtr if parsing succeeded or error status otherwise. */ absl::StatusOr From e01e5635b73d368ee29b565ca410af1035ea8e48 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 12 Sep 2025 15:36:45 +0200 Subject: [PATCH 72/76] Try build ip tagging example in docs Signed-off-by: Kateryna Nezdolii --- docs/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/BUILD b/docs/BUILD index 2e317b672a110..21211250520af 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -33,6 +33,7 @@ filegroup( "root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml", "root/configuration/overview/_include/xds_api/oauth-sds-example.yaml", "root/configuration/security/_include/sds-source-example.yaml", + "root/configuration/http/http_filters/_include/ip-tagging-filter.yaml" ], ) + select({ "//bazel:windows_x86_64": [], From 5893004961a3771633b82536ed58e5a69e7a3564 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 12 Sep 2025 17:12:20 +0200 Subject: [PATCH 73/76] Format doc BUILD file Signed-off-by: Kateryna Nezdolii --- docs/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/BUILD b/docs/BUILD index 21211250520af..b5ff3085c1635 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -33,7 +33,7 @@ filegroup( "root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml", "root/configuration/overview/_include/xds_api/oauth-sds-example.yaml", "root/configuration/security/_include/sds-source-example.yaml", - "root/configuration/http/http_filters/_include/ip-tagging-filter.yaml" + "root/configuration/http/http_filters/_include/ip-tagging-filter.yaml", ], ) + select({ "//bazel:windows_x86_64": [], From d5bfaf0f615c99cc89af5ed61962449dd95cf7a6 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Sun, 14 Sep 2025 11:33:16 +0200 Subject: [PATCH 74/76] Increase cov Signed-off-by: Kateryna Nezdolii --- source/common/config/datasource.cc | 2 -- .../extensions/filters/http/ip_tagging/ip_tagging_filter.cc | 4 ---- .../filters/http/ip_tagging/ip_tagging_filter_test.cc | 3 ++- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/source/common/config/datasource.cc b/source/common/config/datasource.cc index 1b545831c0a92..b486e3bebf5bb 100644 --- a/source/common/config/datasource.cc +++ b/source/common/config/datasource.cc @@ -100,8 +100,6 @@ DynamicData::~DynamicData() { const std::string& DynamicData::data() const { const auto thread_local_data = slot_->get(); - ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::config), warn, - "TLS data in DataSourceProvider retrieved"); return thread_local_data.has_value() ? *thread_local_data->data_ : EMPTY_STRING; } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 476cc33b54fbe..fab1a4afec5f6 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -34,10 +34,6 @@ IpTagsProvider::IpTagsProvider(const envoy::config::core::v3::DataSource& ip_tag RETURN_ONLY_IF_NOT_OK_REF(creation_status); stat_name_set_->rememberBuiltin("reload_success"); stat_name_set_->rememberBuiltin("reload_error"); - if (ip_tags_datasource.filename().empty()) { - creation_status = absl::InvalidArgumentError("Cannot load tags from empty file path."); - return; - } auto tags_or_error = tags_loader_.loadTags(ip_tags_datasource, main_dispatcher, tls); creation_status = tags_or_error.status(); if (tags_or_error.status().ok()) { diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 2ec5060251459..92d2ac160d05a 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -376,7 +376,8 @@ request_type: internal ip_tags_datasource: filename: )EOF"; - initializeFilter(config_yaml, "INVALID_ARGUMENT: Cannot load tags from empty file path."); + initializeFilter(config_yaml, + "INVALID_ARGUMENT: Cannot load tags from empty filename in datasource."); } TEST_F(IpTaggingFilterTest, UnsupportedFormatForIpTagsFile) { From 2186be8847de4c0b794d8bc5ca74d187ee028cc7 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 19 Sep 2025 23:21:43 +0200 Subject: [PATCH 75/76] fix test flake Signed-off-by: Kateryna Nezdolii --- .../filters/http/ip_tagging/ip_tagging_integration_test.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc index c0871b75979f3..6404efe5693ed 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_integration_test.cc @@ -100,7 +100,8 @@ TEST_P(IpTaggingIntegrationTest, FileBasedIpTaggingWithReload) { TestEnvironment::renameFile( TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - timeSystem().advanceTimeWait(std::chrono::seconds(1)); + timeSystem().advanceTimeWait(std::chrono::seconds(2)); + test_server_->waitForCounterGe("ip_tagging_reload.reload_success", 3); response = codec_client_->makeHeaderOnlyRequest( Http::TestRequestHeaderMapImpl{{":method", "GET"}, @@ -190,7 +191,7 @@ TEST_P(IpTaggingIntegrationTest, IptaggingFilterWithReloadNoCrashOnLdsUpdate) { TestEnvironment::renameFile( TestEnvironment::temporaryPath("ip_tagging_test/watcher_new_target.yaml"), TestEnvironment::temporaryPath("ip_tagging_test/watcher_target.yaml")); - timeSystem().advanceTimeWait(std::chrono::seconds(1)); + timeSystem().advanceTimeWait(std::chrono::seconds(3)); test_server_->waitForCounterGe("ip_tagging_reload.reload_success", 3); codec_client_ = makeHttpConnection(lookupPort("http")); auto response = codec_client_->makeHeaderOnlyRequest( From 75cb49443f9c4a4f10062c47a3c75e097aafd3f7 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 19 Sep 2025 23:46:50 +0200 Subject: [PATCH 76/76] api feedback Signed-off-by: Kateryna Nezdolii --- .../extensions/filters/http/ip_tagging/v3/ip_tagging.proto | 2 +- source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc | 2 +- source/extensions/filters/http/ip_tagging/ip_tagging_filter.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto index 0e5e9a6ba9814..292ae686cb259 100644 --- a/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto +++ b/api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto @@ -58,7 +58,7 @@ message IPTagging { // Specifies the content of the IP tag file. // Allow the file to be created with no IP tags. - message IPTagFile { + message IPTags { repeated IPTag ip_tags = 1; } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index fab1a4afec5f6..2a763cf781897 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -137,7 +137,7 @@ IpTagsLoader::loadTags(const envoy::config::core::v3::DataSource& ip_tags_dataso absl::StatusOr IpTagsLoader::refreshTags() { if (data_source_provider_) { - IpTagFileProto ip_tags_proto; + IPTagsProto ip_tags_proto; const auto new_data = data_source_provider_->data(); if (absl::EndsWith(ip_tags_path_, MessageUtil::FileExtensions::get().Yaml)) { auto load_status = diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 16e17e5e5b675..35d20f31b7cab 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -25,7 +25,7 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { -using IpTagFileProto = envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTagFile; +using IPTagsProto = envoy::extensions::filters::http::ip_tagging::v3::IPTagging::IPTags; using LcTrieSharedPtr = std::shared_ptr>; /**