From cd1b7b282e8bef7a063a250808a8382ba25b323b Mon Sep 17 00:00:00 2001 From: Nicolai Antiferov Date: Thu, 16 Oct 2025 21:08:37 +0300 Subject: [PATCH] Feature: Add metrics collection for valkey modules metrics Disclaimer: I don't use valkey modules, metrics are collected based on docker image run output and metric types are suggested based on documentation, naming and code. * Add metrics collection for valkey modules metrics * Add tests suite for valkey modules metrics collection * Drop redis-stack (v7.4) modules metrics collection test suite as it reached EOL * Add comments for redis-stack specific metrics * Remove redis-stack / TEST_REDIS_MODULES_URI * Replace TEST_REDIS_MODULES_URI with `TEST_VALKEY8_BUNDLE_URI` in TestSimultaneousMetricsHttpRequests() fix: #1049 and #1050 --- Makefile | 1 - docker-compose.yml | 5 -- exporter/exporter.go | 52 ++++++++++++++-- exporter/http_test.go | 4 +- exporter/modules_test.go | 131 +++++++++++++++++++++++---------------- 5 files changed, 128 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index d28564a3..2403eb2d 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,6 @@ test: TEST_VALKEY_CLUSTER_PASSWORD_URI="redis://localhost:17006" \ TEST_TILE38_URI="redis://localhost:19851" \ TEST_VALKEY_SENTINEL_URI="redis://localhost:26379" \ - TEST_REDIS_MODULES_URI="redis://localhost:36379" \ go test -v -covermode=atomic -cover -race -coverprofile=coverage.txt -p 1 ./... .PHONY: lint diff --git a/docker-compose.yml b/docker-compose.yml index 50c33aaa..b9a0b96a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,8 +136,3 @@ services: image: tile38/tile38:latest ports: - "19851:9851" - - redis-stack: - image: redis/redis-stack-server:7.4.0-v0 - ports: - - "36379:6379" diff --git a/exporter/exporter.go b/exporter/exporter.go index dd6bc06f..3bc09660 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -313,13 +313,14 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) { // Redis Modules metrics, RediSearch module "search_number_of_indexes": "search_number_of_indexes", "search_used_memory_indexes": "search_used_memory_indexes_bytes", - "search_global_idle": "search_global_idle", - "search_global_total": "search_global_total", - "search_bytes_collected": "search_collected_bytes", "search_dialect_1": "search_dialect_1", "search_dialect_2": "search_dialect_2", "search_dialect_3": "search_dialect_3", "search_dialect_4": "search_dialect_4", + // Legacy redis-stack v7.4 metrics + "search_global_idle": "search_global_idle", + "search_global_total": "search_global_total", + "search_bytes_collected": "search_collected_bytes", // RediSearch module v8.0 "search_number_of_active_indexes": "search_number_of_active_indexes", "search_number_of_active_indexes_running_queries": "search_number_of_active_indexes_running_queries", @@ -336,6 +337,22 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) { "search_gc_total_docs_not_collected": "search_gc_total_docs_not_collected", "search_gc_marked_deleted_vectors": "search_gc_marked_deleted_vectors", "search_errors_indexing_failures": "search_errors_indexing_failures", + // Valkey v8 metrics + "bf_bloom_total_memory_bytes": "bf_bloom_total_memory_bytes", + "bf_bloom_num_objects": "bf_bloom_num_objects", + "bf_bloom_num_filters_across_objects": "bf_bloom_num_filters_across_objects", + "bf_bloom_num_items_across_objects": "bf_bloom_num_items_across_objects", + "bf_bloom_capacity_across_objects": "bf_bloom_capacity_across_objects", + "json_total_memory_bytes": "json_total_memory_bytes", + "json_num_documents": "json_num_documents", + "search_used_memory_bytes": "search_used_memory_bytes", + "search_number_of_attributes": "search_number_of_attributes", + "search_total_indexed_documents": "search_total_indexed_documents", + "search_query_queue_size": "search_query_queue_size", + "search_writer_queue_size": "search_writer_queue_size", + "search_string_interning_store_size": "search_string_interning_store_size", + "search_vector_externing_hash_extern_errors": "search_vector_externing_hash_extern_errors", + "search_vector_externing_num_lru_entries": "search_vector_externing_num_lru_entries", }, metricMapCounters: map[string]string{ @@ -390,8 +407,9 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) { // Redis Modules metrics, RediSearch module "search_total_indexing_time": "search_indexing_time_ms_total", - "search_total_cycles": "search_cycles_total", - "search_total_ms_run": "search_run_ms_total", + // Legacy redis-stack v7.4 metrics + "search_total_cycles": "search_cycles_total", + "search_total_ms_run": "search_run_ms_total", // RediSearch module v8.0 "search_gc_total_cycles": "search_gc_cycles_total", // search_gc metrics were renamed "search_gc_total_ms_run": "search_gc_run_ms_total", // in PR: https://github.com/RediSearch/RediSearch/pull/5616 @@ -399,6 +417,30 @@ func NewRedisExporter(uri string, opts Options) (*Exporter, error) { "search_total_query_commands": "search_query_commands_total", "search_total_query_execution_time_ms": "search_query_execution_time_ms_total", "search_total_active_queries": "search_active_queries_total", + // Valkey v8 metrics + "bf_bloom_defrag_hits": "bf_bloom_defrag_hits_total", + "bf_bloom_defrag_misses": "bf_bloom_defrag_misses_total", + "search_worker_pool_suspend_cnt": "search_worker_pool_suspend_count", + "search_writer_resumed_cnt": "search_writer_resumed_count", + "search_reader_resumed_cnt": "search_reader_resumed_count", + "search_writer_suspension_expired_cnt": "search_writer_suspension_expired_count", + "search_rdb_load_success_cnt": "search_rdb_load_success_count", + "search_rdb_load_failure_cnt": "search_rdb_load_failure_count", + "search_rdb_save_success_cnt": "search_rdb_save_success_count", + "search_rdb_save_failure_cnt": "search_rdb_save_failure_count", + "search_successful_requests_count": "search_successful_requests_count", + "search_failure_requests_count": "search_failure_requests_count", + "search_hybrid_requests_count": "search_hybrid_requests_count", + "search_inline_filtering_requests_count": "search_inline_filtering_requests_count", + "search_hnsw_add_exceptions_count": "search_hnsw_add_exceptions_count", + "search_hnsw_remove_exceptions_count": "search_hnsw_remove_exceptions_count", + "search_hnsw_modify_exceptions_count": "search_hnsw_modify_exceptions_count", + "search_hnsw_search_exceptions_count": "search_hnsw_search_exceptions_count", + "search_hnsw_create_exceptions_count": "search_hnsw_create_exceptions_count", + "search_vector_externing_entry_count": "search_vector_externing_entry_count", + "search_vector_externing_generated_value_cnt": "search_vector_externing_generated_value_count", + "search_vector_externing_lru_promote_cnt": "search_vector_externing_lru_promote_count", + "search_vector_externing_deferred_entry_cnt": "search_vector_externing_deferred_entry_count", }, } diff --git a/exporter/http_test.go b/exporter/http_test.go index 3ed3de36..eec0bacf 100644 --- a/exporter/http_test.go +++ b/exporter/http_test.go @@ -211,7 +211,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" || os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" || os.Getenv("TEST_TILE38_URI") == "" || - os.Getenv("TEST_REDIS_MODULES_URI") == "" { + os.Getenv("TEST_VALKEY8_BUNDLE_URI") == "" { t.Skipf("Skipping TestSimultaneousMetricsHttpRequests, missing env vars") } @@ -236,7 +236,7 @@ func TestSimultaneousMetricsHttpRequests(t *testing.T) { os.Getenv("TEST_REDIS5_URI"), os.Getenv("TEST_REDIS6_URI"), - os.Getenv("TEST_REDIS_MODULES_URI"), + os.Getenv("TEST_VALKEY8_BUNDLE_URI"), // tile38 & Cluster need to be last in this list, so we can identify them when selected, down in line 229 os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI"), diff --git a/exporter/modules_test.go b/exporter/modules_test.go index 9116b461..a7048fb1 100644 --- a/exporter/modules_test.go +++ b/exporter/modules_test.go @@ -8,9 +8,9 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func TestModulesv74(t *testing.T) { - if os.Getenv("TEST_REDIS_MODULES_URI") == "" { - t.Skipf("TEST_REDIS_MODULES_URI not set - skipping") +func TestModulesv80(t *testing.T) { + if os.Getenv("TEST_REDIS8_URI") == "" || os.Getenv("TEST_REDIS_URI") == "" { + t.Skipf("TEST_REDIS8_URI or TEST_REDIS_URI aren't set - skipping") } tsts := []struct { @@ -18,8 +18,8 @@ func TestModulesv74(t *testing.T) { inclModulesMetrics bool wantModulesMetrics bool }{ - {addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: true, wantModulesMetrics: true}, - {addr: os.Getenv("TEST_REDIS_MODULES_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, + {addr: os.Getenv("TEST_REDIS8_URI"), inclModulesMetrics: true, wantModulesMetrics: true}, + {addr: os.Getenv("TEST_REDIS8_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: true, wantModulesMetrics: false}, {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, } @@ -34,19 +34,35 @@ func TestModulesv74(t *testing.T) { }() wantedMetrics := map[string]bool{ - "module_info": false, - "search_number_of_indexes": false, - "search_used_memory_indexes_bytes": false, - "search_indexing_time_ms_total": false, - "search_global_idle": false, - "search_global_total": false, - "search_collected_bytes": false, - "search_cycles_total": false, - "search_run_ms_total": false, - "search_dialect_1": false, - "search_dialect_2": false, - "search_dialect_3": false, - "search_dialect_4": false, + "module_info": false, + "search_number_of_indexes": false, + "search_used_memory_indexes_bytes": false, + "search_indexing_time_ms_total": false, + "search_dialect_1": false, + "search_dialect_2": false, + "search_dialect_3": false, + "search_dialect_4": false, + "search_number_of_active_indexes": false, + "search_number_of_active_indexes_running_queries": false, + "search_number_of_active_indexes_indexing": false, + "search_total_active_write_threads": false, + "search_smallest_memory_index_bytes": false, + "search_largest_memory_index_bytes": false, + "search_used_memory_vector_index_bytes": false, + "search_global_idle_user": false, + "search_global_idle_internal": false, + "search_global_total_user": false, + "search_global_total_internal": false, + "search_gc_collected_bytes": false, + "search_gc_total_docs_not_collected": false, + "search_gc_marked_deleted_vectors": false, + "search_errors_indexing_failures": false, + "search_gc_cycles_total": false, + "search_gc_run_ms_total": false, + "search_queries_processed_total": false, + "search_query_commands_total": false, + "search_query_execution_time_ms_total": false, + "search_active_queries_total": false, } for m := range chM { @@ -73,9 +89,9 @@ func TestModulesv74(t *testing.T) { } } -func TestModulesv80(t *testing.T) { - if os.Getenv("TEST_REDIS8_URI") == "" || os.Getenv("TEST_REDIS_URI") == "" { - t.Skipf("TEST_REDIS8_URI or TEST_REDIS_URI aren't set - skipping") +func TestModulesValkey(t *testing.T) { + if os.Getenv("TEST_VALKEY8_BUNDLE_URI") == "" || os.Getenv("TEST_REDIS_URI") == "" { + t.Skipf("TEST_VALKEY8_BUNDLE_URI or TEST_REDIS_URI aren't set - skipping") } tsts := []struct { @@ -83,8 +99,8 @@ func TestModulesv80(t *testing.T) { inclModulesMetrics bool wantModulesMetrics bool }{ - {addr: os.Getenv("TEST_REDIS8_URI"), inclModulesMetrics: true, wantModulesMetrics: true}, - {addr: os.Getenv("TEST_REDIS8_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, + {addr: os.Getenv("TEST_VALKEY8_BUNDLE_URI"), inclModulesMetrics: true, wantModulesMetrics: true}, + {addr: os.Getenv("TEST_VALKEY8_BUNDLE_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: true, wantModulesMetrics: false}, {addr: os.Getenv("TEST_REDIS_URI"), inclModulesMetrics: false, wantModulesMetrics: false}, } @@ -99,35 +115,46 @@ func TestModulesv80(t *testing.T) { }() wantedMetrics := map[string]bool{ - "module_info": false, - "search_number_of_indexes": false, - "search_used_memory_indexes_bytes": false, - "search_indexing_time_ms_total": false, - "search_dialect_1": false, - "search_dialect_2": false, - "search_dialect_3": false, - "search_dialect_4": false, - "search_number_of_active_indexes": false, - "search_number_of_active_indexes_running_queries": false, - "search_number_of_active_indexes_indexing": false, - "search_total_active_write_threads": false, - "search_smallest_memory_index_bytes": false, - "search_largest_memory_index_bytes": false, - "search_used_memory_vector_index_bytes": false, - "search_global_idle_user": false, - "search_global_idle_internal": false, - "search_global_total_user": false, - "search_global_total_internal": false, - "search_gc_collected_bytes": false, - "search_gc_total_docs_not_collected": false, - "search_gc_marked_deleted_vectors": false, - "search_errors_indexing_failures": false, - "search_gc_cycles_total": false, - "search_gc_run_ms_total": false, - "search_queries_processed_total": false, - "search_query_commands_total": false, - "search_query_execution_time_ms_total": false, - "search_active_queries_total": false, + "module_info": false, + "search_number_of_indexes": false, + "bf_bloom_total_memory_bytes": false, + "bf_bloom_num_objects": false, + "bf_bloom_num_filters_across_objects": false, + "bf_bloom_num_items_across_objects": false, + "bf_bloom_capacity_across_objects": false, + "json_total_memory_bytes": false, + "json_num_documents": false, + "search_used_memory_bytes": false, + "search_number_of_attributes": false, + "search_total_indexed_documents": false, + "search_query_queue_size": false, + "search_writer_queue_size": false, + "search_string_interning_store_size": false, + "search_vector_externing_hash_extern_errors": false, + "search_vector_externing_num_lru_entries": false, + "bf_bloom_defrag_hits_total": false, + "bf_bloom_defrag_misses_total": false, + "search_worker_pool_suspend_count": false, + "search_writer_resumed_count": false, + "search_reader_resumed_count": false, + "search_writer_suspension_expired_count": false, + "search_rdb_load_success_count": false, + "search_rdb_load_failure_count": false, + "search_rdb_save_success_count": false, + "search_rdb_save_failure_count": false, + "search_successful_requests_count": false, + "search_failure_requests_count": false, + "search_hybrid_requests_count": false, + "search_inline_filtering_requests_count": false, + "search_hnsw_add_exceptions_count": false, + "search_hnsw_remove_exceptions_count": false, + "search_hnsw_modify_exceptions_count": false, + "search_hnsw_search_exceptions_count": false, + "search_hnsw_create_exceptions_count": false, + "search_vector_externing_entry_count": false, + "search_vector_externing_generated_value_count": false, + "search_vector_externing_lru_promote_count": false, + "search_vector_externing_deferred_entry_count": false, } for m := range chM {