Skip to content

Commit bfd9d4b

Browse files
oliver006oliver006
committed
wip
1 parent 6288963 commit bfd9d4b

File tree

2 files changed

+126
-101
lines changed

2 files changed

+126
-101
lines changed

‎exporter/redis.go‎

Lines changed: 117 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,22 @@ func (e *Exporter) ScrapeHandler(w http.ResponseWriter, r *http.Request) {
172172
return
173173
}
174174

175-
// todo: allow passing check-keys?
176-
177175
start := time.Now()
178-
exp, _ := NewRedisExporter(target, e.options)
176+
177+
// todo: this needs a test
178+
checkKeys := r.URL.Query().Get("check-keys")
179+
checkSingleKey := r.URL.Query().Get("check-single-keys")
180+
181+
opts := e.options
182+
opts.CheckKeys = checkKeys
183+
opts.CheckSingleKeys = checkSingleKey
184+
185+
exp, err := NewRedisExporter(target, opts)
186+
if err != nil {
187+
http.Error(w, "NewRedisExporter() err: err", 400)
188+
e.targetScrapeRequestErrors.Inc()
189+
return
190+
}
179191
registry := prometheus.NewRegistry()
180192
registry.MustRegister(exp)
181193
h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
@@ -268,11 +280,10 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
268280
metricMapGauges["total_system_memory"] = "total_system_memory_bytes"
269281
}
270282

271-
l := []string{}
272283
e.metricDescriptions = map[string]*prometheus.Desc{}
273-
e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", l)
284+
e.metricDescriptions["up"] = newMetricDescr(opts.Namespace, "up", "Information about the Redis instance", nil)
274285
e.metricDescriptions["instance_info"] = newMetricDescr(opts.Namespace, "instance_info", "Information about the Redis instance", []string{"role", "redis_version", "redis_build_id", "redis_mode", "os"})
275-
e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", l)
286+
e.metricDescriptions["last_scrape_duration"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_duration_seconds", "The last scrape duration", nil)
276287
e.metricDescriptions["scrape_error"] = newMetricDescr(opts.Namespace, "exporter_last_scrape_error", "The last scrape error status.", []string{"err"})
277288

278289
e.metricDescriptions["script_values"] = newMetricDescr(opts.Namespace, "script_value", "Values returned by the collect script", []string{"key"})
@@ -281,17 +292,17 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
281292

282293
e.metricDescriptions["commands_total"] = newMetricDescr(opts.Namespace, "commands_total", `Total number of calls per command`, []string{"cmd"})
283294
e.metricDescriptions["commands_duration_seconds_total"] = newMetricDescr(opts.Namespace, "commands_duration_seconds_total", `Total amount of time in seconds spent per command`, []string{"cmd"})
284-
e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, l)
285-
e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, l)
286-
e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, l)
295+
e.metricDescriptions["slowlog_length"] = newMetricDescr(opts.Namespace, "slowlog_length", `Total slowlog`, nil)
296+
e.metricDescriptions["slowlog_last_id"] = newMetricDescr(opts.Namespace, "slowlog_last_id", `Last id of slowlog`, nil)
297+
e.metricDescriptions["last_slow_execution_duration_seconds"] = newMetricDescr(opts.Namespace, "last_slow_execution_duration_seconds", `The amount of time needed for last slow execution, in seconds`, nil)
287298

288299
e.metricDescriptions["latency_spike_last"] = newMetricDescr(opts.Namespace, "latency_spike_last", `When the latency spike last occurred`, []string{"event_name"})
289300
e.metricDescriptions["latency_spike_seconds"] = newMetricDescr(opts.Namespace, "latency_spike_seconds", `Length of the last latency spike in seconds`, []string{"event_name"})
290301

291302
e.metricDescriptions["slave_info"] = newMetricDescr(opts.Namespace, "slave_info", "Information about the Redis slave", []string{"master_host", "master_port", "read_only"})
292303

293-
e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", l)
294-
e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", l)
304+
e.metricDescriptions["start_time_seconds"] = newMetricDescr(opts.Namespace, "start_time_seconds", "Start time of the Redis instance since unix epoch in seconds.", nil)
305+
e.metricDescriptions["master_link_up"] = newMetricDescr(opts.Namespace, "master_link_up", "Master link status on Redis slave", nil)
295306
e.metricDescriptions["connected_slave_offset"] = newMetricDescr(opts.Namespace, "connected_slave_offset", "Offset of connected slave", []string{"slave_ip", "slave_port", "slave_state"})
296307
e.metricDescriptions["connected_slave_lag_seconds"] = newMetricDescr(opts.Namespace, "connected_slave_lag_seconds", "Lag of connected slave", []string{"slave_ip", "slave_port", "slave_state"})
297308

@@ -309,11 +320,11 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
309320
}
310321

311322
for _, v := range metricMapGauges {
312-
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{})
323+
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil)
313324
}
314325

315326
for _, v := range metricMapCounters {
316-
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", []string{})
327+
ch <- newMetricDescr(e.options.Namespace, v, v+" metric", nil)
317328
}
318329

319330
ch <- e.totalScrapes.Desc()
@@ -485,16 +496,16 @@ func (e *Exporter) registerGaugeValue(ch chan<- prometheus.Metric, metric string
485496
e.registerMetricValue(ch, metric, val, prometheus.GaugeValue, labels...)
486497
}
487498

488-
func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labels ...string) {
499+
func (e *Exporter) registerMetricValue(ch chan<- prometheus.Metric, metric string, val float64, valType prometheus.ValueType, labelValues ...string) {
489500
descr := e.metricDescriptions[metric]
490501
if descr == nil {
491-
descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", []string{})
502+
descr = newMetricDescr(e.options.Namespace, metric, metric+" metric", nil)
492503
}
493504

494-
ch <- prometheus.MustNewConstMetric(descr, valType, val, labels...)
505+
ch <- prometheus.MustNewConstMetric(descr, valType, val, labelValues...)
495506
}
496507

497-
func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) error {
508+
func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []string) {
498509
for i := 0; i < len(info); i += 2 {
499510
log.Debugf("tile38: %s:%s", info[i], info[i+1])
500511

@@ -507,8 +518,6 @@ func (e *Exporter) extractTile38Metrics(ch chan<- prometheus.Metric, info []stri
507518

508519
e.parseAndRegisterMetric(ch, fieldKey, fieldValue)
509520
}
510-
511-
return nil
512521
}
513522

514523
func (e *Exporter) handleMetricsCommandStats(ch chan<- prometheus.Metric, fieldKey string, fieldValue string) {
@@ -699,6 +708,89 @@ func (e *Exporter) extractClusterInfoMetrics(ch chan<- prometheus.Metric, info s
699708
return nil
700709
}
701710

711+
func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
712+
log.Debugf("e.singleKeys: %#v", e.singleKeys)
713+
allKeys := append([]dbKeyPair{}, e.singleKeys...)
714+
715+
log.Debugf("e.keys: %#v", e.keys)
716+
scannedKeys, err := getKeysFromPatterns(c, e.keys)
717+
if err != nil {
718+
log.Errorf("Error expanding key patterns: %#v", err)
719+
} else {
720+
allKeys = append(allKeys, scannedKeys...)
721+
}
722+
723+
log.Debugf("allKeys: %#v", allKeys)
724+
for _, k := range allKeys {
725+
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
726+
log.Debugf("Couldn't select database %#v when getting key info.", k.db)
727+
continue
728+
}
729+
730+
info, err := getKeyInfo(c, k.key)
731+
if err != nil {
732+
switch err {
733+
case errNotFound:
734+
log.Debugf("Key '%s' not found when trying to get type and size.", k.key)
735+
default:
736+
log.Error(err)
737+
}
738+
continue
739+
}
740+
dbLabel := "db" + k.db
741+
e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key)
742+
743+
// Only record value metric if value is float-y
744+
if val, err := redis.Float64(c.Do("GET", k.key)); err == nil {
745+
e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key)
746+
}
747+
}
748+
}
749+
750+
func (e *Exporter) extractLuaScriptMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
751+
if e.LuaScript == nil || len(e.LuaScript) == 0 {
752+
return
753+
}
754+
755+
log.Debug("Evaluating e.LuaScript")
756+
kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0))
757+
if err != nil {
758+
log.Errorf("LuaScript error: %v", err)
759+
return
760+
}
761+
762+
if kv != nil {
763+
for key, stringVal := range kv {
764+
if val, err := strconv.ParseFloat(stringVal, 64); err == nil {
765+
e.registerGaugeValue(ch, "script_values", val, key)
766+
}
767+
}
768+
}
769+
}
770+
771+
func (e *Exporter) extractSlowLogMetrics(ch chan<- prometheus.Metric, c redis.Conn) {
772+
if reply, err := c.Do("SLOWLOG", "LEN"); err == nil {
773+
e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64)))
774+
}
775+
776+
if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil {
777+
var slowlogLastID int64
778+
var lastSlowExecutionDurationSeconds float64
779+
780+
if len(values) > 0 {
781+
if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 {
782+
slowlogLastID = values[0].(int64)
783+
if len(values) > 2 {
784+
lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6
785+
}
786+
}
787+
}
788+
789+
e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID))
790+
e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds)
791+
}
792+
}
793+
702794
func (e *Exporter) parseAndRegisterMetric(ch chan<- prometheus.Metric, fieldKey, fieldValue string) error {
703795
orgMetricName := sanitizeMetricName(fieldKey)
704796
metricName := orgMetricName
@@ -762,7 +854,7 @@ func getKeyInfo(c redis.Conn, key string) (info keyInfo, err error) {
762854
return info, errNotFound
763855
case "string":
764856
if size, err := redis.Int64(c.Do("PFCOUNT", key)); err == nil {
765-
info.keyType = "hyperloglog"
857+
// hyperloglog
766858
info.size = float64(size)
767859
} else if size, err := redis.Int64(c.Do("STRLEN", key)); err == nil {
768860
info.size = float64(size)
@@ -880,12 +972,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
880972
log.Debugf("aborting for addr: %s - redis err: %s", e.redisAddr, err)
881973
return err
882974
}
883-
884975
defer c.Close()
976+
885977
log.Debugf("connected to: %s", e.redisAddr)
886978

887979
dbCount := 0
888-
889980
if config, err := redis.Strings(c.Do(e.options.ConfigCommandName, "GET", "*")); err == nil {
890981
dbCount, err = e.extractConfigMetrics(ch, config)
891982
if err != nil {
@@ -904,9 +995,8 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
904995
return err
905996
}
906997
}
907-
isClusterEnabled := strings.Contains(infoAll, "cluster_enabled:1")
908998

909-
if isClusterEnabled {
999+
if strings.Contains(infoAll, "cluster_enabled:1") {
9101000
if clusterInfo, err := redis.String(doRedisCmd(c, "CLUSTER", "INFO")); err == nil {
9111001
e.extractClusterInfoMetrics(ch, clusterInfo)
9121002

@@ -946,77 +1036,11 @@ func (e *Exporter) scrapeRedisHost(ch chan<- prometheus.Metric) error {
9461036
}
9471037
}
9481038

949-
log.Debugf("e.singleKeys: %#v", e.singleKeys)
950-
allKeys := append([]dbKeyPair{}, e.singleKeys...)
1039+
e.extractCheckKeyMetrics(ch, c)
9511040

952-
log.Debugf("e.keys: %#v", e.keys)
953-
scannedKeys, err := getKeysFromPatterns(c, e.keys)
954-
if err != nil {
955-
log.Errorf("Error expanding key patterns: %#v", err)
956-
} else {
957-
allKeys = append(allKeys, scannedKeys...)
958-
}
959-
960-
log.Debugf("allKeys: %#v", allKeys)
961-
for _, k := range allKeys {
962-
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
963-
log.Debugf("Couldn't select database %#v when getting key info.", k.db)
964-
continue
965-
}
1041+
e.extractLuaScriptMetrics(ch, c)
9661042

967-
info, err := getKeyInfo(c, k.key)
968-
if err != nil {
969-
switch err {
970-
case errNotFound:
971-
log.Debugf("Key '%s' not found when trying to get type and size.", k.key)
972-
default:
973-
log.Error(err)
974-
}
975-
continue
976-
}
977-
dbLabel := "db" + k.db
978-
e.registerGaugeValue(ch, "key_sizes", info.size, dbLabel, k.key)
979-
980-
// Only record value metric if value is float-y
981-
if val, err := redis.Float64(c.Do("GET", k.key)); err == nil {
982-
e.registerGaugeValue(ch, "key_values", val, dbLabel, k.key)
983-
}
984-
}
985-
986-
if e.LuaScript != nil && len(e.LuaScript) > 0 {
987-
log.Debug("e.script")
988-
kv, err := redis.StringMap(doRedisCmd(c, "EVAL", e.LuaScript, 0, 0))
989-
if err != nil {
990-
log.Errorf("Collect script error: %v", err)
991-
} else if kv != nil {
992-
for key, stringVal := range kv {
993-
if val, err := strconv.ParseFloat(stringVal, 64); err == nil {
994-
e.registerGaugeValue(ch, "script_values", val, key)
995-
}
996-
}
997-
}
998-
}
999-
1000-
if reply, err := c.Do("SLOWLOG", "LEN"); err == nil {
1001-
e.registerGaugeValue(ch, "slowlog_length", float64(reply.(int64)))
1002-
}
1003-
1004-
if values, err := redis.Values(c.Do("SLOWLOG", "GET", "1")); err == nil {
1005-
var slowlogLastID int64
1006-
var lastSlowExecutionDurationSeconds float64
1007-
1008-
if len(values) > 0 {
1009-
if values, err = redis.Values(values[0], err); err == nil && len(values) > 0 {
1010-
slowlogLastID = values[0].(int64)
1011-
if len(values) > 2 {
1012-
lastSlowExecutionDurationSeconds = float64(values[2].(int64)) / 1e6
1013-
}
1014-
}
1015-
}
1016-
1017-
e.registerGaugeValue(ch, "slowlog_last_id", float64(slowlogLastID))
1018-
e.registerGaugeValue(ch, "last_slow_execution_duration_seconds", lastSlowExecutionDurationSeconds)
1019-
}
1043+
e.extractSlowLogMetrics(ch, c)
10201044

10211045
log.Debugf("scrapeRedisHost() done")
10221046
return nil

‎exporter/redis_test.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ func TestLatencySpike(t *testing.T) {
209209

210210
func TestTile38(t *testing.T) {
211211
if os.Getenv("TEST_TILE38_URI") == "" {
212-
t.SkipNow()
212+
t.Skipf("TEST_TILE38_URI not set - skipping")
213213
}
214214

215215
e, _ := NewRedisExporter(os.Getenv("TEST_TILE38_URI"), Options{Namespace: "test"})
@@ -776,7 +776,6 @@ func TestKeySizeList(t *testing.T) {
776776
}()
777777

778778
found := false
779-
780779
for m := range chM {
781780
if strings.Contains(m.Desc().String(), "test_key_size") {
782781
found = true
@@ -1060,9 +1059,7 @@ func TestKeysReset(t *testing.T) {
10601059

10611060
func TestClusterMaster(t *testing.T) {
10621061
if os.Getenv("TEST_REDIS_CLUSTER_MASTER_URI") == "" {
1063-
log.Println("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
1064-
t.SkipNow()
1065-
return
1062+
t.Skipf("TEST_REDIS_CLUSTER_MASTER_URI not set - skipping")
10661063
}
10671064

10681065
r := prometheus.NewRegistry()
@@ -1096,6 +1093,9 @@ func TestClusterMaster(t *testing.T) {
10961093
}
10971094

10981095
func TestPasswordProtectedInstance(t *testing.T) {
1096+
if os.Getenv("TEST_PWD_REDIS_URI") == "" {
1097+
t.Skipf("TEST_PWD_REDIS_URI not set - skipping")
1098+
}
10991099
ts := httptest.NewServer(promhttp.Handler())
11001100
defer ts.Close()
11011101

@@ -1119,6 +1119,9 @@ func TestPasswordProtectedInstance(t *testing.T) {
11191119
}
11201120

11211121
func TestPasswordInvalid(t *testing.T) {
1122+
if os.Getenv("TEST_PWD_REDIS_URI") == "" {
1123+
t.Skipf("TEST_PWD_REDIS_URI not set - skipping")
1124+
}
11221125
r := prometheus.NewRegistry()
11231126
prometheus.DefaultGatherer = r
11241127
prometheus.DefaultRegisterer = r
@@ -1148,9 +1151,7 @@ func TestPasswordInvalid(t *testing.T) {
11481151

11491152
func TestClusterSlave(t *testing.T) {
11501153
if os.Getenv("TEST_REDIS_CLUSTER_SLAVE_URI") == "" {
1151-
log.Println("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping")
1152-
t.SkipNow()
1153-
return
1154+
t.Skipf("TEST_REDIS_CLUSTER_SLAVE_URI not set - skipping")
11541155
}
11551156

11561157
r := prometheus.NewRegistry()

0 commit comments

Comments
 (0)