Emit metric tracking empty responses from prometheus

CHANGELOG.md

@@ -90,6 +90,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
 - **Github Scaler**: Add support to control unlabeled job/runner matching ([#6900](https://github.com/kedacore/keda/issues/6900))
 - **Metrics API Scaler**: Support AuthParams for authMode ([#6939](https://github.com/kedacore/keda/issues/6939))
 - **Metrics API Scaler**: Support multiple auth methods simultaneously ([#6642](https://github.com/kedacore/keda/issues/6642))
+- **Prometheus Scaler**: Emit metric tracking empty responses from prometheus ([#7060](https://github.com/kedacore/keda/pull/7060))
 - **Temporal Scaler**: Support custom tlsServerName ([#6820](https://github.com/kedacore/keda/pull/6820))
 
 ### Fixes

pkg/metricscollector/metricscollectors.go

@@ -79,6 +79,9 @@ type MetricsCollector interface {
 
 	// RecordCloudEventQueueStatus record the number of cloudevents that are waiting for emitting
 	RecordCloudEventQueueStatus(namespace string, value int)
+
+	// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+	RecordEmptyUpstreamResponse()
 }
 
 func NewMetricsCollectors(enablePrometheusMetrics bool, enableOpenTelemetryMetrics bool) {
@@ -205,6 +208,13 @@ func RecordCloudEventQueueStatus(namespace string, value int) {
 	}
 }
 
+// RecordEmptyPrometheusMetricError counts the number of times a query returns an empty result
+func RecordEmptyUpstreamResponse() {
+	for _, element := range collectors {
+		element.RecordEmptyUpstreamResponse()
+	}
+}
+
 // Returns the ServerMetrics object for GRPC Server metrics. Used to initialize the GRPC server with the proper intercepts
 // Currently, only Prometheus metrics are supported.
 func GetServerMetrics() *grpcprom.ServerMetrics {

pkg/metricscollector/opentelemetry.go

@@ -34,6 +34,7 @@ var (
 	otCrdTotalsCounterDeprecated     api.Int64UpDownCounter
 	otTriggerRegisteredTotalsCounter api.Int64UpDownCounter
 	otCrdRegisteredTotalsCounter     api.Int64UpDownCounter
+	otEmptyUpstreamResponses         api.Int64Counter
 
 	otelScalerMetricVals                  []OtelMetricFloat64Val
 	otelScalerMetricsLatencyVals          []OtelMetricFloat64Val
@@ -135,6 +136,11 @@ func initMeters() {
 		otLog.Error(err, msg)
 	}
 
+	otEmptyUpstreamResponses, err = meter.Int64Counter("keda.scaler.empty.upstream.responses", api.WithDescription("Number of times a query returns an empty result"))
+	if err != nil {
+		otLog.Error(err, msg)
+	}
+
 	_, err = meter.Float64ObservableGauge(
 		"keda.scaler.metrics.value",
 		api.WithDescription("The current value for each scaler's metric that would be used by the HPA in computing the target average"),
@@ -506,3 +512,8 @@ func (o *OtelMetrics) RecordCloudEventQueueStatus(namespace string, value int) {
 	otCloudEventQueueStatus.measurementOption = opt
 	otCloudEventQueueStatusVals = append(otCloudEventQueueStatusVals, otCloudEventQueueStatus)
 }
+
+// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+func (o *OtelMetrics) RecordEmptyUpstreamResponse() {
+	otEmptyUpstreamResponses.Add(context.Background(), 1, nil)
+}

pkg/metricscollector/prommetrics.go

@@ -104,6 +104,14 @@ var (
 		},
 		[]string{"namespace", "scaledJob"},
 	)
+	emptyUpstreamResponse = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: DefaultPromMetricsNamespace,
+			Subsystem: "scaler",
+			Name:      "empty_upstream_responses_total",
+			Help:      "Number of times a query returns an empty result",
+		},
+	)
 	triggerRegistered = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Namespace: DefaultPromMetricsNamespace,
@@ -168,6 +176,7 @@ func NewPromMetrics() *PromMetrics {
 	metrics.Registry.MustRegister(triggerRegistered)
 	metrics.Registry.MustRegister(crdRegistered)
 	metrics.Registry.MustRegister(scaledJobErrors)
+	metrics.Registry.MustRegister(emptyUpstreamResponse)
 
 	metrics.Registry.MustRegister(buildInfo)
 
@@ -328,6 +337,11 @@ func (p *PromMetrics) RecordCloudEventQueueStatus(namespace string, value int) {
 	cloudeventQueueStatus.With(prometheus.Labels{"namespace": namespace}).Set(float64(value))
 }
 
+// RecordEmptyUpstreamResponse counts the number of times a query returns an empty result
+func (p *PromMetrics) RecordEmptyUpstreamResponse() {
+	emptyUpstreamResponse.Inc()
+}
+
 // Returns a grpcprom server Metrics object and registers the metrics. The object contains
 // interceptors to chain to the server so that all requests served are observed. Intended to be called
 // as part of initialization of metricscollector, hence why this function is not exported

pkg/scalers/prometheus_scaler.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/metrics/pkg/apis/external_metrics"
 
 	kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1"
+	"github.com/kedacore/keda/v2/pkg/metricscollector"
 	"github.com/kedacore/keda/v2/pkg/scalers/authentication"
 	"github.com/kedacore/keda/v2/pkg/scalers/aws"
 	"github.com/kedacore/keda/v2/pkg/scalers/azure"
@@ -255,6 +256,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error
 		if s.metadata.IgnoreNullValues {
 			return 0, nil
 		}
+		metricscollector.RecordEmptyUpstreamResponse()
 		return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the result is empty")
 	} else if len(result.Data.Result) > 1 {
 		return -1, fmt.Errorf("prometheus query %s returned multiple elements", s.metadata.Query)
@@ -265,6 +267,7 @@ func (s *prometheusScaler) ExecutePromQuery(ctx context.Context) (float64, error
 		if s.metadata.IgnoreNullValues {
 			return 0, nil
 		}
+		metricscollector.RecordEmptyUpstreamResponse()
 		return -1, fmt.Errorf("prometheus metrics 'prometheus' target may be lost, the value list is empty")
 	} else if valueLen < 2 {
 		return -1, fmt.Errorf("prometheus query %s didn't return enough values", s.metadata.Query)