rules: skip template labels when querying ALERTS_FOR_STATE for restore

QueryForStateSeries built Select matchers from the raw rule labels,
which can contain Go template expressions such as
`instance_{{ $labels.instance }}`. The stored ALERTS_FOR_STATE series
carry the per-instance evaluated values (e.g. `instance_0`), so the
unevaluated template string never matched, leaving seriesByLabels empty
and silently skipping restoration for every active alert.

Fix by omitting any label whose value contains `{{` from the matcher
list. Static labels (including `__name__` and `alertname`) are never
templated and continue to scope the query to the correct rule. The
in-memory lookup against evaluated alert labels that follows is
unaffected, so the single-query-per-rule optimisation introduced in
#13980 is fully preserved.

Fixes #16883
Ref #13980
Ref #18364

Signed-off-by: Julien Pivotto <291750+roidelapluie@users.noreply.github.com>

rules/alerting.go

@@ -282,6 +282,11 @@ func (r *AlertingRule) QueryForStateSeries(ctx context.Context, q storage.Querie
 	smpl := r.forStateSample(nil, time.Now(), 0)
 	var matchers []*labels.Matcher
 	smpl.Metric.Range(func(l labels.Label) {
+		// Skip labels with template syntax: their values are expanded per alert
+		// instance and would not match the stored series.
+		if strings.Contains(l.Value, "{{") {
+			return
+		}
 		mt, err := labels.NewMatcher(labels.MatchEqual, l.Name, l.Value)
 		if err != nil {
 			panic(err)

rules/alerting_test.go

@@ -741,6 +741,45 @@ func TestQueryForStateSeries(t *testing.T) {
 	}
 }
 
+// TestQueryForStateSeriesTemplateLabels verifies that labels containing Go
+// template syntax are excluded from the matchers used to query ALERTS_FOR_STATE,
+// so that per-instance expanded values stored in the series can still be found.
+func TestQueryForStateSeriesTemplateLabels(t *testing.T) {
+	var gotMatchers []*labels.Matcher
+	querier := &storage.MockQuerier{
+		SelectMockFunction: func(_ bool, _ *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
+			gotMatchers = matchers
+			return storage.EmptySeriesSet()
+		},
+	}
+
+	rule := NewAlertingRule(
+		"TestRule",
+		nil,
+		time.Minute,
+		0,
+		labels.FromStrings("severity", "critical", "instance_ext", "instance_{{ $labels.instance }}"),
+		labels.EmptyLabels(), labels.EmptyLabels(), "", true, nil,
+	)
+
+	_, err := rule.QueryForStateSeries(context.Background(), querier)
+	require.NoError(t, err)
+
+	for _, m := range gotMatchers {
+		require.NotContains(t, m.Value, "{{", "template label %q must not appear in Select matchers", m.Name)
+	}
+
+	// The static label must still be present.
+	found := false
+	for _, m := range gotMatchers {
+		if m.Name == "severity" && m.Value == "critical" {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "static label severity=critical must be present in Select matchers")
+}
+
 // TestSendAlertsDontAffectActiveAlerts tests a fix for https://github.com/prometheus/prometheus/issues/11424.
 func TestSendAlertsDontAffectActiveAlerts(t *testing.T) {
 	rule := NewAlertingRule(