added regexp.MatchString to handle all test cases

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [CHANGE] Rules: Execute 1 query instead of N (where N is the number of alerts within alert rule) when restoring alerts. #13980
 * [ENHANCEMENT] Rules: Add `rule_group_last_restore_duration_seconds` to measure the time it takes to restore a rule group. #13974
 * [ENHANCEMENT] OTLP: Improve remote write format translation performance by using label set hashes for metric identifiers instead of string based ones. #14006 #13991
+* [ENHANCEMENT] TSDB: Optimize querying with regexp matchers. #13620
 * [BUGFIX] OTLP: Don't generate target_info unless at least one identifying label is defined. #13991
 * [BUGFIX] OTLP: Don't generate target_info unless there are metrics. #13991
 

cmd/promtool/main.go

@@ -56,8 +56,8 @@ import (
  "github.com/prometheus/prometheus/model/rulefmt"
  "github.com/prometheus/prometheus/notifier"
  _ "github.com/prometheus/prometheus/plugins" // Register plugins.
- "github.com/prometheus/prometheus/promql"
  "github.com/prometheus/prometheus/promql/parser"
+ "github.com/prometheus/prometheus/promql/promqltest"
  "github.com/prometheus/prometheus/scrape"
  "github.com/prometheus/prometheus/util/documentcli"
 )
@@ -377,7 +377,7 @@ func main() {
 
  case testRulesCmd.FullCommand():
  os.Exit(RulesUnitTest(
- promql.LazyLoaderOpts{
+ promqltest.LazyLoaderOpts{
  EnableAtModifier: true,
  EnableNegativeOffset: true,
  },

cmd/promtool/tsdb_test.go

@@ -26,7 +26,7 @@ import (
 
  "github.com/stretchr/testify/require"
 
- "github.com/prometheus/prometheus/promql"
+ "github.com/prometheus/prometheus/promql/promqltest"
  "github.com/prometheus/prometheus/tsdb"
 )
 
@@ -88,7 +88,7 @@ func normalizeNewLine(b []byte) []byte {
 }
 
 func TestTSDBDump(t *testing.T) {
- storage := promql.LoadedStorage(t, `
+ storage := promqltest.LoadedStorage(t, `
  load 1m
  metric{foo="bar", baz="abc"} 1 2 3 4 5
  heavy_metric{foo="bar"} 5 4 3 2 1
@@ -158,7 +158,7 @@ func TestTSDBDump(t *testing.T) {
 }
 
 func TestTSDBDumpOpenMetrics(t *testing.T) {
- storage := promql.LoadedStorage(t, `
+ storage := promqltest.LoadedStorage(t, `
  load 1m
  my_counter{foo="bar", baz="abc"} 1 2 3 4 5
  my_gauge{bar="foo", abc="baz"} 9 8 0 4 7

cmd/promtool/unittest.go

@@ -36,13 +36,14 @@ import (
  "github.com/prometheus/prometheus/model/labels"
  "github.com/prometheus/prometheus/promql"
  "github.com/prometheus/prometheus/promql/parser"
+ "github.com/prometheus/prometheus/promql/promqltest"
  "github.com/prometheus/prometheus/rules"
  "github.com/prometheus/prometheus/storage"
 )
 
 // RulesUnitTest does unit testing of rules based on the unit testing files provided.
 // More info about the file format can be found in the docs.
-func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
+func RulesUnitTest(queryOpts promqltest.LazyLoaderOpts, runStrings []string, diffFlag bool, files ...string) int {
  failed := false
 
  var run *regexp.Regexp
@@ -69,7 +70,7 @@ func RulesUnitTest(queryOpts promql.LazyLoaderOpts, runStrings []string, diffFla
  return successExitCode
 }
 
-func ruleUnitTest(filename string, queryOpts promql.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
+func ruleUnitTest(filename string, queryOpts promqltest.LazyLoaderOpts, run *regexp.Regexp, diffFlag bool) []error {
  fmt.Println("Unit Testing: ", filename)
 
  b, err := os.ReadFile(filename)
@@ -175,9 +176,9 @@ type testGroup struct {
 }
 
 // test performs the unit tests.
-func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promql.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) (outErr []error) {
+func (tg *testGroup) test(evalInterval time.Duration, groupOrderMap map[string]int, queryOpts promqltest.LazyLoaderOpts, diffFlag bool, ruleFiles ...string) (outErr []error) {
  // Setup testing suite.
- suite, err := promql.NewLazyLoader(tg.seriesLoadingString(), queryOpts)
+ suite, err := promqltest.NewLazyLoader(tg.seriesLoadingString(), queryOpts)
  if err != nil {
  return []error{err}
  }
@@ -413,7 +414,7 @@ Outer:
  gotSamples = append(gotSamples, parsedSample{
  Labels: s.Metric.Copy(),
  Value: s.F,
- Histogram: promql.HistogramTestExpression(s.H),
+ Histogram: promqltest.HistogramTestExpression(s.H),
  })
  }
 
@@ -443,7 +444,7 @@ Outer:
  expSamples = append(expSamples, parsedSample{
  Labels: lb,
  Value: s.Value,
- Histogram: promql.HistogramTestExpression(hist),
+ Histogram: promqltest.HistogramTestExpression(hist),
  })
  }
 

cmd/promtool/unittest_test.go

@@ -18,7 +18,7 @@ import (
 
  "github.com/stretchr/testify/require"
 
- "github.com/prometheus/prometheus/promql"
+ "github.com/prometheus/prometheus/promql/promqltest"
 )
 
 func TestRulesUnitTest(t *testing.T) {
@@ -28,7 +28,7 @@ func TestRulesUnitTest(t *testing.T) {
  tests := []struct {
  name string
  args args
- queryOpts promql.LazyLoaderOpts
+ queryOpts promqltest.LazyLoaderOpts
  want int
  }{
  {
@@ -92,7 +92,7 @@ func TestRulesUnitTest(t *testing.T) {
  args: args{
  files: []string{"./testdata/at-modifier-test.yml"},
  },
- queryOpts: promql.LazyLoaderOpts{
+ queryOpts: promqltest.LazyLoaderOpts{
  EnableAtModifier: true,
  },
  want: 0,
@@ -109,7 +109,7 @@ func TestRulesUnitTest(t *testing.T) {
  args: args{
  files: []string{"./testdata/negative-offset-test.yml"},
  },
- queryOpts: promql.LazyLoaderOpts{
+ queryOpts: promqltest.LazyLoaderOpts{
  EnableNegativeOffset: true,
  },
  want: 0,
@@ -119,7 +119,7 @@ func TestRulesUnitTest(t *testing.T) {
  args: args{
  files: []string{"./testdata/no-test-group-interval.yml"},
  },
- queryOpts: promql.LazyLoaderOpts{
+ queryOpts: promqltest.LazyLoaderOpts{
  EnableNegativeOffset: true,
  },
  want: 0,
@@ -142,7 +142,7 @@ func TestRulesUnitTestRun(t *testing.T) {
  tests := []struct {
  name string
  args args
- queryOpts promql.LazyLoaderOpts
+ queryOpts promqltest.LazyLoaderOpts
  want int
  }{
  {

go.mod

@@ -60,7 +60,6 @@ require (
  github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c
  github.com/stretchr/testify v1.9.0
  github.com/vultr/govultr/v2 v2.17.2
- go.opentelemetry.io/collector/featuregate v1.5.0
  go.opentelemetry.io/collector/pdata v1.5.0
  go.opentelemetry.io/collector/semconv v0.98.0
  go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0
@@ -151,7 +150,6 @@ require (
  github.com/hashicorp/go-multierror v1.1.1 // indirect
  github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
  github.com/hashicorp/go-rootcerts v1.0.2 // indirect
- github.com/hashicorp/go-version v1.6.0 // indirect
  github.com/hashicorp/golang-lru v0.6.0 // indirect
  github.com/hashicorp/serf v0.10.1 // indirect
  github.com/imdario/mergo v0.3.16 // indirect

go.sum

@@ -722,8 +722,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/collector/featuregate v1.5.0 h1:uK8qnYQKz1TMkK+FDTFsywg/EybW/gbnOUaPNUkRznM=
-go.opentelemetry.io/collector/featuregate v1.5.0/go.mod h1:w7nUODKxEi3FLf1HslCiE6YWtMtOOrMnSwsDam8Mg9w=
 go.opentelemetry.io/collector/pdata v1.5.0 h1:1fKTmUpr0xCOhP/B0VEvtz7bYPQ45luQ8XFyA07j8LE=
 go.opentelemetry.io/collector/pdata v1.5.0/go.mod h1:TYj8aKRWZyT/KuKQXKyqSEvK/GV+slFaDMEI+Ke64Yw=
 go.opentelemetry.io/collector/semconv v0.98.0 h1:zO4L4TmlxXoYu8UgPeYElGY19BW7wPjM+quL5CzoOoY=

model/labels/regexp.go

@@ -16,6 +16,7 @@ package labels
 import (
  "slices"
  "strings"
+ "unicode/utf8"
 
  "github.com/grafana/regexp"
  "github.com/grafana/regexp/syntax"
@@ -790,7 +791,16 @@ func (m *equalMultiStringMapMatcher) Matches(s string) bool {
  }
 
  _, ok := m.values[s]
- return ok
+ if ok || m.caseSensitive {
+ return ok
+ }
+ for k := range m.values {
+ val := strings.EqualFold(k, s)
+ if val {
+ return true
+ }
+ }
+ return false
 }
 
 // anyStringWithoutNewlineMatcher is a stringMatcher which matches any string
@@ -827,8 +837,7 @@ type zeroOrOneCharacterStringMatcher struct {
 }
 
 func (m *zeroOrOneCharacterStringMatcher) Matches(s string) bool {
- // Zero or one.
- if len(s) > 1 {
+ if moreThanOneRune(s) {
  return false
  }
 
@@ -840,6 +849,27 @@ func (m *zeroOrOneCharacterStringMatcher) Matches(s string) bool {
  return s[0] != '\n'
 }
 
+// moreThanOneRune returns true if there are more than one runes in the string.
+// It doesn't check whether the string is valid UTF-8.
+// The return value should be always equal to utf8.RuneCountInString(s) > 1,
+// but the function is optimized for the common case where the string prefix is ASCII.
+func moreThanOneRune(s string) bool {
+ // If len(s) is exactly one or zero, there can't be more than one rune.
+ // Exit through this path quickly.
+ if len(s) <= 1 {
+ return false
+ }
+
+ // There's one or more bytes:
+ // If first byte is ASCII then there are multiple runes if there are more bytes after that.
+ if s[0] < utf8.RuneSelf {
+ return len(s) > 1
+ }
+
+ // Less common case: first is a multibyte rune.
+ return utf8.RuneCountInString(s) > 1
+}
+
 // trueMatcher is a stringMatcher which matches any string (always returns true).
 type trueMatcher struct{}
 

model/labels/regexp_test.go

@@ -84,7 +84,7 @@ var (
  "foo", " foo bar", "bar", "buzz\nbar", "bar foo", "bfoo", "\n", "\nfoo", "foo\n", "hello foo world", "hello foo\n world", "",
  "FOO", "Foo", "OO", "Oo", "\nfoo\n", strings.Repeat("f", 20), "prometheus", "prometheus_api_v1", "prometheus_api_v1_foo",
  "10.0.1.20", "10.0.2.10", "10.0.3.30", "10.0.4.40",
- "foofoo0", "foofoo",
+ "foofoo0", "foofoo", "😀foo0", "ſſs",
 
  // Values matching / not matching the test regexps on long alternations.
  "zQPbMkNO", "zQPbMkNo", "jyyfj00j0061", "jyyfj00j006", "jyyfj00j00612", "NNSPdvMi", "NNSPdvMiXXX", "NNSPdvMixxx", "nnSPdvMi", "nnSPdvMiXXX",

model/relabel/relabel.go

@@ -48,7 +48,7 @@ const (
  Drop Action = "drop"
  // KeepEqual drops targets for which the input does not match the target.
  KeepEqual Action = "keepequal"
- // Drop drops targets for which the input does match the target.
+ // DropEqual drops targets for which the input does match the target.
  DropEqual Action = "dropequal"
  // HashMod sets a label to the modulus of a hash of labels.
  HashMod Action = "hashmod"

promql/bench_test.go

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package promql
+package promql_test
 
 import (
  "context"
@@ -23,13 +23,14 @@ import (
 
  "github.com/prometheus/prometheus/model/histogram"
  "github.com/prometheus/prometheus/model/labels"
+ "github.com/prometheus/prometheus/promql"
  "github.com/prometheus/prometheus/promql/parser"
  "github.com/prometheus/prometheus/storage"
  "github.com/prometheus/prometheus/tsdb/tsdbutil"
  "github.com/prometheus/prometheus/util/teststorage"
 )
 
-func setupRangeQueryTestData(stor *teststorage.TestStorage, _ *Engine, interval, numIntervals int) error {
+func setupRangeQueryTestData(stor *teststorage.TestStorage, _ *promql.Engine, interval, numIntervals int) error {
  ctx := context.Background()
 
  metrics := []labels.Labels{}
@@ -249,13 +250,13 @@ func BenchmarkRangeQuery(b *testing.B) {
  stor := teststorage.New(b)
  stor.DB.DisableCompactions() // Don't want auto-compaction disrupting timings.
  defer stor.Close()
- opts := EngineOpts{
+ opts := promql.EngineOpts{
  Logger: nil,
  Reg: nil,
  MaxSamples: 50000000,
  Timeout: 100 * time.Second,
  }
- engine := NewEngine(opts)
+ engine := promql.NewEngine(opts)
 
  const interval = 10000 // 10s interval.
  // A day of data plus 10k steps.
@@ -324,7 +325,7 @@ func BenchmarkNativeHistograms(b *testing.B) {
  },
  }
 
- opts := EngineOpts{
+ opts := promql.EngineOpts{
  Logger: nil,
  Reg: nil,
  MaxSamples: 50000000,
@@ -338,7 +339,7 @@ func BenchmarkNativeHistograms(b *testing.B) {
 
  for _, tc := range cases {
  b.Run(tc.name, func(b *testing.B) {
- ng := NewEngine(opts)
+ ng := promql.NewEngine(opts)
  for i := 0; i < b.N; i++ {
  qry, err := ng.NewRangeQuery(context.Background(), testStorage, nil, tc.query, start, end, step)
  if err != nil {

promql/engine.go

@@ -573,7 +573,8 @@ func (ng *Engine) validateOpts(expr parser.Expr) error {
  return validationErr
 }
 
-func (ng *Engine) newTestQuery(f func(context.Context) error) Query {
+// NewTestQuery: inject special behaviour into Query for testing.
+func (ng *Engine) NewTestQuery(f func(context.Context) error) Query {
  qry := &query{
  q: "test statement",
  stmt: parser.TestStmt(f),
@@ -2024,25 +2025,21 @@ func (ev *evaluator) rangeEvalTimestampFunctionOverVectorSelector(vs *parser.Vec
  vec := make(Vector, 0, len(vs.Series))
  for i, s := range vs.Series {
  it := seriesIterators[i]
- t, f, h, ok := ev.vectorSelectorSingle(it, vs, enh.Ts)
- if ok {
- vec = append(vec, Sample{
- Metric: s.Labels(),
- T: t,
- F: f,
- H: h,
- })
- histSize := 0
- if h != nil {
- histSize := h.Size() / 16 // 16 bytes per sample.
- ev.currentSamples += histSize
- }
- ev.currentSamples++
+ t, _, _, ok := ev.vectorSelectorSingle(it, vs, enh.Ts)
+ if !ok {
+ continue
+ }
 
- ev.samplesStats.IncrementSamplesAtTimestamp(enh.Ts, int64(1+histSize))
- if ev.currentSamples > ev.maxSamples {
- ev.error(ErrTooManySamples(env))
- }
+ // Note that we ignore the sample values because call only cares about the timestamp.
+ vec = append(vec, Sample{
+ Metric: s.Labels(),
+ T: t,
+ })
+
+ ev.currentSamples++
+ ev.samplesStats.IncrementSamplesAtTimestamp(enh.Ts, 1)
+ if ev.currentSamples > ev.maxSamples {
+ ev.error(ErrTooManySamples(env))
  }
  }
  ev.samplesStats.UpdatePeak(ev.currentSamples)