"Fossies" - the Fresh Open Source Software Archive

Member "prometheus-2.15.2/tsdb/index/index_test.go" (6 Jan 2020, 12983 Bytes) of package /linux/misc/prometheus-2.15.2.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "index_test.go": 2.15.1_vs_2.15.2.

    1 // Copyright 2017 The Prometheus Authors
    2 // Licensed under the Apache License, Version 2.0 (the "License");
    3 // you may not use this file except in compliance with the License.
    4 // You may obtain a copy of the License at
    5 //
    6 // http://www.apache.org/licenses/LICENSE-2.0
    7 //
    8 // Unless required by applicable law or agreed to in writing, software
    9 // distributed under the License is distributed on an "AS IS" BASIS,
   10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   11 // See the License for the specific language governing permissions and
   12 // limitations under the License.
   13 
   14 package index
   15 
   16 import (
   17     "context"
   18     "fmt"
   19     "io/ioutil"
   20     "math/rand"
   21     "os"
   22     "path/filepath"
   23     "sort"
   24     "testing"
   25 
   26     "github.com/pkg/errors"
   27     "github.com/prometheus/prometheus/pkg/labels"
   28     "github.com/prometheus/prometheus/tsdb/chunkenc"
   29     "github.com/prometheus/prometheus/tsdb/chunks"
   30     "github.com/prometheus/prometheus/tsdb/encoding"
   31     "github.com/prometheus/prometheus/util/testutil"
   32 )
   33 
   34 type series struct {
   35     l      labels.Labels
   36     chunks []chunks.Meta
   37 }
   38 
   39 type mockIndex struct {
   40     series     map[uint64]series
   41     labelIndex map[string][]string
   42     postings   map[labels.Label][]uint64
   43     symbols    map[string]struct{}
   44 }
   45 
   46 func newMockIndex() mockIndex {
   47     ix := mockIndex{
   48         series:     make(map[uint64]series),
   49         labelIndex: make(map[string][]string),
   50         postings:   make(map[labels.Label][]uint64),
   51         symbols:    make(map[string]struct{}),
   52     }
   53     ix.postings[allPostingsKey] = []uint64{}
   54     return ix
   55 }
   56 
   57 func (m mockIndex) Symbols() (map[string]struct{}, error) {
   58     return m.symbols, nil
   59 }
   60 
   61 func (m mockIndex) AddSeries(ref uint64, l labels.Labels, chunks ...chunks.Meta) error {
   62     if _, ok := m.series[ref]; ok {
   63         return errors.Errorf("series with reference %d already added", ref)
   64     }
   65     for _, lbl := range l {
   66         m.symbols[lbl.Name] = struct{}{}
   67         m.symbols[lbl.Value] = struct{}{}
   68         if _, ok := m.postings[lbl]; !ok {
   69             m.postings[lbl] = []uint64{}
   70         }
   71         m.postings[lbl] = append(m.postings[lbl], ref)
   72     }
   73     m.postings[allPostingsKey] = append(m.postings[allPostingsKey], ref)
   74 
   75     s := series{l: l}
   76     // Actual chunk data is not stored in the index.
   77     for _, c := range chunks {
   78         c.Chunk = nil
   79         s.chunks = append(s.chunks, c)
   80     }
   81     m.series[ref] = s
   82 
   83     return nil
   84 }
   85 
   86 func (m mockIndex) Close() error {
   87     return nil
   88 }
   89 
   90 func (m mockIndex) LabelValues(names ...string) (StringTuples, error) {
   91     // TODO support composite indexes
   92     if len(names) != 1 {
   93         return nil, errors.New("composite indexes not supported yet")
   94     }
   95 
   96     return NewStringTuples(m.labelIndex[names[0]], 1)
   97 }
   98 
   99 func (m mockIndex) Postings(name string, values ...string) (Postings, error) {
  100     p := []Postings{}
  101     for _, value := range values {
  102         l := labels.Label{Name: name, Value: value}
  103         p = append(p, m.SortedPostings(NewListPostings(m.postings[l])))
  104     }
  105     return Merge(p...), nil
  106 }
  107 
  108 func (m mockIndex) SortedPostings(p Postings) Postings {
  109     ep, err := ExpandPostings(p)
  110     if err != nil {
  111         return ErrPostings(errors.Wrap(err, "expand postings"))
  112     }
  113 
  114     sort.Slice(ep, func(i, j int) bool {
  115         return labels.Compare(m.series[ep[i]].l, m.series[ep[j]].l) < 0
  116     })
  117     return NewListPostings(ep)
  118 }
  119 
  120 func (m mockIndex) Series(ref uint64, lset *labels.Labels, chks *[]chunks.Meta) error {
  121     s, ok := m.series[ref]
  122     if !ok {
  123         return errors.New("not found")
  124     }
  125     *lset = append((*lset)[:0], s.l...)
  126     *chks = append((*chks)[:0], s.chunks...)
  127 
  128     return nil
  129 }
  130 
  131 func TestIndexRW_Create_Open(t *testing.T) {
  132     dir, err := ioutil.TempDir("", "test_index_create")
  133     testutil.Ok(t, err)
  134     defer func() {
  135         testutil.Ok(t, os.RemoveAll(dir))
  136     }()
  137 
  138     fn := filepath.Join(dir, indexFilename)
  139 
  140     // An empty index must still result in a readable file.
  141     iw, err := NewWriter(context.Background(), fn)
  142     testutil.Ok(t, err)
  143     testutil.Ok(t, iw.Close())
  144 
  145     ir, err := NewFileReader(fn)
  146     testutil.Ok(t, err)
  147     testutil.Ok(t, ir.Close())
  148 
  149     // Modify magic header must cause open to fail.
  150     f, err := os.OpenFile(fn, os.O_WRONLY, 0666)
  151     testutil.Ok(t, err)
  152     _, err = f.WriteAt([]byte{0, 0}, 0)
  153     testutil.Ok(t, err)
  154     f.Close()
  155 
  156     _, err = NewFileReader(dir)
  157     testutil.NotOk(t, err)
  158 }
  159 
  160 func TestIndexRW_Postings(t *testing.T) {
  161     dir, err := ioutil.TempDir("", "test_index_postings")
  162     testutil.Ok(t, err)
  163     defer func() {
  164         testutil.Ok(t, os.RemoveAll(dir))
  165     }()
  166 
  167     fn := filepath.Join(dir, indexFilename)
  168 
  169     iw, err := NewWriter(context.Background(), fn)
  170     testutil.Ok(t, err)
  171 
  172     series := []labels.Labels{
  173         labels.FromStrings("a", "1", "b", "1"),
  174         labels.FromStrings("a", "1", "b", "2"),
  175         labels.FromStrings("a", "1", "b", "3"),
  176         labels.FromStrings("a", "1", "b", "4"),
  177     }
  178 
  179     testutil.Ok(t, iw.AddSymbol("1"))
  180     testutil.Ok(t, iw.AddSymbol("2"))
  181     testutil.Ok(t, iw.AddSymbol("3"))
  182     testutil.Ok(t, iw.AddSymbol("4"))
  183     testutil.Ok(t, iw.AddSymbol("a"))
  184     testutil.Ok(t, iw.AddSymbol("b"))
  185 
  186     // Postings lists are only written if a series with the respective
  187     // reference was added before.
  188     testutil.Ok(t, iw.AddSeries(1, series[0]))
  189     testutil.Ok(t, iw.AddSeries(2, series[1]))
  190     testutil.Ok(t, iw.AddSeries(3, series[2]))
  191     testutil.Ok(t, iw.AddSeries(4, series[3]))
  192 
  193     testutil.Ok(t, iw.Close())
  194 
  195     ir, err := NewFileReader(fn)
  196     testutil.Ok(t, err)
  197 
  198     p, err := ir.Postings("a", "1")
  199     testutil.Ok(t, err)
  200 
  201     var l labels.Labels
  202     var c []chunks.Meta
  203 
  204     for i := 0; p.Next(); i++ {
  205         err := ir.Series(p.At(), &l, &c)
  206 
  207         testutil.Ok(t, err)
  208         testutil.Equals(t, 0, len(c))
  209         testutil.Equals(t, series[i], l)
  210     }
  211     testutil.Ok(t, p.Err())
  212 
  213     // The label incides are no longer used, so test them by hand here.
  214     labelIndices := map[string][]string{}
  215     testutil.Ok(t, ReadOffsetTable(ir.b, ir.toc.LabelIndicesTable, func(key []string, off uint64, _ int) error {
  216         if len(key) != 1 {
  217             return errors.Errorf("unexpected key length for label indices table %d", len(key))
  218         }
  219 
  220         d := encoding.NewDecbufAt(ir.b, int(off), castagnoliTable)
  221         vals := []string{}
  222         nc := d.Be32int()
  223         if nc != 1 {
  224             return errors.Errorf("unexpected nuumber of label indices table names %d", nc)
  225         }
  226         for i := d.Be32(); i > 0; i-- {
  227             v, err := ir.lookupSymbol(d.Be32())
  228             if err != nil {
  229                 return err
  230             }
  231             vals = append(vals, v)
  232         }
  233         labelIndices[key[0]] = vals
  234         return d.Err()
  235     }))
  236     testutil.Equals(t, map[string][]string{
  237         "a": []string{"1"},
  238         "b": []string{"1", "2", "3", "4"},
  239     }, labelIndices)
  240 
  241     testutil.Ok(t, ir.Close())
  242 }
  243 
  244 func TestPostingsMany(t *testing.T) {
  245     dir, err := ioutil.TempDir("", "test_postings_many")
  246     testutil.Ok(t, err)
  247     defer func() {
  248         testutil.Ok(t, os.RemoveAll(dir))
  249     }()
  250 
  251     fn := filepath.Join(dir, indexFilename)
  252 
  253     iw, err := NewWriter(context.Background(), fn)
  254     testutil.Ok(t, err)
  255 
  256     // Create a label in the index which has 999 values.
  257     symbols := map[string]struct{}{}
  258     series := []labels.Labels{}
  259     for i := 1; i < 1000; i++ {
  260         v := fmt.Sprintf("%03d", i)
  261         series = append(series, labels.FromStrings("i", v, "foo", "bar"))
  262         symbols[v] = struct{}{}
  263     }
  264     symbols["i"] = struct{}{}
  265     symbols["foo"] = struct{}{}
  266     symbols["bar"] = struct{}{}
  267     syms := []string{}
  268     for s := range symbols {
  269         syms = append(syms, s)
  270     }
  271     sort.Strings(syms)
  272     for _, s := range syms {
  273         testutil.Ok(t, iw.AddSymbol(s))
  274     }
  275 
  276     for i, s := range series {
  277         testutil.Ok(t, iw.AddSeries(uint64(i), s))
  278     }
  279     testutil.Ok(t, iw.Close())
  280 
  281     ir, err := NewFileReader(fn)
  282     testutil.Ok(t, err)
  283     defer func() { testutil.Ok(t, ir.Close()) }()
  284 
  285     cases := []struct {
  286         in []string
  287     }{
  288         // Simple cases, everything is present.
  289         {in: []string{"002"}},
  290         {in: []string{"031", "032", "033"}},
  291         {in: []string{"032", "033"}},
  292         {in: []string{"127", "128"}},
  293         {in: []string{"127", "128", "129"}},
  294         {in: []string{"127", "129"}},
  295         {in: []string{"128", "129"}},
  296         {in: []string{"998", "999"}},
  297         {in: []string{"999"}},
  298         // Before actual values.
  299         {in: []string{"000"}},
  300         {in: []string{"000", "001"}},
  301         {in: []string{"000", "002"}},
  302         // After actual values.
  303         {in: []string{"999a"}},
  304         {in: []string{"999", "999a"}},
  305         {in: []string{"998", "999", "999a"}},
  306         // In the middle of actual values.
  307         {in: []string{"126a", "127", "128"}},
  308         {in: []string{"127", "127a", "128"}},
  309         {in: []string{"127", "127a", "128", "128a", "129"}},
  310         {in: []string{"127", "128a", "129"}},
  311         {in: []string{"128", "128a", "129"}},
  312         {in: []string{"128", "129", "129a"}},
  313         {in: []string{"126a", "126b", "127", "127a", "127b", "128", "128a", "128b", "129", "129a", "129b"}},
  314     }
  315 
  316     for _, c := range cases {
  317         it, err := ir.Postings("i", c.in...)
  318         testutil.Ok(t, err)
  319 
  320         got := []string{}
  321         var lbls labels.Labels
  322         var metas []chunks.Meta
  323         for it.Next() {
  324             testutil.Ok(t, ir.Series(it.At(), &lbls, &metas))
  325             got = append(got, lbls.Get("i"))
  326         }
  327         testutil.Ok(t, it.Err())
  328         exp := []string{}
  329         for _, e := range c.in {
  330             if _, ok := symbols[e]; ok && e != "l" {
  331                 exp = append(exp, e)
  332             }
  333         }
  334         testutil.Equals(t, exp, got, fmt.Sprintf("input: %v", c.in))
  335     }
  336 
  337 }
  338 
  339 func TestPersistence_index_e2e(t *testing.T) {
  340     dir, err := ioutil.TempDir("", "test_persistence_e2e")
  341     testutil.Ok(t, err)
  342     defer func() {
  343         testutil.Ok(t, os.RemoveAll(dir))
  344     }()
  345 
  346     lbls, err := labels.ReadLabels(filepath.Join("..", "testdata", "20kseries.json"), 20000)
  347     testutil.Ok(t, err)
  348 
  349     // Sort labels as the index writer expects series in sorted order.
  350     sort.Sort(labels.Slice(lbls))
  351 
  352     symbols := map[string]struct{}{}
  353     for _, lset := range lbls {
  354         for _, l := range lset {
  355             symbols[l.Name] = struct{}{}
  356             symbols[l.Value] = struct{}{}
  357         }
  358     }
  359 
  360     var input indexWriterSeriesSlice
  361 
  362     // Generate ChunkMetas for every label set.
  363     for i, lset := range lbls {
  364         var metas []chunks.Meta
  365 
  366         for j := 0; j <= (i % 20); j++ {
  367             metas = append(metas, chunks.Meta{
  368                 MinTime: int64(j * 10000),
  369                 MaxTime: int64((j + 1) * 10000),
  370                 Ref:     rand.Uint64(),
  371                 Chunk:   chunkenc.NewXORChunk(),
  372             })
  373         }
  374         input = append(input, &indexWriterSeries{
  375             labels: lset,
  376             chunks: metas,
  377         })
  378     }
  379 
  380     iw, err := NewWriter(context.Background(), filepath.Join(dir, indexFilename))
  381     testutil.Ok(t, err)
  382 
  383     syms := []string{}
  384     for s := range symbols {
  385         syms = append(syms, s)
  386     }
  387     sort.Strings(syms)
  388     for _, s := range syms {
  389         testutil.Ok(t, iw.AddSymbol(s))
  390     }
  391 
  392     // Population procedure as done by compaction.
  393     var (
  394         postings = NewMemPostings()
  395         values   = map[string]map[string]struct{}{}
  396     )
  397 
  398     mi := newMockIndex()
  399 
  400     for i, s := range input {
  401         err = iw.AddSeries(uint64(i), s.labels, s.chunks...)
  402         testutil.Ok(t, err)
  403         testutil.Ok(t, mi.AddSeries(uint64(i), s.labels, s.chunks...))
  404 
  405         for _, l := range s.labels {
  406             valset, ok := values[l.Name]
  407             if !ok {
  408                 valset = map[string]struct{}{}
  409                 values[l.Name] = valset
  410             }
  411             valset[l.Value] = struct{}{}
  412         }
  413         postings.Add(uint64(i), s.labels)
  414     }
  415 
  416     err = iw.Close()
  417     testutil.Ok(t, err)
  418 
  419     ir, err := NewFileReader(filepath.Join(dir, indexFilename))
  420     testutil.Ok(t, err)
  421 
  422     for p := range mi.postings {
  423         gotp, err := ir.Postings(p.Name, p.Value)
  424         testutil.Ok(t, err)
  425 
  426         expp, err := mi.Postings(p.Name, p.Value)
  427         testutil.Ok(t, err)
  428 
  429         var lset, explset labels.Labels
  430         var chks, expchks []chunks.Meta
  431 
  432         for gotp.Next() {
  433             testutil.Assert(t, expp.Next() == true, "")
  434 
  435             ref := gotp.At()
  436 
  437             err := ir.Series(ref, &lset, &chks)
  438             testutil.Ok(t, err)
  439 
  440             err = mi.Series(expp.At(), &explset, &expchks)
  441             testutil.Ok(t, err)
  442             testutil.Equals(t, explset, lset)
  443             testutil.Equals(t, expchks, chks)
  444         }
  445         testutil.Assert(t, expp.Next() == false, "Expected no more postings for %q=%q", p.Name, p.Value)
  446         testutil.Ok(t, gotp.Err())
  447     }
  448 
  449     for k, v := range mi.labelIndex {
  450         tplsExp, err := NewStringTuples(v, 1)
  451         testutil.Ok(t, err)
  452 
  453         tplsRes, err := ir.LabelValues(k)
  454         testutil.Ok(t, err)
  455 
  456         testutil.Equals(t, tplsExp.Len(), tplsRes.Len())
  457         for i := 0; i < tplsExp.Len(); i++ {
  458             strsExp, err := tplsExp.At(i)
  459             testutil.Ok(t, err)
  460 
  461             strsRes, err := tplsRes.At(i)
  462             testutil.Ok(t, err)
  463 
  464             testutil.Equals(t, strsExp, strsRes)
  465         }
  466     }
  467 
  468     gotSymbols := []string{}
  469     it := ir.Symbols()
  470     for it.Next() {
  471         gotSymbols = append(gotSymbols, it.At())
  472     }
  473     testutil.Ok(t, it.Err())
  474     expSymbols := []string{}
  475     for s := range mi.symbols {
  476         expSymbols = append(expSymbols, s)
  477     }
  478     sort.Strings(expSymbols)
  479     testutil.Equals(t, expSymbols, gotSymbols)
  480 
  481     testutil.Ok(t, ir.Close())
  482 }
  483 
  484 func TestDecbufUvariantWithInvalidBuffer(t *testing.T) {
  485     b := realByteSlice([]byte{0x81, 0x81, 0x81, 0x81, 0x81, 0x81})
  486 
  487     db := encoding.NewDecbufUvarintAt(b, 0, castagnoliTable)
  488     testutil.NotOk(t, db.Err())
  489 }
  490 
  491 func TestReaderWithInvalidBuffer(t *testing.T) {
  492     b := realByteSlice([]byte{0x81, 0x81, 0x81, 0x81, 0x81, 0x81})
  493 
  494     _, err := NewReader(b)
  495     testutil.NotOk(t, err)
  496 }
  497 
  498 // TestNewFileReaderErrorNoOpenFiles ensures that in case of an error no file remains open.
  499 func TestNewFileReaderErrorNoOpenFiles(t *testing.T) {
  500     dir := testutil.NewTemporaryDirectory("block", t)
  501 
  502     idxName := filepath.Join(dir.Path(), "index")
  503     err := ioutil.WriteFile(idxName, []byte("corrupted contents"), 0644)
  504     testutil.Ok(t, err)
  505 
  506     _, err = NewFileReader(idxName)
  507     testutil.NotOk(t, err)
  508 
  509     // dir.Close will fail on Win if idxName fd is not closed on error path.
  510     dir.Close()
  511 }