"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "tsdb/index/index.go" between
prometheus-2.15.1.tar.gz and prometheus-2.15.2.tar.gz

About: Prometheus is a monitoring system, alerting toolkit and time series database.

index.go  (prometheus-2.15.1):index.go  (prometheus-2.15.2)
skipping to change at line 514 skipping to change at line 514
hashPos := w.f.pos hashPos := w.f.pos
// Leave space for the hash. We can only calculate it // Leave space for the hash. We can only calculate it
// now that the number of symbols is known, so mmap and do it from there. // now that the number of symbols is known, so mmap and do it from there.
if err := w.write([]byte("hash")); err != nil { if err := w.write([]byte("hash")); err != nil {
return err return err
} }
if err := w.f.flush(); err != nil { if err := w.f.flush(); err != nil {
return err return err
} }
var err error sf, err := fileutil.OpenMmapFile(w.f.name)
w.symbolFile, err = fileutil.OpenMmapFile(w.f.name)
if err != nil { if err != nil {
return err return err
} }
w.symbolFile = sf
hash := crc32.Checksum(w.symbolFile.Bytes()[w.toc.Symbols+4:hashPos], cas tagnoliTable) hash := crc32.Checksum(w.symbolFile.Bytes()[w.toc.Symbols+4:hashPos], cas tagnoliTable)
w.buf1.Reset() w.buf1.Reset()
w.buf1.PutBE32(hash) w.buf1.PutBE32(hash)
if err := w.writeAt(w.buf1.Get(), hashPos); err != nil { if err := w.writeAt(w.buf1.Get(), hashPos); err != nil {
return err return err
} }
// Load in the symbol table efficiently for the rest of the index writing . // Load in the symbol table efficiently for the rest of the index writing .
w.symbols, err = NewSymbols(realByteSlice(w.symbolFile.Bytes()), FormatV2 , int(w.toc.Symbols)) w.symbols, err = NewSymbols(realByteSlice(w.symbolFile.Bytes()), FormatV2 , int(w.toc.Symbols))
if err != nil { if err != nil {
skipping to change at line 703 skipping to change at line 703
w.buf1.PutBE32int(int(w.cntPO)) // Count. w.buf1.PutBE32int(int(w.cntPO)) // Count.
w.buf1.WriteToHash(w.crc32) w.buf1.WriteToHash(w.crc32)
if err := w.write(w.buf1.Get()); err != nil { if err := w.write(w.buf1.Get()); err != nil {
return err return err
} }
f, err := fileutil.OpenMmapFile(w.fPO.name) f, err := fileutil.OpenMmapFile(w.fPO.name)
if err != nil { if err != nil {
return err return err
} }
defer f.Close() defer func() {
if f != nil {
f.Close()
}
}()
d := encoding.NewDecbufRaw(realByteSlice(f.Bytes()), int(w.fPO.pos)) d := encoding.NewDecbufRaw(realByteSlice(f.Bytes()), int(w.fPO.pos))
cnt := w.cntPO cnt := w.cntPO
for d.Err() == nil && cnt > 0 { for d.Err() == nil && cnt > 0 {
w.buf1.Reset() w.buf1.Reset()
w.buf1.PutUvarint(d.Uvarint()) // Keycount. w.buf1.PutUvarint(d.Uvarint()) // Keycount.
w.buf1.PutUvarintStr(yoloString(d.UvarintBytes())) // Label name. w.buf1.PutUvarintStr(yoloString(d.UvarintBytes())) // Label name.
w.buf1.PutUvarintStr(yoloString(d.UvarintBytes())) // Label value . w.buf1.PutUvarintStr(yoloString(d.UvarintBytes())) // Label value .
w.buf1.PutUvarint64(d.Uvarint64() + adjustment) // Offset. w.buf1.PutUvarint64(d.Uvarint64() + adjustment) // Offset.
w.buf1.WriteToHash(w.crc32) w.buf1.WriteToHash(w.crc32)
if err := w.write(w.buf1.Get()); err != nil { if err := w.write(w.buf1.Get()); err != nil {
return err return err
} }
cnt-- cnt--
} }
if d.Err() != nil { if d.Err() != nil {
return d.Err() return d.Err()
} }
// Cleanup temporary file. // Cleanup temporary file.
if err := f.Close(); err != nil {
return err
}
f = nil
if err := w.fPO.close(); err != nil { if err := w.fPO.close(); err != nil {
return err return err
} }
if err := w.fPO.remove(); err != nil { if err := w.fPO.remove(); err != nil {
return err return err
} }
w.fPO = nil w.fPO = nil
// Write out the length. // Write out the length.
w.buf1.Reset() w.buf1.Reset()
skipping to change at line 965 skipping to change at line 973
func (s uint32slice) Len() int { return len(s) } func (s uint32slice) Len() int { return len(s) }
func (s uint32slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s uint32slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s uint32slice) Less(i, j int) bool { return s[i] < s[j] } func (s uint32slice) Less(i, j int) bool { return s[i] < s[j] }
type labelIndexHashEntry struct { type labelIndexHashEntry struct {
keys []string keys []string
offset uint64 offset uint64
} }
func (w *Writer) Close() error { func (w *Writer) Close() error {
if err := w.ensureStage(idxStageDone); err != nil { // Even if this fails, we need to close all the files.
return err ensureErr := w.ensureStage(idxStageDone)
}
if w.symbolFile != nil { if w.symbolFile != nil {
if err := w.symbolFile.Close(); err != nil { if err := w.symbolFile.Close(); err != nil {
return err return err
} }
} }
if w.fP != nil { if w.fP != nil {
if err := w.fP.close(); err != nil { if err := w.fP.close(); err != nil {
return err return err
} }
} }
if w.fPO != nil { if w.fPO != nil {
if err := w.fPO.close(); err != nil { if err := w.fPO.close(); err != nil {
return err return err
} }
} }
return w.f.close() if err := w.f.close(); err != nil {
return err
}
return ensureErr
} }
// StringTuples provides access to a sorted list of string tuples. // StringTuples provides access to a sorted list of string tuples.
type StringTuples interface { type StringTuples interface {
// Total number of tuples in the list. // Total number of tuples in the list.
Len() int Len() int
// At returns the tuple at position i. // At returns the tuple at position i.
At(i int) ([]string, error) At(i int) ([]string, error)
} }
skipping to change at line 1016 skipping to change at line 1027
type Reader struct { type Reader struct {
b ByteSlice b ByteSlice
toc *TOC toc *TOC
// Close that releases the underlying resources of the byte slice. // Close that releases the underlying resources of the byte slice.
c io.Closer c io.Closer
// Map of LabelName to a list of some LabelValues's position in the offse t table. // Map of LabelName to a list of some LabelValues's position in the offse t table.
// The first and last values for each name are always present. // The first and last values for each name are always present.
postings map[string][]postingOffset postings map[string][]postingOffset
// For the v1 format, labelname -> labelvalue -> offset.
postingsV1 map[string]map[string]uint64
symbols *Symbols symbols *Symbols
nameSymbols map[uint32]string // Cache of the label name symbol lookups, nameSymbols map[uint32]string // Cache of the label name symbol lookups,
// as there are not many and they are half of all lookups. // as there are not many and they are half of all lookups.
dec *Decoder dec *Decoder
version int version int
} }
skipping to change at line 1105 skipping to change at line 1118
r.toc, err = NewTOCFromByteSlice(b) r.toc, err = NewTOCFromByteSlice(b)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "read TOC") return nil, errors.Wrap(err, "read TOC")
} }
r.symbols, err = NewSymbols(r.b, r.version, int(r.toc.Symbols)) r.symbols, err = NewSymbols(r.b, r.version, int(r.toc.Symbols))
if err != nil { if err != nil {
return nil, errors.Wrap(err, "read symbols") return nil, errors.Wrap(err, "read symbols")
} }
var lastKey []string if r.version == FormatV1 {
lastOff := 0 // Earlier V1 formats don't have a sorted postings offset table,
valueCount := 0 so
// For the postings offset table we keep every label name but only every // load the whole offset table into memory.
nth r.postingsV1 = map[string]map[string]uint64{}
// label value (plus the first and last one), to save memory. if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []st
if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []string, _ ring, off uint64, _ int) error {
uint64, off int) error { if len(key) != 2 {
if len(key) != 2 { return errors.Errorf("unexpected key length for p
return errors.Errorf("unexpected key length for posting t osting table %d", len(key))
able %d", len(key)) }
if _, ok := r.postingsV1[key[0]]; !ok {
r.postingsV1[key[0]] = map[string]uint64{}
r.postings[key[0]] = nil // Used to get a list of
labelnames in places.
}
r.postingsV1[key[0]][key[1]] = off
return nil
}); err != nil {
return nil, errors.Wrap(err, "read postings table")
} }
if _, ok := r.postings[key[0]]; !ok { } else {
// Next label name. var lastKey []string
r.postings[key[0]] = []postingOffset{} lastOff := 0
if lastKey != nil { valueCount := 0
// Always include last value for each label name. // For the postings offset table we keep every label name but onl
r.postings[lastKey[0]] = append(r.postings[lastKe y every nth
y[0]], postingOffset{value: lastKey[1], off: lastOff}) // label value (plus the first and last one), to save memory.
} if err := ReadOffsetTable(r.b, r.toc.PostingsTable, func(key []st
lastKey = nil ring, _ uint64, off int) error {
valueCount = 0 if len(key) != 2 {
} return errors.Errorf("unexpected key length for p
if valueCount%32 == 0 { osting table %d", len(key))
r.postings[key[0]] = append(r.postings[key[0]], postingOf }
fset{value: key[1], off: off}) if _, ok := r.postings[key[0]]; !ok {
lastKey = nil // Next label name.
} else { r.postings[key[0]] = []postingOffset{}
lastKey = key if lastKey != nil {
lastOff = off // Always include last value for each lab
el name.
r.postings[lastKey[0]] = append(r.posting
s[lastKey[0]], postingOffset{value: lastKey[1], off: lastOff})
}
lastKey = nil
valueCount = 0
}
if valueCount%32 == 0 {
r.postings[key[0]] = append(r.postings[key[0]], p
ostingOffset{value: key[1], off: off})
lastKey = nil
} else {
lastKey = key
lastOff = off
}
valueCount++
return nil
}); err != nil {
return nil, errors.Wrap(err, "read postings table")
}
if lastKey != nil {
r.postings[lastKey[0]] = append(r.postings[lastKey[0]], p
ostingOffset{value: lastKey[1], off: lastOff})
}
// Trim any extra space in the slices.
for k, v := range r.postings {
l := make([]postingOffset, len(v))
copy(l, v)
r.postings[k] = l
} }
valueCount++
return nil
}); err != nil {
return nil, errors.Wrap(err, "read postings table")
}
if lastKey != nil {
r.postings[lastKey[0]] = append(r.postings[lastKey[0]], postingOf
fset{value: lastKey[1], off: lastOff})
}
// Trim any extra space in the slices.
for k, v := range r.postings {
l := make([]postingOffset, len(v))
copy(l, v)
r.postings[k] = l
} }
r.nameSymbols = make(map[uint32]string, len(r.postings)) r.nameSymbols = make(map[uint32]string, len(r.postings))
for k := range r.postings { for k := range r.postings {
if k == "" { if k == "" {
continue continue
} }
off, err := r.symbols.ReverseLookup(k) off, err := r.symbols.ReverseLookup(k)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "reverse symbol lookup") return nil, errors.Wrap(err, "reverse symbol lookup")
skipping to change at line 1400 skipping to change at line 1432
return uint64(r.symbols.Size()) return uint64(r.symbols.Size())
} }
// LabelValues returns value tuples that exist for the given label name tuples. // LabelValues returns value tuples that exist for the given label name tuples.
// It is not safe to use the return value beyond the lifetime of the byte slice // It is not safe to use the return value beyond the lifetime of the byte slice
// passed into the Reader. // passed into the Reader.
func (r *Reader) LabelValues(names ...string) (StringTuples, error) { func (r *Reader) LabelValues(names ...string) (StringTuples, error) {
if len(names) != 1 { if len(names) != 1 {
return nil, errors.Errorf("only one label name supported") return nil, errors.Errorf("only one label name supported")
} }
if r.version == FormatV1 {
e, ok := r.postingsV1[names[0]]
if !ok {
return emptyStringTuples{}, nil
}
values := make([]string, 0, len(e))
for k := range e {
values = append(values, k)
}
sort.Strings(values)
return NewStringTuples(values, 1)
}
e, ok := r.postings[names[0]] e, ok := r.postings[names[0]]
if !ok { if !ok {
return emptyStringTuples{}, nil return emptyStringTuples{}, nil
} }
if len(e) == 0 { if len(e) == 0 {
return emptyStringTuples{}, nil return emptyStringTuples{}, nil
} }
values := make([]string, 0, len(e)*symbolFactor) values := make([]string, 0, len(e)*symbolFactor)
d := encoding.NewDecbufAt(r.b, int(r.toc.PostingsTable), nil) d := encoding.NewDecbufAt(r.b, int(r.toc.PostingsTable), nil)
skipping to change at line 1459 skipping to change at line 1504
offset = id * 16 offset = id * 16
} }
d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable) d := encoding.NewDecbufUvarintAt(r.b, int(offset), castagnoliTable)
if d.Err() != nil { if d.Err() != nil {
return d.Err() return d.Err()
} }
return errors.Wrap(r.dec.Series(d.Get(), lbls, chks), "read series") return errors.Wrap(r.dec.Series(d.Get(), lbls, chks), "read series")
} }
func (r *Reader) Postings(name string, values ...string) (Postings, error) { func (r *Reader) Postings(name string, values ...string) (Postings, error) {
if r.version == FormatV1 {
e, ok := r.postingsV1[name]
if !ok {
return EmptyPostings(), nil
}
res := make([]Postings, 0, len(values))
for _, v := range values {
postingsOff, ok := e[v]
if !ok {
continue
}
// Read from the postings table.
d := encoding.NewDecbufAt(r.b, int(postingsOff), castagno
liTable)
_, p, err := r.dec.Postings(d.Get())
if err != nil {
return nil, errors.Wrap(err, "decode postings")
}
res = append(res, p)
}
return Merge(res...), nil
}
e, ok := r.postings[name] e, ok := r.postings[name]
if !ok { if !ok {
return EmptyPostings(), nil return EmptyPostings(), nil
} }
if len(values) == 0 { if len(values) == 0 {
return EmptyPostings(), nil return EmptyPostings(), nil
} }
res := make([]Postings, 0, len(values)) res := make([]Postings, 0, len(values))
 End of changes. 12 change blocks. 
50 lines changed or deleted 123 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)