"Fossies" - the Fresh Open Source Software Archive

Member "AdGuardHome-0.104.3/internal/querylog/qlog_file_test.go" (19 Nov 2020, 9028 Bytes) of package /linux/misc/dns/AdGuardHome-0.104.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Go source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "qlog_file_test.go": 0.104.1_vs_0.104.3.

    1 package querylog
    2 
    3 import (
    4     "encoding/binary"
    5     "io"
    6     "io/ioutil"
    7     "math"
    8     "net"
    9     "os"
   10     "strings"
   11     "testing"
   12     "time"
   13 
   14     "github.com/stretchr/testify/assert"
   15 )
   16 
   17 func TestQLogFileEmpty(t *testing.T) {
   18     testDir := prepareTestDir()
   19     defer func() { _ = os.RemoveAll(testDir) }()
   20     testFile := prepareTestFile(testDir, 0)
   21 
   22     // create the new QLogFile instance
   23     q, err := NewQLogFile(testFile)
   24     assert.Nil(t, err)
   25     assert.NotNil(t, q)
   26     defer q.Close()
   27 
   28     // seek to the start
   29     pos, err := q.SeekStart()
   30     assert.Nil(t, err)
   31     assert.Equal(t, int64(0), pos)
   32 
   33     // try reading anyway
   34     line, err := q.ReadNext()
   35     assert.Equal(t, io.EOF, err)
   36     assert.Equal(t, "", line)
   37 }
   38 
   39 func TestQLogFileLarge(t *testing.T) {
   40     // should be large enough
   41     count := 50000
   42 
   43     testDir := prepareTestDir()
   44     defer func() { _ = os.RemoveAll(testDir) }()
   45     testFile := prepareTestFile(testDir, count)
   46 
   47     // create the new QLogFile instance
   48     q, err := NewQLogFile(testFile)
   49     assert.Nil(t, err)
   50     assert.NotNil(t, q)
   51     defer q.Close()
   52 
   53     // seek to the start
   54     pos, err := q.SeekStart()
   55     assert.Nil(t, err)
   56     assert.NotEqual(t, int64(0), pos)
   57 
   58     read := 0
   59     var line string
   60     for err == nil {
   61         line, err = q.ReadNext()
   62         if err == nil {
   63             assert.True(t, len(line) > 0)
   64             read += 1
   65         }
   66     }
   67 
   68     assert.Equal(t, count, read)
   69     assert.Equal(t, io.EOF, err)
   70 }
   71 
   72 func TestQLogFileSeekLargeFile(t *testing.T) {
   73     // more or less big file
   74     count := 10000
   75 
   76     testDir := prepareTestDir()
   77     defer func() { _ = os.RemoveAll(testDir) }()
   78     testFile := prepareTestFile(testDir, count)
   79 
   80     // create the new QLogFile instance
   81     q, err := NewQLogFile(testFile)
   82     assert.Nil(t, err)
   83     assert.NotNil(t, q)
   84     defer q.Close()
   85 
   86     // CASE 1: NOT TOO OLD LINE
   87     testSeekLineQLogFile(t, q, 300)
   88 
   89     // CASE 2: OLD LINE
   90     testSeekLineQLogFile(t, q, count-300)
   91 
   92     // CASE 3: FIRST LINE
   93     testSeekLineQLogFile(t, q, 0)
   94 
   95     // CASE 4: LAST LINE
   96     testSeekLineQLogFile(t, q, count)
   97 
   98     // CASE 5: Seek non-existent (too low)
   99     _, _, err = q.Seek(123)
  100     assert.NotNil(t, err)
  101 
  102     // CASE 6: Seek non-existent (too high)
  103     ts, _ := time.Parse(time.RFC3339, "2100-01-02T15:04:05Z07:00")
  104     _, _, err = q.Seek(ts.UnixNano())
  105     assert.NotNil(t, err)
  106 
  107     // CASE 7: "Almost" found
  108     line, err := getQLogFileLine(q, count/2)
  109     assert.Nil(t, err)
  110     // ALMOST the record we need
  111     timestamp := readQLogTimestamp(line) - 1
  112     assert.NotEqual(t, uint64(0), timestamp)
  113     _, depth, err := q.Seek(timestamp)
  114     assert.NotNil(t, err)
  115     assert.True(t, depth <= int(math.Log2(float64(count))+3))
  116 }
  117 
  118 func TestQLogFileSeekSmallFile(t *testing.T) {
  119     // more or less big file
  120     count := 10
  121 
  122     testDir := prepareTestDir()
  123     defer func() { _ = os.RemoveAll(testDir) }()
  124     testFile := prepareTestFile(testDir, count)
  125 
  126     // create the new QLogFile instance
  127     q, err := NewQLogFile(testFile)
  128     assert.Nil(t, err)
  129     assert.NotNil(t, q)
  130     defer q.Close()
  131 
  132     // CASE 1: NOT TOO OLD LINE
  133     testSeekLineQLogFile(t, q, 2)
  134 
  135     // CASE 2: OLD LINE
  136     testSeekLineQLogFile(t, q, count-2)
  137 
  138     // CASE 3: FIRST LINE
  139     testSeekLineQLogFile(t, q, 0)
  140 
  141     // CASE 4: LAST LINE
  142     testSeekLineQLogFile(t, q, count)
  143 
  144     // CASE 5: Seek non-existent (too low)
  145     _, _, err = q.Seek(123)
  146     assert.NotNil(t, err)
  147 
  148     // CASE 6: Seek non-existent (too high)
  149     ts, _ := time.Parse(time.RFC3339, "2100-01-02T15:04:05Z07:00")
  150     _, _, err = q.Seek(ts.UnixNano())
  151     assert.NotNil(t, err)
  152 
  153     // CASE 7: "Almost" found
  154     line, err := getQLogFileLine(q, count/2)
  155     assert.Nil(t, err)
  156     // ALMOST the record we need
  157     timestamp := readQLogTimestamp(line) - 1
  158     assert.NotEqual(t, uint64(0), timestamp)
  159     _, depth, err := q.Seek(timestamp)
  160     assert.NotNil(t, err)
  161     assert.True(t, depth <= int(math.Log2(float64(count))+3))
  162 }
  163 
  164 func testSeekLineQLogFile(t *testing.T, q *QLogFile, lineNumber int) {
  165     line, err := getQLogFileLine(q, lineNumber)
  166     assert.Nil(t, err)
  167     ts := readQLogTimestamp(line)
  168     assert.NotEqual(t, uint64(0), ts)
  169 
  170     // try seeking to that line now
  171     pos, _, err := q.Seek(ts)
  172     assert.Nil(t, err)
  173     assert.NotEqual(t, int64(0), pos)
  174 
  175     testLine, err := q.ReadNext()
  176     assert.Nil(t, err)
  177     assert.Equal(t, line, testLine)
  178 }
  179 
  180 func getQLogFileLine(q *QLogFile, lineNumber int) (string, error) {
  181     _, err := q.SeekStart()
  182     if err != nil {
  183         return "", err
  184     }
  185 
  186     for i := 1; i < lineNumber; i++ {
  187         _, err := q.ReadNext()
  188         if err != nil {
  189             return "", err
  190         }
  191     }
  192     return q.ReadNext()
  193 }
  194 
  195 // Check adding and loading (with filtering) entries from disk and memory
  196 func TestQLogFile(t *testing.T) {
  197     testDir := prepareTestDir()
  198     defer func() { _ = os.RemoveAll(testDir) }()
  199     testFile := prepareTestFile(testDir, 2)
  200 
  201     // create the new QLogFile instance
  202     q, err := NewQLogFile(testFile)
  203     assert.Nil(t, err)
  204     assert.NotNil(t, q)
  205     defer q.Close()
  206 
  207     // seek to the start
  208     pos, err := q.SeekStart()
  209     assert.Nil(t, err)
  210     assert.True(t, pos > 0)
  211 
  212     // read first line
  213     line, err := q.ReadNext()
  214     assert.Nil(t, err)
  215     assert.True(t, strings.Contains(line, "0.0.0.2"), line)
  216     assert.True(t, strings.HasPrefix(line, "{"), line)
  217     assert.True(t, strings.HasSuffix(line, "}"), line)
  218 
  219     // read second line
  220     line, err = q.ReadNext()
  221     assert.Nil(t, err)
  222     assert.Equal(t, int64(0), q.position)
  223     assert.True(t, strings.Contains(line, "0.0.0.1"), line)
  224     assert.True(t, strings.HasPrefix(line, "{"), line)
  225     assert.True(t, strings.HasSuffix(line, "}"), line)
  226 
  227     // try reading again (there's nothing to read anymore)
  228     line, err = q.ReadNext()
  229     assert.Equal(t, io.EOF, err)
  230     assert.Equal(t, "", line)
  231 }
  232 
  233 // prepareTestFile - prepares a test query log file with the specified number of lines
  234 func prepareTestFile(dir string, linesCount int) string {
  235     return prepareTestFiles(dir, 1, linesCount)[0]
  236 }
  237 
  238 // prepareTestFiles - prepares several test query log files
  239 // each of them -- with the specified linesCount
  240 func prepareTestFiles(dir string, filesCount, linesCount int) []string {
  241     format := `{"IP":"${IP}","T":"${TIMESTAMP}","QH":"example.org","QT":"A","QC":"IN","Answer":"AAAAAAABAAEAAAAAB2V4YW1wbGUDb3JnAAABAAEHZXhhbXBsZQNvcmcAAAEAAQAAAAAABAECAwQ=","Result":{},"Elapsed":0,"Upstream":"upstream"}`
  242 
  243     lineTime, _ := time.Parse(time.RFC3339Nano, "2020-02-18T22:36:35.920973+03:00")
  244     lineIP := uint32(0)
  245 
  246     files := make([]string, filesCount)
  247     for j := 0; j < filesCount; j++ {
  248         f, _ := ioutil.TempFile(dir, "*.txt")
  249         files[filesCount-j-1] = f.Name()
  250 
  251         for i := 0; i < linesCount; i++ {
  252             lineIP += 1
  253             lineTime = lineTime.Add(time.Second)
  254 
  255             ip := make(net.IP, 4)
  256             binary.BigEndian.PutUint32(ip, lineIP)
  257 
  258             line := format
  259             line = strings.ReplaceAll(line, "${IP}", ip.String())
  260             line = strings.ReplaceAll(line, "${TIMESTAMP}", lineTime.Format(time.RFC3339Nano))
  261 
  262             _, _ = f.WriteString(line)
  263             _, _ = f.WriteString("\n")
  264         }
  265     }
  266 
  267     return files
  268 }
  269 
  270 func TestQLogSeek(t *testing.T) {
  271     testDir := prepareTestDir()
  272     defer func() { _ = os.RemoveAll(testDir) }()
  273 
  274     d := `{"T":"2020-08-31T18:44:23.911246629+03:00","QH":"wfqvjymurpwegyv","QT":"A","QC":"IN","CP":"","Answer":"","Result":{},"Elapsed":66286385,"Upstream":"tls://dns-unfiltered.adguard.com:853"}
  275 {"T":"2020-08-31T18:44:25.376690873+03:00"}
  276 {"T":"2020-08-31T18:44:25.382540454+03:00"}`
  277     f, _ := ioutil.TempFile(testDir, "*.txt")
  278     _, _ = f.WriteString(d)
  279     defer f.Close()
  280 
  281     q, err := NewQLogFile(f.Name())
  282     assert.Nil(t, err)
  283     defer q.Close()
  284 
  285     target, _ := time.Parse(time.RFC3339, "2020-08-31T18:44:25.376690873+03:00")
  286 
  287     _, depth, err := q.Seek(target.UnixNano())
  288     assert.Nil(t, err)
  289     assert.Equal(t, 1, depth)
  290 }
  291 
  292 func TestQLogSeek_ErrTSTooLate(t *testing.T) {
  293     testDir := prepareTestDir()
  294     t.Cleanup(func() {
  295         _ = os.RemoveAll(testDir)
  296     })
  297 
  298     d := `{"T":"2020-08-31T18:44:23.911246629+03:00","QH":"wfqvjymurpwegyv","QT":"A","QC":"IN","CP":"","Answer":"","Result":{},"Elapsed":66286385,"Upstream":"tls://dns-unfiltered.adguard.com:853"}
  299 {"T":"2020-08-31T18:44:25.376690873+03:00"}
  300 {"T":"2020-08-31T18:44:25.382540454+03:00"}
  301 `
  302     f, err := ioutil.TempFile(testDir, "*.txt")
  303     assert.Nil(t, err)
  304     defer f.Close()
  305 
  306     _, err = f.WriteString(d)
  307     assert.Nil(t, err)
  308 
  309     q, err := NewQLogFile(f.Name())
  310     assert.Nil(t, err)
  311     defer q.Close()
  312 
  313     target, err := time.Parse(time.RFC3339, "2020-08-31T18:44:25.382540454+03:00")
  314     assert.Nil(t, err)
  315 
  316     _, depth, err := q.Seek(target.UnixNano() + int64(time.Second))
  317     assert.Equal(t, ErrTSTooLate, err)
  318     assert.Equal(t, 2, depth)
  319 }
  320 
  321 func TestQLogSeek_ErrTSTooEarly(t *testing.T) {
  322     testDir := prepareTestDir()
  323     t.Cleanup(func() {
  324         _ = os.RemoveAll(testDir)
  325     })
  326 
  327     d := `{"T":"2020-08-31T18:44:23.911246629+03:00","QH":"wfqvjymurpwegyv","QT":"A","QC":"IN","CP":"","Answer":"","Result":{},"Elapsed":66286385,"Upstream":"tls://dns-unfiltered.adguard.com:853"}
  328 {"T":"2020-08-31T18:44:25.376690873+03:00"}
  329 {"T":"2020-08-31T18:44:25.382540454+03:00"}
  330 `
  331     f, err := ioutil.TempFile(testDir, "*.txt")
  332     assert.Nil(t, err)
  333     defer f.Close()
  334 
  335     _, err = f.WriteString(d)
  336     assert.Nil(t, err)
  337 
  338     q, err := NewQLogFile(f.Name())
  339     assert.Nil(t, err)
  340     defer q.Close()
  341 
  342     target, err := time.Parse(time.RFC3339, "2020-08-31T18:44:23.911246629+03:00")
  343     assert.Nil(t, err)
  344 
  345     _, depth, err := q.Seek(target.UnixNano() - int64(time.Second))
  346     assert.Equal(t, ErrTSTooEarly, err)
  347     assert.Equal(t, 1, depth)
  348 }