Loki: backend: use streaming JSON parser, try2 (#48752)
* converter: remove __name__ customization because Loki does not do that Loki does not handle __name__ in a special way. for Prometheus, the caller can implement the formatting by themselves * converter: change labels-formatting the labels.String() method does not handle strange values well * loki: backend: use streaming-json parser * more idiomatic code Co-authored-by: Todd Treece <360020+toddtreece@users.noreply.github.com> * simpler row-length check * simpler code * fixed converter/prom tests Co-authored-by: Todd Treece <360020+toddtreece@users.noreply.github.com>
This commit is contained in:
Gábor Farkas
co-authored by
Todd Treece
Todd Treece
parent
8ed3fb1f2d
commit
da74dba7c8
+99
-15
@@ -2,6 +2,7 @@ package loki
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -39,20 +40,30 @@ func TestFormatName(t *testing.T) {
|
||||
|
||||
func TestAdjustFrame(t *testing.T) {
|
||||
t.Run("logs-frame metadata should be set correctly", func(t *testing.T) {
|
||||
time1 := time.Date(2022, 1, 2, 3, 4, 5, 6, time.UTC)
|
||||
time2 := time.Date(2022, 1, 2, 3, 5, 5, 6, time.UTC)
|
||||
time3 := time.Date(2022, 1, 2, 3, 5, 5, 6, time.UTC)
|
||||
time4 := time.Date(2022, 1, 2, 3, 6, 5, 6, time.UTC)
|
||||
|
||||
timeNs1 := strconv.FormatInt(time1.UnixNano(), 10)
|
||||
timeNs2 := strconv.FormatInt(time2.UnixNano(), 10)
|
||||
timeNs3 := strconv.FormatInt(time3.UnixNano(), 10)
|
||||
timeNs4 := strconv.FormatInt(time4.UnixNano(), 10)
|
||||
|
||||
frame := data.NewFrame("",
|
||||
data.NewField("labels", nil, []json.RawMessage{
|
||||
data.NewField("__labels", nil, []json.RawMessage{
|
||||
json.RawMessage(`{"level":"info"}`),
|
||||
json.RawMessage(`{"level":"error"}`),
|
||||
json.RawMessage(`{"level":"error"}`),
|
||||
json.RawMessage(`{"level":"info"}`),
|
||||
}),
|
||||
data.NewField("time", nil, []time.Time{
|
||||
time.Date(2022, 1, 2, 3, 4, 5, 6, time.UTC),
|
||||
time.Date(2022, 1, 2, 3, 5, 5, 6, time.UTC),
|
||||
time.Date(2022, 1, 2, 3, 5, 5, 6, time.UTC),
|
||||
time.Date(2022, 1, 2, 3, 6, 5, 6, time.UTC),
|
||||
data.NewField("Time", nil, []time.Time{
|
||||
time1, time2, time3, time4,
|
||||
}),
|
||||
data.NewField("Line", nil, []string{"line1", "line2", "line2", "line3"}),
|
||||
data.NewField("TS", nil, []string{
|
||||
timeNs1, timeNs2, timeNs3, timeNs4,
|
||||
}),
|
||||
data.NewField("line", nil, []string{"line1", "line2", "line2", "line3"}),
|
||||
)
|
||||
|
||||
frame.RefID = "A"
|
||||
@@ -68,14 +79,6 @@ func TestAdjustFrame(t *testing.T) {
|
||||
fields := frame.Fields
|
||||
|
||||
require.Equal(t, 5, len(fields))
|
||||
tsNsField := fields[3]
|
||||
require.Equal(t, "tsNs", tsNsField.Name)
|
||||
require.Equal(t, data.FieldTypeString, tsNsField.Type())
|
||||
require.Equal(t, 4, tsNsField.Len())
|
||||
require.Equal(t, "1641092645000000006", tsNsField.At(0))
|
||||
require.Equal(t, "1641092705000000006", tsNsField.At(1))
|
||||
require.Equal(t, "1641092705000000006", tsNsField.At(2))
|
||||
require.Equal(t, "1641092765000000006", tsNsField.At(3))
|
||||
|
||||
idField := fields[4]
|
||||
require.Equal(t, "id", idField.Name)
|
||||
@@ -136,4 +139,85 @@ func TestAdjustFrame(t *testing.T) {
|
||||
require.NotNil(t, timeFieldConfig)
|
||||
require.Equal(t, float64(42000), timeFieldConfig.Interval)
|
||||
})
|
||||
|
||||
t.Run("should parse response stats", func(t *testing.T) {
|
||||
stats := map[string]interface{}{
|
||||
"summary": map[string]interface{}{
|
||||
"bytesProcessedPerSecond": 1,
|
||||
"linesProcessedPerSecond": 2,
|
||||
"totalBytesProcessed": 3,
|
||||
"totalLinesProcessed": 4,
|
||||
"execTime": 5.5,
|
||||
},
|
||||
|
||||
"store": map[string]interface{}{
|
||||
"totalChunksRef": 6,
|
||||
"totalChunksDownloaded": 7,
|
||||
"chunksDownloadTime": 8.8,
|
||||
"headChunkBytes": 9,
|
||||
"headChunkLines": 10,
|
||||
"decompressedBytes": 11,
|
||||
"decompressedLines": 12,
|
||||
"compressedBytes": 13,
|
||||
"totalDuplicates": 14,
|
||||
},
|
||||
|
||||
"ingester": map[string]interface{}{
|
||||
"totalReached": 15,
|
||||
"totalChunksMatched": 16,
|
||||
"totalBatches": 17,
|
||||
"totalLinesSent": 18,
|
||||
"headChunkBytes": 19,
|
||||
"headChunkLines": 20,
|
||||
"decompressedBytes": 21,
|
||||
"decompressedLines": 22,
|
||||
"compressedBytes": 23,
|
||||
"totalDuplicates": 24,
|
||||
},
|
||||
}
|
||||
|
||||
meta := data.FrameMeta{
|
||||
Custom: map[string]interface{}{
|
||||
"stats": stats,
|
||||
},
|
||||
}
|
||||
|
||||
expected := []data.QueryStat{
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Summary: bytes processed per second", Unit: "Bps"}, Value: 1},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Summary: lines processed per second", Unit: ""}, Value: 2},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Summary: total bytes processed", Unit: "decbytes"}, Value: 3},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Summary: total lines processed", Unit: ""}, Value: 4},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Summary: exec time", Unit: "s"}, Value: 5.5},
|
||||
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: total chunks ref", Unit: ""}, Value: 6},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: total chunks downloaded", Unit: ""}, Value: 7},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: chunks download time", Unit: "s"}, Value: 8.8},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: head chunk bytes", Unit: "decbytes"}, Value: 9},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: head chunk lines", Unit: ""}, Value: 10},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: decompressed bytes", Unit: "decbytes"}, Value: 11},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: decompressed lines", Unit: ""}, Value: 12},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: compressed bytes", Unit: "decbytes"}, Value: 13},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Store: total duplicates", Unit: ""}, Value: 14},
|
||||
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: total reached", Unit: ""}, Value: 15},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: total chunks matched", Unit: ""}, Value: 16},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: total batches", Unit: ""}, Value: 17},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: total lines sent", Unit: ""}, Value: 18},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: head chunk bytes", Unit: "decbytes"}, Value: 19},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: head chunk lines", Unit: ""}, Value: 20},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: decompressed bytes", Unit: "decbytes"}, Value: 21},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: decompressed lines", Unit: ""}, Value: 22},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: compressed bytes", Unit: "decbytes"}, Value: 23},
|
||||
{FieldConfig: data.FieldConfig{DisplayName: "Ingester: total duplicates", Unit: ""}, Value: 24},
|
||||
}
|
||||
|
||||
result := parseStats(meta.Custom)
|
||||
|
||||
// NOTE: i compare it item-by-item otherwise the test-fail-error-message is very hard to read
|
||||
require.Len(t, result, len(expected))
|
||||
|
||||
for i := 0; i < len(result); i++ {
|
||||
require.Equal(t, expected[i], result[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user