Files
grafana/pkg/tsdb/influxdb/influxql/util/util.go
T
ismail simsek c088d003f2 InfluxDB: Implement InfluxQL json streaming parser (#76934)
* Have the first iteration

* Prepare bench testing

* rename the test files

* Remove unnecessary test file

* Introduce influxqlStreamingParser feature flag

* Apply streaming parser feature flag

* Add new tests

* More tests

* return executedQueryString only in first frame

* add frame meta and config

* Update golden json files

* Support tags/labels

* more tests

* more tests

* Don't change original response_parser.go

* provide context

* create util package

* don't pass the row

* update converter with formatted frameName

* add executedQueryString info only to first frame

* update golden files

* rename

* update test file

* use pointer values

* update testdata

* update parsing

* update converter for null values

* prepare converter for table response

* clean up

* return timeField in fields

* handle no time column responses

* better nil field handling

* refactor the code

* add table tests

* fix config for table

* table response format

* fix value

* if there is no time column set name

* linting

* refactoring

* handle the status code

* add tracing

* Update pkg/tsdb/influxdb/influxql/converter/converter_test.go

Co-authored-by: İnanç Gümüş <m@inanc.io>

* fix import

* update test data

* sanity

* sanity

* linting

* simplicity

* return empty rsp

* rename to prevent confusion

* nullableJson field type for null values

* better handling null values

* remove duplicate test file

* fix healthcheck

* use util for pointer

* move bench test to root

* provide fake feature manager

* add more tests

* partial fix for null values in table response format

* handle partial null fields

* comments for easy testing

* move frameName allocation in readSeries

* one less append operation

* performance improvement by making string conversion once

pkg: github.com/grafana/grafana/pkg/tsdb/influxdb/influxql
             │ stream2.txt │            stream3.txt             │
             │   sec/op    │   sec/op     vs base               │
ParseJson-10   314.4m ± 1%   303.9m ± 1%  -3.34% (p=0.000 n=10)

             │ stream2.txt  │             stream3.txt              │
             │     B/op     │     B/op      vs base                │
ParseJson-10   425.2Mi ± 0%   382.7Mi ± 0%  -10.00% (p=0.000 n=10)

             │ stream2.txt │            stream3.txt             │
             │  allocs/op  │  allocs/op   vs base               │
ParseJson-10   7.224M ± 0%   6.689M ± 0%  -7.41% (p=0.000 n=10)

* add comment lines

---------

Co-authored-by: İnanç Gümüş <m@inanc.io>
2023-12-06 12:39:05 +01:00

156 lines
3.5 KiB
Go

package util
import (
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/tsdb/influxdb/models"
)
var (
legendFormat = regexp.MustCompile(`\[\[([\@\/\w-]+)(\.[\@\/\w-]+)*\]\]*|\$([\@\w-]+?)*`)
)
const (
GraphVisType data.VisType = "graph"
TableVisType data.VisType = "table"
LogsVisType data.VisType = "logs"
)
func FormatFrameName(rowName, column string, tags map[string]string, query models.Query, frameName []byte) []byte {
if query.Alias == "" {
return BuildFrameNameFromQuery(rowName, column, tags, frameName, query.ResultFormat)
}
nameSegment := strings.Split(rowName, ".")
result := legendFormat.ReplaceAllFunc([]byte(query.Alias), func(in []byte) []byte {
aliasFormat := string(in)
aliasFormat = strings.Replace(aliasFormat, "[[", "", 1)
aliasFormat = strings.Replace(aliasFormat, "]]", "", 1)
aliasFormat = strings.Replace(aliasFormat, "$", "", 1)
if aliasFormat == "m" || aliasFormat == "measurement" {
return []byte(rowName)
}
if aliasFormat == "col" {
return []byte(column)
}
pos, err := strconv.Atoi(aliasFormat)
if err == nil && len(nameSegment) > pos {
return []byte(nameSegment[pos])
}
if !strings.HasPrefix(aliasFormat, "tag_") {
return in
}
tagKey := strings.Replace(aliasFormat, "tag_", "", 1)
tagValue, exist := tags[tagKey]
if exist {
return []byte(tagValue)
}
return in
})
return result
}
func BuildFrameNameFromQuery(rowName, column string, tags map[string]string, frameName []byte, resultFormat string) []byte {
if resultFormat != "table" {
frameName = append(frameName, rowName...)
frameName = append(frameName, '.')
}
frameName = append(frameName, column...)
if len(tags) == 0 {
return frameName
}
frameName = append(frameName, ' ', '{', ' ')
first := true
for k, v := range tags {
if !first {
frameName = append(frameName, ',', ' ')
} else {
first = false
}
frameName = append(frameName, k...)
frameName = append(frameName, ':', ' ')
frameName = append(frameName, v...)
}
return append(frameName, ' ', '}')
}
func ParseTimestamp(value any) (time.Time, error) {
timestampNumber, ok := value.(json.Number)
if !ok {
return time.Time{}, fmt.Errorf("timestamp-value has invalid type: %#v", value)
}
timestampInMilliseconds, err := timestampNumber.Int64()
if err != nil {
return time.Time{}, err
}
// currently in the code the influxdb-timestamps are requested with
// milliseconds-precision, meaning these values are milliseconds
t := time.UnixMilli(timestampInMilliseconds).UTC()
return t, nil
}
func Typeof(values [][]any, colIndex int) string {
for _, value := range values {
if value != nil && value[colIndex] != nil {
return fmt.Sprintf("%T", value[colIndex])
}
}
return "null"
}
func ParseNumber(value any) *float64 {
// NOTE: we use pointers-to-float64 because we need
// to represent null-json-values. they come for example
// when we do a group-by with fill(null)
if value == nil {
// this is what json-nulls become
return nil
}
number, ok := value.(json.Number)
if !ok {
// in the current implementation, errors become nils
return nil
}
fvalue, err := number.Float64()
if err != nil {
// in the current implementation, errors become nils
return nil
}
return &fvalue
}
func GetVisType(resFormat string) data.VisType {
switch resFormat {
case "table":
return TableVisType
case "logs":
return LogsVisType
default:
return GraphVisType
}
}
func ToPtr[T any](v T) *T {
return &v
}