Azure Monitor: Add logs query builder (#99055)

2025-04-02 08:15:25 -06:00
parent 44ca402116
commit 3b73ebb210
42 changed files with 3306 additions and 59 deletions
@@ -157,6 +157,10 @@ type AzureLogsQuery struct {
 	BasicLogsQuery *bool `json:"basicLogsQuery,omitempty"`
 	// Workspace ID. This was removed in Grafana 8, but remains for backwards compat.
 	Workspace *string `json:"workspace,omitempty"`
+	// Denotes if logs query editor is in builder mode
+	Mode *LogsEditorMode `json:"mode,omitempty"`
+	// Builder query to be executed.
+	BuilderQuery *BuilderQueryExpression `json:"builderQuery,omitempty"`
 	// @deprecated Use resources instead
 	Resource *string `json:"resource,omitempty"`
 	// @deprecated Use dashboardTime instead
@@ -177,6 +181,217 @@ const (
 	ResultFormatLogs       ResultFormat = "logs"
 )

+type LogsEditorMode string
+
+const (
+	LogsEditorModeBuilder LogsEditorMode = "builder"
+	LogsEditorModeRaw     LogsEditorMode = "raw"
+)
+
+type BuilderQueryExpression struct {
+	From        *BuilderQueryEditorPropertyExpression     `json:"from,omitempty"`
+	Columns     *BuilderQueryEditorColumnsExpression      `json:"columns,omitempty"`
+	Where       *BuilderQueryEditorWhereExpressionArray   `json:"where,omitempty"`
+	Reduce      *BuilderQueryEditorReduceExpressionArray  `json:"reduce,omitempty"`
+	GroupBy     *BuilderQueryEditorGroupByExpressionArray `json:"groupBy,omitempty"`
+	Limit       *int64                                    `json:"limit,omitempty"`
+	OrderBy     *BuilderQueryEditorOrderByExpressionArray `json:"orderBy,omitempty"`
+	FuzzySearch *BuilderQueryEditorWhereExpressionArray   `json:"fuzzySearch,omitempty"`
+	TimeFilter  *BuilderQueryEditorWhereExpressionArray   `json:"timeFilter,omitempty"`
+}
+
+// NewBuilderQueryExpression creates a new BuilderQueryExpression object.
+func NewBuilderQueryExpression() *BuilderQueryExpression {
+	return &BuilderQueryExpression{}
+}
+
+type BuilderQueryEditorPropertyExpression struct {
+	Property BuilderQueryEditorProperty       `json:"property"`
+	Type     BuilderQueryEditorExpressionType `json:"type"`
+}
+
+// NewBuilderQueryEditorPropertyExpression creates a new BuilderQueryEditorPropertyExpression object.
+func NewBuilderQueryEditorPropertyExpression() *BuilderQueryEditorPropertyExpression {
+	return &BuilderQueryEditorPropertyExpression{
+		Property: *NewBuilderQueryEditorProperty(),
+	}
+}
+
+type BuilderQueryEditorProperty struct {
+	Type BuilderQueryEditorPropertyType `json:"type"`
+	Name string                         `json:"name"`
+}
+
+// NewBuilderQueryEditorProperty creates a new BuilderQueryEditorProperty object.
+func NewBuilderQueryEditorProperty() *BuilderQueryEditorProperty {
+	return &BuilderQueryEditorProperty{}
+}
+
+type BuilderQueryEditorPropertyType string
+
+const (
+	BuilderQueryEditorPropertyTypeNumber   BuilderQueryEditorPropertyType = "number"
+	BuilderQueryEditorPropertyTypeString   BuilderQueryEditorPropertyType = "string"
+	BuilderQueryEditorPropertyTypeBoolean  BuilderQueryEditorPropertyType = "boolean"
+	BuilderQueryEditorPropertyTypeDatetime BuilderQueryEditorPropertyType = "datetime"
+	BuilderQueryEditorPropertyTypeTimeSpan BuilderQueryEditorPropertyType = "time_span"
+	BuilderQueryEditorPropertyTypeFunction BuilderQueryEditorPropertyType = "function"
+	BuilderQueryEditorPropertyTypeInterval BuilderQueryEditorPropertyType = "interval"
+)
+
+type BuilderQueryEditorExpressionType string
+
+const (
+	BuilderQueryEditorExpressionTypeProperty          BuilderQueryEditorExpressionType = "property"
+	BuilderQueryEditorExpressionTypeOperator          BuilderQueryEditorExpressionType = "operator"
+	BuilderQueryEditorExpressionTypeReduce            BuilderQueryEditorExpressionType = "reduce"
+	BuilderQueryEditorExpressionTypeFunctionParameter BuilderQueryEditorExpressionType = "function_parameter"
+	BuilderQueryEditorExpressionTypeGroupBy           BuilderQueryEditorExpressionType = "group_by"
+	BuilderQueryEditorExpressionTypeOr                BuilderQueryEditorExpressionType = "or"
+	BuilderQueryEditorExpressionTypeAnd               BuilderQueryEditorExpressionType = "and"
+	BuilderQueryEditorExpressionTypeOrderBy           BuilderQueryEditorExpressionType = "order_by"
+)
+
+type BuilderQueryEditorColumnsExpression struct {
+	Columns []string                         `json:"columns,omitempty"`
+	Type    BuilderQueryEditorExpressionType `json:"type"`
+}
+
+// NewBuilderQueryEditorColumnsExpression creates a new BuilderQueryEditorColumnsExpression object.
+func NewBuilderQueryEditorColumnsExpression() *BuilderQueryEditorColumnsExpression {
+	return &BuilderQueryEditorColumnsExpression{}
+}
+
+type BuilderQueryEditorWhereExpressionArray struct {
+	Expressions []BuilderQueryEditorWhereExpression `json:"expressions"`
+	Type        BuilderQueryEditorExpressionType    `json:"type"`
+}
+
+// NewBuilderQueryEditorWhereExpressionArray creates a new BuilderQueryEditorWhereExpressionArray object.
+func NewBuilderQueryEditorWhereExpressionArray() *BuilderQueryEditorWhereExpressionArray {
+	return &BuilderQueryEditorWhereExpressionArray{}
+}
+
+type BuilderQueryEditorWhereExpression struct {
+	Type        BuilderQueryEditorExpressionType         `json:"type"`
+	Expressions []BuilderQueryEditorWhereExpressionItems `json:"expressions"`
+}
+
+// NewBuilderQueryEditorWhereExpression creates a new BuilderQueryEditorWhereExpression object.
+func NewBuilderQueryEditorWhereExpression() *BuilderQueryEditorWhereExpression {
+	return &BuilderQueryEditorWhereExpression{}
+}
+
+type BuilderQueryEditorWhereExpressionItems struct {
+	Property BuilderQueryEditorProperty       `json:"property"`
+	Operator BuilderQueryEditorOperator       `json:"operator"`
+	Type     BuilderQueryEditorExpressionType `json:"type"`
+}
+
+// NewBuilderQueryEditorWhereExpressionItems creates a new BuilderQueryEditorWhereExpressionItems object.
+func NewBuilderQueryEditorWhereExpressionItems() *BuilderQueryEditorWhereExpressionItems {
+	return &BuilderQueryEditorWhereExpressionItems{
+		Property: *NewBuilderQueryEditorProperty(),
+		Operator: *NewBuilderQueryEditorOperator(),
+	}
+}
+
+type BuilderQueryEditorOperator struct {
+	Name       string  `json:"name"`
+	Value      string  `json:"value"`
+	LabelValue *string `json:"labelValue,omitempty"`
+}
+
+// NewBuilderQueryEditorOperator creates a new BuilderQueryEditorOperator object.
+func NewBuilderQueryEditorOperator() *BuilderQueryEditorOperator {
+	return &BuilderQueryEditorOperator{}
+}
+
+type BuilderQueryEditorReduceExpressionArray struct {
+	Expressions []BuilderQueryEditorReduceExpression `json:"expressions"`
+	Type        BuilderQueryEditorExpressionType     `json:"type"`
+}
+
+// NewBuilderQueryEditorReduceExpressionArray creates a new BuilderQueryEditorReduceExpressionArray object.
+func NewBuilderQueryEditorReduceExpressionArray() *BuilderQueryEditorReduceExpressionArray {
+	return &BuilderQueryEditorReduceExpressionArray{}
+}
+
+type BuilderQueryEditorReduceExpression struct {
+	Property   *BuilderQueryEditorProperty                     `json:"property,omitempty"`
+	Reduce     *BuilderQueryEditorProperty                     `json:"reduce,omitempty"`
+	Parameters []BuilderQueryEditorFunctionParameterExpression `json:"parameters,omitempty"`
+	Focus      *bool                                           `json:"focus,omitempty"`
+}
+
+// NewBuilderQueryEditorReduceExpression creates a new BuilderQueryEditorReduceExpression object.
+func NewBuilderQueryEditorReduceExpression() *BuilderQueryEditorReduceExpression {
+	return &BuilderQueryEditorReduceExpression{}
+}
+
+type BuilderQueryEditorFunctionParameterExpression struct {
+	Value     string                           `json:"value"`
+	FieldType BuilderQueryEditorPropertyType   `json:"fieldType"`
+	Type      BuilderQueryEditorExpressionType `json:"type"`
+}
+
+// NewBuilderQueryEditorFunctionParameterExpression creates a new BuilderQueryEditorFunctionParameterExpression object.
+func NewBuilderQueryEditorFunctionParameterExpression() *BuilderQueryEditorFunctionParameterExpression {
+	return &BuilderQueryEditorFunctionParameterExpression{}
+}
+
+type BuilderQueryEditorGroupByExpressionArray struct {
+	Expressions []BuilderQueryEditorGroupByExpression `json:"expressions"`
+	Type        BuilderQueryEditorExpressionType      `json:"type"`
+}
+
+// NewBuilderQueryEditorGroupByExpressionArray creates a new BuilderQueryEditorGroupByExpressionArray object.
+func NewBuilderQueryEditorGroupByExpressionArray() *BuilderQueryEditorGroupByExpressionArray {
+	return &BuilderQueryEditorGroupByExpressionArray{}
+}
+
+type BuilderQueryEditorGroupByExpression struct {
+	Property *BuilderQueryEditorProperty       `json:"property,omitempty"`
+	Interval *BuilderQueryEditorProperty       `json:"interval,omitempty"`
+	Focus    *bool                             `json:"focus,omitempty"`
+	Type     *BuilderQueryEditorExpressionType `json:"type,omitempty"`
+}
+
+// NewBuilderQueryEditorGroupByExpression creates a new BuilderQueryEditorGroupByExpression object.
+func NewBuilderQueryEditorGroupByExpression() *BuilderQueryEditorGroupByExpression {
+	return &BuilderQueryEditorGroupByExpression{}
+}
+
+type BuilderQueryEditorOrderByExpressionArray struct {
+	Expressions []BuilderQueryEditorOrderByExpression `json:"expressions"`
+	Type        BuilderQueryEditorExpressionType      `json:"type"`
+}
+
+// NewBuilderQueryEditorOrderByExpressionArray creates a new BuilderQueryEditorOrderByExpressionArray object.
+func NewBuilderQueryEditorOrderByExpressionArray() *BuilderQueryEditorOrderByExpressionArray {
+	return &BuilderQueryEditorOrderByExpressionArray{}
+}
+
+type BuilderQueryEditorOrderByExpression struct {
+	Property BuilderQueryEditorProperty       `json:"property"`
+	Order    BuilderQueryEditorOrderByOptions `json:"order"`
+	Type     BuilderQueryEditorExpressionType `json:"type"`
+}
+
+// NewBuilderQueryEditorOrderByExpression creates a new BuilderQueryEditorOrderByExpression object.
+func NewBuilderQueryEditorOrderByExpression() *BuilderQueryEditorOrderByExpression {
+	return &BuilderQueryEditorOrderByExpression{
+		Property: *NewBuilderQueryEditorProperty(),
+	}
+}
+
+type BuilderQueryEditorOrderByOptions string
+
+const (
+	BuilderQueryEditorOrderByOptionsAsc  BuilderQueryEditorOrderByOptions = "asc"
+	BuilderQueryEditorOrderByOptionsDesc BuilderQueryEditorOrderByOptions = "desc"
+)
+
 type AzureResourceGraphQuery struct {
 	// Azure Resource Graph KQL query to be executed.
 	Query *string `json:"query,omitempty"`
@@ -391,6 +606,23 @@ const (
 	AzureQueryTypeCustomMetricNamesQuery    AzureQueryType = "Azure Custom Metric Names"
 )

+type SelectableValue struct {
+	Label string `json:"label"`
+	Value string `json:"value"`
+}
+
+// NewSelectableValue creates a new SelectableValue object.
+func NewSelectableValue() *SelectableValue {
+	return &SelectableValue{}
+}
+
+type BuilderQueryEditorOperatorType = StringOrBoolOrFloat64OrSelectableValue
+
+// NewBuilderQueryEditorOperatorType creates a new BuilderQueryEditorOperatorType object.
+func NewBuilderQueryEditorOperatorType() *BuilderQueryEditorOperatorType {
+	return NewStringOrBoolOrFloat64OrSelectableValue()
+}
+
 type GrafanaTemplateVariableQueryType string

 const (
@@ -569,3 +801,15 @@ func (resource *AppInsightsMetricNameQueryOrAppInsightsGroupByQueryOrSubscriptio

 	return fmt.Errorf("could not unmarshal resource with `kind = %v`", discriminator)
 }
+
+type StringOrBoolOrFloat64OrSelectableValue struct {
+	String          *string          `json:"String,omitempty"`
+	Bool            *bool            `json:"Bool,omitempty"`
+	Float64         *float64         `json:"Float64,omitempty"`
+	SelectableValue *SelectableValue `json:"SelectableValue,omitempty"`
+}
+
+// NewStringOrBoolOrFloat64OrSelectableValue creates a new StringOrBoolOrFloat64OrSelectableValue object.
+func NewStringOrBoolOrFloat64OrSelectableValue() *StringOrBoolOrFloat64OrSelectableValue {
+	return &StringOrBoolOrFloat64OrSelectableValue{}
+}
@@ -28,6 +28,18 @@ import (
 	"github.com/grafana/grafana/pkg/tsdb/azuremonitor/utils"
 )

+// Returns tables with the `HasData` field set to true
+func filterTablesWithData(tables []types.MetadataTable) []types.MetadataTable {
+	filtered := []types.MetadataTable{}
+	for _, table := range tables {
+		if table.HasData {
+			filtered = append(filtered, table)
+		}
+	}
+
+	return filtered
+}
+
 func (e *AzureLogAnalyticsDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) (http.ResponseWriter, error) {
 	if req.URL.Path == "/usage/basiclogs" {
 		newUrl := &url.URL{
@@ -36,7 +48,57 @@ func (e *AzureLogAnalyticsDatasource) ResourceRequest(rw http.ResponseWriter, re
 			Path:   "/v1/query",
 		}
 		return e.GetBasicLogsUsage(req.Context(), newUrl.String(), cli, rw, req.Body)
+	} else if strings.Contains(req.URL.Path, "/metadata") {
+		// Add necessary headers
+		req.Header.Set("Prefer", "metadata-format-v4,exclude-resourcetypes,exclude-customfunctions")
+		queryParams := req.URL.Query()
+		// Add necessary query params
+		queryParams.Add("select", "categories,solutions,tables,workspaces")
+		req.URL.RawQuery = queryParams.Encode()
+		resp, err := cli.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch metadata: %w", err)
+		}
+
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				e.Logger.Warn("Failed to close response body for metadata request", "err", err)
+			}
+		}()
+
+		encoding := resp.Header.Get("Content-Encoding")
+		body, err := decode(encoding, resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read metadata response: %w", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("metadata API error: %s", string(body))
+		}
+
+		var metadata types.AzureLogAnalyticsMetadata
+		err = json.Unmarshal(body, &metadata)
+		if err != nil {
+			return nil, fmt.Errorf("failed to unmarshal metadata response: %w", err)
+		}
+		metadata.Tables = filterTablesWithData(metadata.Tables)
+
+		responseBody, err := json.Marshal(metadata)
+		if err != nil {
+			return nil, fmt.Errorf("failed to marshal metadata response: %w", err)
+		}
+
+		rw.Header().Set("Content-Type", "application/json")
+		rw.WriteHeader(http.StatusOK)
+		_, err = rw.Write(responseBody)
+		if err != nil {
+			return nil, fmt.Errorf("failed to write metadata response: %w", err)
+		}
+
+		return rw, nil
 	}
+
+	// Default behavior for other requests
 	return e.Proxy.Do(rw, req, cli)
 }

@@ -1,12 +1,16 @@
 package loganalytics

 import (
+	"compress/flate"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"regexp"
 	"strconv"
 	"strings"
 	"time"

+	"github.com/andybalholm/brotli"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
@@ -136,3 +140,40 @@ func ConvertTime(timeStamp string) (time.Time, error) {
 func GetDataVolumeRawQuery(table string) string {
 	return fmt.Sprintf("Usage \n| where DataType == \"%s\"\n| where IsBillable == true\n| summarize BillableDataGB = round(sum(Quantity) / 1000, 3)", table)
 }
+
+// This function handles various compression mechanisms that may have been used on a response body
+func decode(encoding string, original io.ReadCloser) ([]byte, error) {
+	var reader io.Reader
+	var err error
+	switch encoding {
+	case "gzip":
+		reader, err = gzip.NewReader(original)
+		if err != nil {
+			return nil, err
+		}
+		defer func() {
+			if err := reader.(io.ReadCloser).Close(); err != nil {
+				backend.Logger.Warn("Failed to close reader body", "err", err)
+			}
+		}()
+	case "deflate":
+		reader = flate.NewReader(original)
+		defer func() {
+			if err := reader.(io.ReadCloser).Close(); err != nil {
+				backend.Logger.Warn("Failed to close reader body", "err", err)
+			}
+		}()
+	case "br":
+		reader = brotli.NewReader(original)
+	case "":
+		reader = original
+	default:
+		return nil, fmt.Errorf("unexpected encoding type %v", err)
+	}
+
+	body, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}
@@ -210,3 +210,119 @@ type SubscriptionsResponse struct {
 }

 var ErrorAzureHealthCheck = errors.New("health check failed")
+
+// AzureLogAnalyticsMetadata represents the metadata response from the Azure Log Analytics API.
+// Types are taken from https://learn.microsoft.com/en-us/rest/api/loganalytics/metadata/get
+type AzureLogAnalyticsMetadata struct {
+	Applications  []MetadataApplications  `json:"applications"`
+	Categories    []MetadataCategories    `json:"categories"`
+	Functions     []MetadataFunctions     `json:"functions"`
+	Permissions   []MetadataPermissions   `json:"permissions"`
+	Queries       []MetadataQueries       `json:"queries"`
+	ResourceTypes []MetadataResourceTypes `json:"resourceTypes"`
+	Resources     []Resources             `json:"resources"`
+	Solutions     []MetadataSolutions     `json:"solutions"`
+	Tables        []MetadataTable         `json:"tables"`
+	Workspaces    []MetadataWorkspaces    `json:"workspaces"`
+}
+
+// MetadataTable represents a table entry in the metadata response.
+type MetadataTable struct {
+	Columns        []Columns      `json:"columns"`
+	Description    string         `json:"description"`
+	ID             string         `json:"id"`
+	Labels         []string       `json:"labels"`
+	Name           string         `json:"name"`
+	Properties     map[string]any `json:"properties"`
+	Tags           map[string]any `json:"tags"`
+	TimespanColumn string         `json:"timespanColumn"`
+	TableType      string         `json:"tableType"`
+	HasData        bool           `json:"hasData"`
+}
+
+type Columns struct {
+	Description      string         `json:"description"`
+	IsPreferredFacet bool           `json:"isPreferredFacet"`
+	Name             string         `json:"name"`
+	Source           map[string]any `json:"source"`
+	Type             string         `json:"type"`
+}
+
+type MetadataApplications struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Region     string `json:"region"`
+	ResourceId string `json:"resourceId"`
+}
+
+type MetadataWorkspaces struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Region     string `json:"region"`
+	ResourceId string `json:"resourceId"`
+}
+
+type MetadataCategories struct {
+	Description string `json:"description"`
+	DisplayName string `json:"displayName"`
+	ID          string `json:"id"`
+}
+
+type MetadataFunctions struct {
+	Body        string         `json:"string"`
+	Description string         `json:"description"`
+	DisplayName string         `json:"displayName"`
+	ID          string         `json:"id"`
+	Name        string         `json:"name"`
+	Parameters  string         `json:"parameters"`
+	Properties  map[string]any `json:"properties"`
+	Tags        map[string]any `json:"tags"`
+}
+
+type MetadataPermissions struct {
+	Applications []Applications `json:"applications"`
+	Resources    []Resources    `json:"resources"`
+	Workspaces   []Workspaces   `json:"workspaces"`
+}
+
+type MetadataQueries struct {
+	Body        string         `json:"string"`
+	Description string         `json:"description"`
+	DisplayName string         `json:"displayName"`
+	ID          string         `json:"id"`
+	Labels      []string       `json:"labels"`
+	Properties  map[string]any `json:"properties"`
+	Tags        map[string]any `json:"tags"`
+}
+
+type MetadataResourceTypes struct {
+	Description string         `json:"description"`
+	DisplayName string         `json:"displayName"`
+	ID          string         `json:"id"`
+	Labels      []string       `json:"labels"`
+	Properties  map[string]any `json:"properties"`
+	Tags        map[string]any `json:"tags"`
+	Type        string         `json:"type"`
+}
+
+type MetadataSolutions struct {
+	Description string         `json:"description"`
+	DisplayName string         `json:"displayName"`
+	ID          string         `json:"id"`
+	Name        string         `json:"name"`
+	Properties  map[string]any `json:"properties"`
+	Tags        map[string]any `json:"tags"`
+}
+
+type Applications struct {
+	ResourceId string `json:"resourceId"`
+}
+
+type Resources struct {
+	DenyTables []string `json:"denyTables"`
+	ResourceId string   `json:"resourceId"`
+}
+type Workspaces struct {
+	DenyTables []string `json:"denyTables"`
+	ResourceId string   `json:"resourceId"`
+}