Azure Monitor: Add logs query builder (#99055)

This commit is contained in:
Alyssa (Bull) Joyner
2025-04-02 08:15:25 -06:00
committed by GitHub
parent 44ca402116
commit 3b73ebb210
42 changed files with 3306 additions and 59 deletions
+7
View File
@@ -1818,6 +1818,13 @@ var (
HideFromAdminPage: true,
HideFromDocs: true,
},
{
Name: "azureMonitorLogsBuilderEditor",
Description: "Enables the logs builder mode for the Azure Monitor data source",
Stage: FeatureStagePublicPreview,
Owner: grafanaPartnerPluginsSquad,
Expression: "false",
},
{
Name: "localeFormatPreference",
Description: "Specify the locale so we can show the correct format for numbers and dates",
+1
View File
@@ -239,6 +239,7 @@ inviteUserExperimental,experimental,@grafana/sharing-squad,false,false,true
noBackdropBlur,experimental,@grafana/grafana-frontend-platform,false,false,true
alertingMigrationUI,experimental,@grafana/alerting-squad,false,false,true
unifiedStorageHistoryPruner,experimental,@grafana/search-and-storage,false,false,false
azureMonitorLogsBuilderEditor,preview,@grafana/partner-datasources,false,false,false
localeFormatPreference,experimental,@grafana/grafana-frontend-platform,false,false,false
unifiedStorageGrpcConnectionPool,experimental,@grafana/search-and-storage,false,false,false
alertingRuleRecoverDeleted,GA,@grafana/alerting-squad,false,false,true
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
239 noBackdropBlur experimental @grafana/grafana-frontend-platform false false true
240 alertingMigrationUI experimental @grafana/alerting-squad false false true
241 unifiedStorageHistoryPruner experimental @grafana/search-and-storage false false false
242 azureMonitorLogsBuilderEditor preview @grafana/partner-datasources false false false
243 localeFormatPreference experimental @grafana/grafana-frontend-platform false false false
244 unifiedStorageGrpcConnectionPool experimental @grafana/search-and-storage false false false
245 alertingRuleRecoverDeleted GA @grafana/alerting-squad false false true
+4
View File
@@ -967,6 +967,10 @@ const (
// Enables the unified storage history pruner
FlagUnifiedStorageHistoryPruner = "unifiedStorageHistoryPruner"
// FlagAzureMonitorLogsBuilderEditor
// Enables the logs builder mode for the Azure Monitor data source
FlagAzureMonitorLogsBuilderEditor = "azureMonitorLogsBuilderEditor"
// FlagLocaleFormatPreference
// Specify the locale so we can show the correct format for numbers and dates
FlagLocaleFormatPreference = "localeFormatPreference"
+16
View File
@@ -901,6 +901,22 @@
"codeowner": "@grafana/partner-datasources"
}
},
{
"metadata": {
"name": "azureMonitorLogsBuilderEditor",
"resourceVersion": "1743001017970",
"creationTimestamp": "2025-03-19T22:51:49Z",
"annotations": {
"grafana.app/updatedTimestamp": "2025-03-26 14:56:57.970732 +0000 UTC"
}
},
"spec": {
"description": "Enables the logs builder mode for the Azure Monitor data source",
"stage": "preview",
"codeowner": "@grafana/partner-datasources",
"expression": "false"
}
},
{
"metadata": {
"name": "azureMonitorPrometheusExemplars",
@@ -157,6 +157,10 @@ type AzureLogsQuery struct {
BasicLogsQuery *bool `json:"basicLogsQuery,omitempty"`
// Workspace ID. This was removed in Grafana 8, but remains for backwards compat.
Workspace *string `json:"workspace,omitempty"`
// Denotes if logs query editor is in builder mode
Mode *LogsEditorMode `json:"mode,omitempty"`
// Builder query to be executed.
BuilderQuery *BuilderQueryExpression `json:"builderQuery,omitempty"`
// @deprecated Use resources instead
Resource *string `json:"resource,omitempty"`
// @deprecated Use dashboardTime instead
@@ -177,6 +181,217 @@ const (
ResultFormatLogs ResultFormat = "logs"
)
type LogsEditorMode string
const (
LogsEditorModeBuilder LogsEditorMode = "builder"
LogsEditorModeRaw LogsEditorMode = "raw"
)
type BuilderQueryExpression struct {
From *BuilderQueryEditorPropertyExpression `json:"from,omitempty"`
Columns *BuilderQueryEditorColumnsExpression `json:"columns,omitempty"`
Where *BuilderQueryEditorWhereExpressionArray `json:"where,omitempty"`
Reduce *BuilderQueryEditorReduceExpressionArray `json:"reduce,omitempty"`
GroupBy *BuilderQueryEditorGroupByExpressionArray `json:"groupBy,omitempty"`
Limit *int64 `json:"limit,omitempty"`
OrderBy *BuilderQueryEditorOrderByExpressionArray `json:"orderBy,omitempty"`
FuzzySearch *BuilderQueryEditorWhereExpressionArray `json:"fuzzySearch,omitempty"`
TimeFilter *BuilderQueryEditorWhereExpressionArray `json:"timeFilter,omitempty"`
}
// NewBuilderQueryExpression creates a new BuilderQueryExpression object.
func NewBuilderQueryExpression() *BuilderQueryExpression {
return &BuilderQueryExpression{}
}
type BuilderQueryEditorPropertyExpression struct {
Property BuilderQueryEditorProperty `json:"property"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorPropertyExpression creates a new BuilderQueryEditorPropertyExpression object.
func NewBuilderQueryEditorPropertyExpression() *BuilderQueryEditorPropertyExpression {
return &BuilderQueryEditorPropertyExpression{
Property: *NewBuilderQueryEditorProperty(),
}
}
type BuilderQueryEditorProperty struct {
Type BuilderQueryEditorPropertyType `json:"type"`
Name string `json:"name"`
}
// NewBuilderQueryEditorProperty creates a new BuilderQueryEditorProperty object.
func NewBuilderQueryEditorProperty() *BuilderQueryEditorProperty {
return &BuilderQueryEditorProperty{}
}
type BuilderQueryEditorPropertyType string
const (
BuilderQueryEditorPropertyTypeNumber BuilderQueryEditorPropertyType = "number"
BuilderQueryEditorPropertyTypeString BuilderQueryEditorPropertyType = "string"
BuilderQueryEditorPropertyTypeBoolean BuilderQueryEditorPropertyType = "boolean"
BuilderQueryEditorPropertyTypeDatetime BuilderQueryEditorPropertyType = "datetime"
BuilderQueryEditorPropertyTypeTimeSpan BuilderQueryEditorPropertyType = "time_span"
BuilderQueryEditorPropertyTypeFunction BuilderQueryEditorPropertyType = "function"
BuilderQueryEditorPropertyTypeInterval BuilderQueryEditorPropertyType = "interval"
)
type BuilderQueryEditorExpressionType string
const (
BuilderQueryEditorExpressionTypeProperty BuilderQueryEditorExpressionType = "property"
BuilderQueryEditorExpressionTypeOperator BuilderQueryEditorExpressionType = "operator"
BuilderQueryEditorExpressionTypeReduce BuilderQueryEditorExpressionType = "reduce"
BuilderQueryEditorExpressionTypeFunctionParameter BuilderQueryEditorExpressionType = "function_parameter"
BuilderQueryEditorExpressionTypeGroupBy BuilderQueryEditorExpressionType = "group_by"
BuilderQueryEditorExpressionTypeOr BuilderQueryEditorExpressionType = "or"
BuilderQueryEditorExpressionTypeAnd BuilderQueryEditorExpressionType = "and"
BuilderQueryEditorExpressionTypeOrderBy BuilderQueryEditorExpressionType = "order_by"
)
type BuilderQueryEditorColumnsExpression struct {
Columns []string `json:"columns,omitempty"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorColumnsExpression creates a new BuilderQueryEditorColumnsExpression object.
func NewBuilderQueryEditorColumnsExpression() *BuilderQueryEditorColumnsExpression {
return &BuilderQueryEditorColumnsExpression{}
}
type BuilderQueryEditorWhereExpressionArray struct {
Expressions []BuilderQueryEditorWhereExpression `json:"expressions"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorWhereExpressionArray creates a new BuilderQueryEditorWhereExpressionArray object.
func NewBuilderQueryEditorWhereExpressionArray() *BuilderQueryEditorWhereExpressionArray {
return &BuilderQueryEditorWhereExpressionArray{}
}
type BuilderQueryEditorWhereExpression struct {
Type BuilderQueryEditorExpressionType `json:"type"`
Expressions []BuilderQueryEditorWhereExpressionItems `json:"expressions"`
}
// NewBuilderQueryEditorWhereExpression creates a new BuilderQueryEditorWhereExpression object.
func NewBuilderQueryEditorWhereExpression() *BuilderQueryEditorWhereExpression {
return &BuilderQueryEditorWhereExpression{}
}
type BuilderQueryEditorWhereExpressionItems struct {
Property BuilderQueryEditorProperty `json:"property"`
Operator BuilderQueryEditorOperator `json:"operator"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorWhereExpressionItems creates a new BuilderQueryEditorWhereExpressionItems object.
func NewBuilderQueryEditorWhereExpressionItems() *BuilderQueryEditorWhereExpressionItems {
return &BuilderQueryEditorWhereExpressionItems{
Property: *NewBuilderQueryEditorProperty(),
Operator: *NewBuilderQueryEditorOperator(),
}
}
type BuilderQueryEditorOperator struct {
Name string `json:"name"`
Value string `json:"value"`
LabelValue *string `json:"labelValue,omitempty"`
}
// NewBuilderQueryEditorOperator creates a new BuilderQueryEditorOperator object.
func NewBuilderQueryEditorOperator() *BuilderQueryEditorOperator {
return &BuilderQueryEditorOperator{}
}
type BuilderQueryEditorReduceExpressionArray struct {
Expressions []BuilderQueryEditorReduceExpression `json:"expressions"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorReduceExpressionArray creates a new BuilderQueryEditorReduceExpressionArray object.
func NewBuilderQueryEditorReduceExpressionArray() *BuilderQueryEditorReduceExpressionArray {
return &BuilderQueryEditorReduceExpressionArray{}
}
type BuilderQueryEditorReduceExpression struct {
Property *BuilderQueryEditorProperty `json:"property,omitempty"`
Reduce *BuilderQueryEditorProperty `json:"reduce,omitempty"`
Parameters []BuilderQueryEditorFunctionParameterExpression `json:"parameters,omitempty"`
Focus *bool `json:"focus,omitempty"`
}
// NewBuilderQueryEditorReduceExpression creates a new BuilderQueryEditorReduceExpression object.
func NewBuilderQueryEditorReduceExpression() *BuilderQueryEditorReduceExpression {
return &BuilderQueryEditorReduceExpression{}
}
type BuilderQueryEditorFunctionParameterExpression struct {
Value string `json:"value"`
FieldType BuilderQueryEditorPropertyType `json:"fieldType"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorFunctionParameterExpression creates a new BuilderQueryEditorFunctionParameterExpression object.
func NewBuilderQueryEditorFunctionParameterExpression() *BuilderQueryEditorFunctionParameterExpression {
return &BuilderQueryEditorFunctionParameterExpression{}
}
type BuilderQueryEditorGroupByExpressionArray struct {
Expressions []BuilderQueryEditorGroupByExpression `json:"expressions"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorGroupByExpressionArray creates a new BuilderQueryEditorGroupByExpressionArray object.
func NewBuilderQueryEditorGroupByExpressionArray() *BuilderQueryEditorGroupByExpressionArray {
return &BuilderQueryEditorGroupByExpressionArray{}
}
type BuilderQueryEditorGroupByExpression struct {
Property *BuilderQueryEditorProperty `json:"property,omitempty"`
Interval *BuilderQueryEditorProperty `json:"interval,omitempty"`
Focus *bool `json:"focus,omitempty"`
Type *BuilderQueryEditorExpressionType `json:"type,omitempty"`
}
// NewBuilderQueryEditorGroupByExpression creates a new BuilderQueryEditorGroupByExpression object.
func NewBuilderQueryEditorGroupByExpression() *BuilderQueryEditorGroupByExpression {
return &BuilderQueryEditorGroupByExpression{}
}
type BuilderQueryEditorOrderByExpressionArray struct {
Expressions []BuilderQueryEditorOrderByExpression `json:"expressions"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorOrderByExpressionArray creates a new BuilderQueryEditorOrderByExpressionArray object.
func NewBuilderQueryEditorOrderByExpressionArray() *BuilderQueryEditorOrderByExpressionArray {
return &BuilderQueryEditorOrderByExpressionArray{}
}
type BuilderQueryEditorOrderByExpression struct {
Property BuilderQueryEditorProperty `json:"property"`
Order BuilderQueryEditorOrderByOptions `json:"order"`
Type BuilderQueryEditorExpressionType `json:"type"`
}
// NewBuilderQueryEditorOrderByExpression creates a new BuilderQueryEditorOrderByExpression object.
func NewBuilderQueryEditorOrderByExpression() *BuilderQueryEditorOrderByExpression {
return &BuilderQueryEditorOrderByExpression{
Property: *NewBuilderQueryEditorProperty(),
}
}
type BuilderQueryEditorOrderByOptions string
const (
BuilderQueryEditorOrderByOptionsAsc BuilderQueryEditorOrderByOptions = "asc"
BuilderQueryEditorOrderByOptionsDesc BuilderQueryEditorOrderByOptions = "desc"
)
type AzureResourceGraphQuery struct {
// Azure Resource Graph KQL query to be executed.
Query *string `json:"query,omitempty"`
@@ -391,6 +606,23 @@ const (
AzureQueryTypeCustomMetricNamesQuery AzureQueryType = "Azure Custom Metric Names"
)
type SelectableValue struct {
Label string `json:"label"`
Value string `json:"value"`
}
// NewSelectableValue creates a new SelectableValue object.
func NewSelectableValue() *SelectableValue {
return &SelectableValue{}
}
type BuilderQueryEditorOperatorType = StringOrBoolOrFloat64OrSelectableValue
// NewBuilderQueryEditorOperatorType creates a new BuilderQueryEditorOperatorType object.
func NewBuilderQueryEditorOperatorType() *BuilderQueryEditorOperatorType {
return NewStringOrBoolOrFloat64OrSelectableValue()
}
type GrafanaTemplateVariableQueryType string
const (
@@ -569,3 +801,15 @@ func (resource *AppInsightsMetricNameQueryOrAppInsightsGroupByQueryOrSubscriptio
return fmt.Errorf("could not unmarshal resource with `kind = %v`", discriminator)
}
type StringOrBoolOrFloat64OrSelectableValue struct {
String *string `json:"String,omitempty"`
Bool *bool `json:"Bool,omitempty"`
Float64 *float64 `json:"Float64,omitempty"`
SelectableValue *SelectableValue `json:"SelectableValue,omitempty"`
}
// NewStringOrBoolOrFloat64OrSelectableValue creates a new StringOrBoolOrFloat64OrSelectableValue object.
func NewStringOrBoolOrFloat64OrSelectableValue() *StringOrBoolOrFloat64OrSelectableValue {
return &StringOrBoolOrFloat64OrSelectableValue{}
}
@@ -28,6 +28,18 @@ import (
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/utils"
)
// Returns tables with the `HasData` field set to true
func filterTablesWithData(tables []types.MetadataTable) []types.MetadataTable {
filtered := []types.MetadataTable{}
for _, table := range tables {
if table.HasData {
filtered = append(filtered, table)
}
}
return filtered
}
func (e *AzureLogAnalyticsDatasource) ResourceRequest(rw http.ResponseWriter, req *http.Request, cli *http.Client) (http.ResponseWriter, error) {
if req.URL.Path == "/usage/basiclogs" {
newUrl := &url.URL{
@@ -36,7 +48,57 @@ func (e *AzureLogAnalyticsDatasource) ResourceRequest(rw http.ResponseWriter, re
Path: "/v1/query",
}
return e.GetBasicLogsUsage(req.Context(), newUrl.String(), cli, rw, req.Body)
} else if strings.Contains(req.URL.Path, "/metadata") {
// Add necessary headers
req.Header.Set("Prefer", "metadata-format-v4,exclude-resourcetypes,exclude-customfunctions")
queryParams := req.URL.Query()
// Add necessary query params
queryParams.Add("select", "categories,solutions,tables,workspaces")
req.URL.RawQuery = queryParams.Encode()
resp, err := cli.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch metadata: %w", err)
}
defer func() {
if err := resp.Body.Close(); err != nil {
e.Logger.Warn("Failed to close response body for metadata request", "err", err)
}
}()
encoding := resp.Header.Get("Content-Encoding")
body, err := decode(encoding, resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read metadata response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("metadata API error: %s", string(body))
}
var metadata types.AzureLogAnalyticsMetadata
err = json.Unmarshal(body, &metadata)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal metadata response: %w", err)
}
metadata.Tables = filterTablesWithData(metadata.Tables)
responseBody, err := json.Marshal(metadata)
if err != nil {
return nil, fmt.Errorf("failed to marshal metadata response: %w", err)
}
rw.Header().Set("Content-Type", "application/json")
rw.WriteHeader(http.StatusOK)
_, err = rw.Write(responseBody)
if err != nil {
return nil, fmt.Errorf("failed to write metadata response: %w", err)
}
return rw, nil
}
// Default behavior for other requests
return e.Proxy.Do(rw, req, cli)
}
@@ -1,12 +1,16 @@
package loganalytics
import (
"compress/flate"
"compress/gzip"
"fmt"
"io"
"regexp"
"strconv"
"strings"
"time"
"github.com/andybalholm/brotli"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
@@ -136,3 +140,40 @@ func ConvertTime(timeStamp string) (time.Time, error) {
func GetDataVolumeRawQuery(table string) string {
return fmt.Sprintf("Usage \n| where DataType == \"%s\"\n| where IsBillable == true\n| summarize BillableDataGB = round(sum(Quantity) / 1000, 3)", table)
}
// This function handles various compression mechanisms that may have been used on a response body
func decode(encoding string, original io.ReadCloser) ([]byte, error) {
var reader io.Reader
var err error
switch encoding {
case "gzip":
reader, err = gzip.NewReader(original)
if err != nil {
return nil, err
}
defer func() {
if err := reader.(io.ReadCloser).Close(); err != nil {
backend.Logger.Warn("Failed to close reader body", "err", err)
}
}()
case "deflate":
reader = flate.NewReader(original)
defer func() {
if err := reader.(io.ReadCloser).Close(); err != nil {
backend.Logger.Warn("Failed to close reader body", "err", err)
}
}()
case "br":
reader = brotli.NewReader(original)
case "":
reader = original
default:
return nil, fmt.Errorf("unexpected encoding type %v", err)
}
body, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
return body, nil
}
+116
View File
@@ -210,3 +210,119 @@ type SubscriptionsResponse struct {
}
var ErrorAzureHealthCheck = errors.New("health check failed")
// AzureLogAnalyticsMetadata represents the metadata response from the Azure Log Analytics API.
// Types are taken from https://learn.microsoft.com/en-us/rest/api/loganalytics/metadata/get
type AzureLogAnalyticsMetadata struct {
Applications []MetadataApplications `json:"applications"`
Categories []MetadataCategories `json:"categories"`
Functions []MetadataFunctions `json:"functions"`
Permissions []MetadataPermissions `json:"permissions"`
Queries []MetadataQueries `json:"queries"`
ResourceTypes []MetadataResourceTypes `json:"resourceTypes"`
Resources []Resources `json:"resources"`
Solutions []MetadataSolutions `json:"solutions"`
Tables []MetadataTable `json:"tables"`
Workspaces []MetadataWorkspaces `json:"workspaces"`
}
// MetadataTable represents a table entry in the metadata response.
type MetadataTable struct {
Columns []Columns `json:"columns"`
Description string `json:"description"`
ID string `json:"id"`
Labels []string `json:"labels"`
Name string `json:"name"`
Properties map[string]any `json:"properties"`
Tags map[string]any `json:"tags"`
TimespanColumn string `json:"timespanColumn"`
TableType string `json:"tableType"`
HasData bool `json:"hasData"`
}
type Columns struct {
Description string `json:"description"`
IsPreferredFacet bool `json:"isPreferredFacet"`
Name string `json:"name"`
Source map[string]any `json:"source"`
Type string `json:"type"`
}
type MetadataApplications struct {
ID string `json:"id"`
Name string `json:"name"`
Region string `json:"region"`
ResourceId string `json:"resourceId"`
}
type MetadataWorkspaces struct {
ID string `json:"id"`
Name string `json:"name"`
Region string `json:"region"`
ResourceId string `json:"resourceId"`
}
type MetadataCategories struct {
Description string `json:"description"`
DisplayName string `json:"displayName"`
ID string `json:"id"`
}
type MetadataFunctions struct {
Body string `json:"string"`
Description string `json:"description"`
DisplayName string `json:"displayName"`
ID string `json:"id"`
Name string `json:"name"`
Parameters string `json:"parameters"`
Properties map[string]any `json:"properties"`
Tags map[string]any `json:"tags"`
}
type MetadataPermissions struct {
Applications []Applications `json:"applications"`
Resources []Resources `json:"resources"`
Workspaces []Workspaces `json:"workspaces"`
}
type MetadataQueries struct {
Body string `json:"string"`
Description string `json:"description"`
DisplayName string `json:"displayName"`
ID string `json:"id"`
Labels []string `json:"labels"`
Properties map[string]any `json:"properties"`
Tags map[string]any `json:"tags"`
}
type MetadataResourceTypes struct {
Description string `json:"description"`
DisplayName string `json:"displayName"`
ID string `json:"id"`
Labels []string `json:"labels"`
Properties map[string]any `json:"properties"`
Tags map[string]any `json:"tags"`
Type string `json:"type"`
}
type MetadataSolutions struct {
Description string `json:"description"`
DisplayName string `json:"displayName"`
ID string `json:"id"`
Name string `json:"name"`
Properties map[string]any `json:"properties"`
Tags map[string]any `json:"tags"`
}
type Applications struct {
ResourceId string `json:"resourceId"`
}
type Resources struct {
DenyTables []string `json:"denyTables"`
ResourceId string `json:"resourceId"`
}
type Workspaces struct {
DenyTables []string `json:"denyTables"`
ResourceId string `json:"resourceId"`
}