diff --git a/docs/sources/datasources/elasticsearch/configure-elasticsearch-data-source.md b/docs/sources/datasources/elasticsearch/configure-elasticsearch-data-source.md index eacc91861df..cd34e8501d0 100644 --- a/docs/sources/datasources/elasticsearch/configure-elasticsearch-data-source.md +++ b/docs/sources/datasources/elasticsearch/configure-elasticsearch-data-source.md @@ -127,7 +127,7 @@ Additional settings are optional settings that can be configured for more contro The following settings are specific to the Elasticsearch data source. -- **Index name** - Use the index settings to specify a default for the `time field` and your Elasticsearch index's name. You can use a time pattern, such as `YYYY.MM.DD`, or a wildcard for the index name. +- **Index name** - Use the index settings to specify a default for the `time field` and your Elasticsearch index's name. You can use a time pattern, for example `[logstash-]YYYY.MM.DD`, or a wildcard for the index name. When specifying a time pattern, the fixed part(s) of the pattern should be wrapped in square brackets. - **Pattern** - Select the matching pattern if using one in your index name. Options include: @@ -138,6 +138,8 @@ The following settings are specific to the Elasticsearch data source. - monthly - yearly +Only select a pattern option if you have specified a time pattern in the Index name field. + - **Time field name** - Name of the time field. The default value is @timestamp. You can enter a different name. - **Max concurrent shard requests** - Sets the number of shards being queried at the same time. The default is `5`. For more information on shards see [Elasticsearch's documentation](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/scalability.html#scalability). diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 868e566ddf8..746e670dc78 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -162,7 +162,11 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) { var err error - multiRequests := c.createMultiSearchRequests(r.Requests) + multiRequests, err := c.createMultiSearchRequests(r.Requests) + if err != nil { + return nil, err + } + queryParams := c.getMultiSearchQueryParameters() _, span := tracing.DefaultTracer().Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch", trace.WithAttributes( attribute.String("queryParams", queryParams), @@ -429,14 +433,14 @@ func skipUnknownField(dec *json.Decoder) error { } } -func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest { +func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) ([]*multiRequest, error) { multiRequests := []*multiRequest{} for _, searchReq := range searchRequests { indices, err := c.indexPattern.GetIndices(searchReq.TimeRange) if err != nil { - c.logger.Error("Failed to get indices from index pattern", "error", err) - continue + err := fmt.Errorf("failed to get indices from index pattern. %s", err) + return nil, backend.DownstreamError(err) } mr := multiRequest{ header: map[string]any{ @@ -451,7 +455,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque multiRequests = append(multiRequests, &mr) } - return multiRequests + return multiRequests, nil } func (c *baseClientImpl) getMultiSearchQueryParameters() string { diff --git a/pkg/tsdb/elasticsearch/client/client_test.go b/pkg/tsdb/elasticsearch/client/client_test.go index dff8b39139d..8f257873232 100644 --- a/pkg/tsdb/elasticsearch/client/client_test.go +++ b/pkg/tsdb/elasticsearch/client/client_test.go @@ -183,6 +183,21 @@ func TestClient_ExecuteMultisearch(t *testing.T) { require.Contains(t, bodyString, "metrics-2018.05.17") }) + t.Run("Should return DownstreamError when index is invalid", func(t *testing.T) { + ds := &DatasourceInfo{ + URL: "test", + Database: "index-with-no-pattern", + Interval: intervalMonthly, + } + + c, err := NewClient(context.Background(), ds, log.NewNullLogger()) + require.NoError(t, err) + + _, err = c.ExecuteMultisearch(&MultiSearchRequest{Requests: []*SearchRequest{{}}}) + assert.Equal(t, "failed to get indices from index pattern. invalid index pattern index-with-no-pattern. Specify an index with a time pattern or select 'No pattern'", err.Error()) + require.True(t, backend.IsDownstreamError(err)) + }) + t.Run("Should return DownstreamError when decoding response fails", func(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.Header().Set("Content-Type", "application/x-ndjson") diff --git a/pkg/tsdb/elasticsearch/client/index_pattern.go b/pkg/tsdb/elasticsearch/client/index_pattern.go index ff01d759c5c..06da9004678 100644 --- a/pkg/tsdb/elasticsearch/client/index_pattern.go +++ b/pkg/tsdb/elasticsearch/client/index_pattern.go @@ -136,7 +136,12 @@ func (ip *dynamicIndexPattern) GetIndices(timeRange backend.TimeRange) ([]string indices := make([]string, 0) for _, t := range intervals { - indices = append(indices, formatDate(t, ip.pattern)) + index, err := formatDate(t, ip.pattern) + if err != nil { + return []string{}, err + } else { + indices = append(indices, index) + } } return indices, nil @@ -251,13 +256,16 @@ func (i *yearlyInterval) Generate(from, to time.Time) []time.Time { return intervals } -func formatDate(t time.Time, pattern string) string { +func formatDate(t time.Time, pattern string) (string, error) { var formattedDatePatterns []string var bases []string base := "" isBaseFirst := false baseStart := strings.Index(pattern, "[") + if baseStart == -1 { + return "", fmt.Errorf("invalid index pattern %s. Specify an index with a time pattern or select 'No pattern'", pattern) + } for baseStart != -1 { var datePattern string @@ -344,7 +352,7 @@ func formatDate(t time.Time, pattern string) string { fullPattern = append(fullPattern, bases...) } - return strings.Join(fullPattern, "") + return strings.Join(fullPattern, ""), nil } func patternToLayout(pattern string) string { diff --git a/pkg/tsdb/elasticsearch/client/index_pattern_test.go b/pkg/tsdb/elasticsearch/client/index_pattern_test.go index a30408cf4a8..7468fc4580e 100644 --- a/pkg/tsdb/elasticsearch/client/index_pattern_test.go +++ b/pkg/tsdb/elasticsearch/client/index_pattern_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -108,6 +109,21 @@ func TestIndexPattern(t *testing.T) { }) }) + t.Run("Dynamic index pattern with error", func(t *testing.T) { + from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) + to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) + timeRange := backend.TimeRange{ + From: from, + To: to, + } + ip, err := NewIndexPattern(intervalHourly, "kibana-sample-data-logs") + require.NoError(t, err) + indices, err := ip.GetIndices(timeRange) + assert.Equal(t, indices, []string{}) + require.Error(t, err) + assert.Equal(t, err.Error(), "invalid index pattern kibana-sample-data-logs. Specify an index with a time pattern or select 'No pattern'") + }) + t.Run("Hourly interval", func(t *testing.T) { t.Run("Should return 1 interval", func(t *testing.T) { from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC)