ElasticSearch: Improve index pattern error messaging and docs (#103899)

docs/sources/datasources/elasticsearch/configure-elasticsearch-data-source.md

@@ -127,7 +127,7 @@ Additional settings are optional settings that can be configured for more contro
 
 The following settings are specific to the Elasticsearch data source.
 
-- **Index name** - Use the index settings to specify a default for the `time field` and your Elasticsearch index's name. You can use a time pattern, such as `YYYY.MM.DD`, or a wildcard for the index name.
+- **Index name** - Use the index settings to specify a default for the `time field` and your Elasticsearch index's name. You can use a time pattern, for example `[logstash-]YYYY.MM.DD`, or a wildcard for the index name. When specifying a time pattern, the fixed part(s) of the pattern should be wrapped in square brackets.
 
 - **Pattern** - Select the matching pattern if using one in your index name. Options include:
 
@@ -138,6 +138,8 @@ The following settings are specific to the Elasticsearch data source.
   - monthly
   - yearly
 
+Only select a pattern option if you have specified a time pattern in the Index name field.
+
 - **Time field name** - Name of the time field. The default value is @timestamp. You can enter a different name.
 
 - **Max concurrent shard requests** - Sets the number of shards being queried at the same time. The default is `5`. For more information on shards see [Elasticsearch's documentation](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/scalability.html#scalability).

pkg/tsdb/elasticsearch/client/client.go

@@ -162,7 +162,11 @@ func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body [
 
 func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) {
 	var err error
-	multiRequests := c.createMultiSearchRequests(r.Requests)
+	multiRequests, err := c.createMultiSearchRequests(r.Requests)
+	if err != nil {
+		return nil, err
+	}
+
 	queryParams := c.getMultiSearchQueryParameters()
 	_, span := tracing.DefaultTracer().Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch", trace.WithAttributes(
 		attribute.String("queryParams", queryParams),
@@ -429,14 +433,14 @@ func skipUnknownField(dec *json.Decoder) error {
 	}
 }
 
-func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) []*multiRequest {
+func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchRequest) ([]*multiRequest, error) {
 	multiRequests := []*multiRequest{}
 
 	for _, searchReq := range searchRequests {
 		indices, err := c.indexPattern.GetIndices(searchReq.TimeRange)
 		if err != nil {
-			c.logger.Error("Failed to get indices from index pattern", "error", err)
-			continue
+			err := fmt.Errorf("failed to get indices from index pattern. %s", err)
+			return nil, backend.DownstreamError(err)
 		}
 		mr := multiRequest{
 			header: map[string]any{
@@ -451,7 +455,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
 		multiRequests = append(multiRequests, &mr)
 	}
 
-	return multiRequests
+	return multiRequests, nil
 }
 
 func (c *baseClientImpl) getMultiSearchQueryParameters() string {

pkg/tsdb/elasticsearch/client/client_test.go

@@ -183,6 +183,21 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
 		require.Contains(t, bodyString, "metrics-2018.05.17")
 	})
 
+	t.Run("Should return DownstreamError when index is invalid", func(t *testing.T) {
+		ds := &DatasourceInfo{
+			URL:      "test",
+			Database: "index-with-no-pattern",
+			Interval: intervalMonthly,
+		}
+
+		c, err := NewClient(context.Background(), ds, log.NewNullLogger())
+		require.NoError(t, err)
+
+		_, err = c.ExecuteMultisearch(&MultiSearchRequest{Requests: []*SearchRequest{{}}})
+		assert.Equal(t, "failed to get indices from index pattern. invalid index pattern index-with-no-pattern. Specify an index with a time pattern or select 'No pattern'", err.Error())
+		require.True(t, backend.IsDownstreamError(err))
+	})
+
 	t.Run("Should return DownstreamError when decoding response fails", func(t *testing.T) {
 		ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 			rw.Header().Set("Content-Type", "application/x-ndjson")

pkg/tsdb/elasticsearch/client/index_pattern.go

@@ -136,7 +136,12 @@ func (ip *dynamicIndexPattern) GetIndices(timeRange backend.TimeRange) ([]string
 	indices := make([]string, 0)
 
 	for _, t := range intervals {
-		indices = append(indices, formatDate(t, ip.pattern))
+		index, err := formatDate(t, ip.pattern)
+		if err != nil {
+			return []string{}, err
+		} else {
+			indices = append(indices, index)
+		}
 	}
 
 	return indices, nil
@@ -251,13 +256,16 @@ func (i *yearlyInterval) Generate(from, to time.Time) []time.Time {
 	return intervals
 }
 
-func formatDate(t time.Time, pattern string) string {
+func formatDate(t time.Time, pattern string) (string, error) {
 	var formattedDatePatterns []string
 	var bases []string
 	base := ""
 	isBaseFirst := false
 
 	baseStart := strings.Index(pattern, "[")
+	if baseStart == -1 {
+		return "", fmt.Errorf("invalid index pattern %s. Specify an index with a time pattern or select 'No pattern'", pattern)
+	}
 	for baseStart != -1 {
 		var datePattern string
 
@@ -344,7 +352,7 @@ func formatDate(t time.Time, pattern string) string {
 		fullPattern = append(fullPattern, bases...)
 	}
 
-	return strings.Join(fullPattern, "")
+	return strings.Join(fullPattern, ""), nil
 }
 
 func patternToLayout(pattern string) string {

pkg/tsdb/elasticsearch/client/index_pattern_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -108,6 +109,21 @@ func TestIndexPattern(t *testing.T) {
 		})
 	})
 
+	t.Run("Dynamic index pattern with error", func(t *testing.T) {
+		from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
+		to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
+		timeRange := backend.TimeRange{
+			From: from,
+			To:   to,
+		}
+		ip, err := NewIndexPattern(intervalHourly, "kibana-sample-data-logs")
+		require.NoError(t, err)
+		indices, err := ip.GetIndices(timeRange)
+		assert.Equal(t, indices, []string{})
+		require.Error(t, err)
+		assert.Equal(t, err.Error(), "invalid index pattern kibana-sample-data-logs. Specify an index with a time pattern or select 'No pattern'")
+	})
+
 	t.Run("Hourly interval", func(t *testing.T) {
 		t.Run("Should return 1 interval", func(t *testing.T) {
 			from := time.Date(2018, 1, 1, 23, 1, 1, 0, time.UTC)