public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

elasticsearch: add 7.x version support (#16646)

Browse Source 
Adds a new version option 7.0+ (70 internally).
Version 7.0+ doesn't include document types in index mappings 
so support for handling this have been added.
Version 7.0+ returns number of hits in a different way so 
support for handling this have been added.
Version 7.0+ doesn't support sending 
max_concurrent_shard_requests in multisearch header so 
support for sending this in query string have been added.
Update elastic6 docker block and dashboards (devenv) to use 
6.7.1 images, filebeat index name is now filebeat-YYYY.MM.DD 
and dashboard include correct tags and links.
Add elastic7 docker block and provisioning (devenv).
Updates documentation regarding new version.

Closes #15622
This commit is contained in:
Alcides Viamontes E
2019-04-25 09:41:13 +02:00
committed by Marcus Efraimsson
parent 42b745a098
commit cff2be0d66
22 changed files with 5183 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
devenv/datasources.yaml
+32 -1
View File
@@ -99,13 +99,44 @@ datasources:
- name: gdev-elasticsearch-v6-filebeat
type: elasticsearch
access: proxy
database: "[filebeat-6.2.4-]YYYY.MM.DD"
database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:11200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 60
- name: gdev-elasticsearch-v7-metrics
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
timeInterval: 10s
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-logs
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-filebeat
type: elasticsearch
access: proxy
database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-mysql
type: mysql
url: localhost:3306
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_compare.json
+10 -4
View File
@@ -17,11 +17,13 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"iteration": 1545263815779,
"iteration": 1555595032099,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
@@ -5841,9 +5843,9 @@
}
],
"refresh": false,
"schemaVersion": 16,
"schemaVersion": 18,
"style": "dark",
"tags": ["gdev", "elasticsearch"],
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
@@ -5852,7 +5854,9 @@
"value": "gdev-elasticsearch-v5-metrics"
},
"hide": 0,
"includeAll": false,
"label": "Version One",
"multi": false,
"name": "version_one",
"options": [],
"query": "elasticsearch",
@@ -5867,7 +5871,9 @@
"value": "gdev-elasticsearch-v6-metrics"
},
"hide": 0,
"includeAll": false,
"label": "Version Two",
"multi": false,
"name": "version_two",
"options": [],
"query": "elasticsearch",
@@ -5889,5 +5895,5 @@
"timezone": "",
"title": "Datasource tests - Elasticsearch comparison",
"uid": "fuFWehBmk",
"version": 4
"version": 2
}
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v2.json
+3 -1
View File
@@ -31,8 +31,10 @@
"iteration": 1554310942895,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v5.json
+3 -3
View File
@@ -31,10 +31,10 @@
"iteration": 1554310560048,
"links": [
{
"asDropdown": false,
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"title": "Dashboard",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6.json
+3 -1
View File
@@ -31,8 +31,10 @@
"iteration": 1554310839317,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6_filebeat.json
+9 -1
View File
@@ -16,7 +16,15 @@
"gnetId": null,
"graphTooltip": 0,
"iteration": 1554902936982,
"links": [],
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7.json
+683
View File
@@ -0,0 +1,683 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
},
{
"datasource": "gdev-elasticsearch-v7-logs",
"enable": false,
"iconColor": "rgba(255, 96, 96, 1)",
"limit": 100,
"name": "test",
"query": "",
"showIn": 0,
"textField": "description",
"type": "alert"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2342,
"iteration": 1555593977614,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "1",
"size": "5"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "max"
}
],
"query": "*",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Top 5 servers",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 7
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {
"percents": [25, 50, 75, 95, 99]
},
"type": "percentiles"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Percentiles & Metric filter",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {
"std_deviation_bounds_lower": true,
"std_deviation_bounds_upper": true
},
"settings": {},
"type": "extended_stats"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Standard dev",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@hostname",
"value": "@hostname"
},
{
"text": "Average",
"value": "Average"
},
{
"text": "Max",
"value": "Max"
},
{
"text": "Sum",
"value": "Sum"
}
],
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 13
},
"id": 6,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "_term",
"size": "0"
},
"type": "terms"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
},
{
"field": "@value",
"id": "3",
"meta": {},
"settings": {},
"type": "max"
},
{
"field": "@value",
"id": "4",
"meta": {},
"settings": {},
"type": "sum"
}
],
"refId": "B",
"timeField": "@timestamp"
}
],
"title": "ES Metrics",
"transform": "table",
"type": "table"
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "@message",
"value": "@message"
},
{
"text": "tags",
"value": "tags"
},
{
"text": "description",
"value": "description"
}
],
"datasource": "gdev-elasticsearch-v7-logs",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 20
},
"id": 5,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
}
],
"targets": [
{
"bucketAggs": [],
"dsType": "elasticsearch",
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"title": "ES Log query",
"transform": "json",
"type": "table"
},
{
"circleMaxSize": 30,
"circleMinSize": 2,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"datasource": "gdev-elasticsearch-v7-metrics",
"decimals": 0,
"esGeoPoint": "@location",
"esMetric": "Average",
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"hideEmpty": false,
"hideZero": false,
"id": 8,
"initialZoom": "1",
"links": [],
"locationData": "geohash",
"mapCenter": "(0°, 0°)",
"mapCenterLatitude": 0,
"mapCenterLongitude": 0,
"maxDataPoints": 1,
"mouseWheelZoom": false,
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "@location",
"id": "3",
"settings": {
"precision": 2
},
"type": "geohash_grid"
}
],
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": "0,10",
"title": "World map panel",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"valueName": "total"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-metrics",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"status": "Stable",
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"],
"type": "timepicker"
},
"timezone": "browser",
"title": "Datasource tests - Elasticsearch v7",
"uid": "Y-RvmuRWk",
"version": 1
}
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7_filebeat.json
+264
View File
@@ -0,0 +1,264 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2341,
"iteration": 1555591591930,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {
"error": "red"
},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-filebeat",
"fill": 1,
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "fields.level",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_term",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 1,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Panel Title",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "fields.level",
"value": "fields.level"
},
{
"text": "message",
"value": "message"
}
],
"datasource": "gdev-elasticsearch-v7-filebeat",
"fontSize": "100%",
"gridPos": {
"h": 22,
"w": 24,
"x": 0,
"y": 5
},
"id": 2,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"alias": "Level",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "fields.level",
"thresholds": [""],
"type": "string",
"unit": "short",
"valueMaps": []
},
{
"alias": "Message",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"decimals": 2,
"pattern": "message",
"preserveFormat": false,
"sanitize": false,
"thresholds": [],
"type": "string",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [],
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel Title",
"transform": "json",
"type": "table"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-filebeat",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"]
},
"timezone": "",
"title": "Datasource tests - Elasticsearch v7 Filebeat",
"uid": "M94gguRWz",
"version": 1
}
devenv/docker/blocks/elastic6/docker-compose.yaml
+2 -2
View File
@@ -1,7 +1,7 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch6:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.7.1
command: elasticsearch
ports:
- "11200:9200"
@@ -15,7 +15,7 @@
FD_PORT: 11200
filebeat6:
image: docker.elastic.co/beats/filebeat:6.2.4
image: docker.elastic.co/beats/filebeat-oss:6.7.1
command: filebeat -e -strict.perms=false
volumes:
- ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
devenv/docker/blocks/elastic6/filebeat.yml
+1867 -72
View File
File diff suppressed because it is too large Load Diff
devenv/docker/blocks/elastic7/docker-compose.yaml
+23
View File
@@ -0,0 +1,23 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch7:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0
command: elasticsearch -E "discovery.type=single-node"
ports:
- "12200:9200"
- "12300:9300"
fake-elastic7-data:
image: grafana/fake-data-gen
network_mode: bridge
environment:
FD_DATASOURCE: elasticsearch7
FD_PORT: 12200
filebeat7:
image: docker.elastic.co/beats/filebeat-oss:7.0.0
command: filebeat -e -strict.perms=false
volumes:
- ./docker/blocks/elastic7/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/log:/var/log:ro
- ../data/log:/var/log/grafana:ro
devenv/docker/blocks/elastic7/elasticsearch.yml
+2
View File
@@ -0,0 +1,2 @@
script.inline: on
script.indexed: on
devenv/docker/blocks/elastic7/filebeat.yml
+1978
View File
File diff suppressed because it is too large Load Diff
docs/sources/administration/provisioning.md
+1 -1
View File
@@ -146,7 +146,7 @@ Since not all datasources have the same configuration settings we only have the
| tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. |
| graphiteVersion | string | Graphite | Graphite version |
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source |
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60) |
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60/70) |
| timeField | string | Elasticsearch | Which field that should be used as timestamp |
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
| authType | string | Cloudwatch | Auth provider. keys/credentials/arn |
docs/sources/features/datasources/elasticsearch.md
+4 -2
View File
@@ -58,10 +58,12 @@ a time pattern for the index name or a wildcard.
### Elasticsearch version
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed.
Currently the versions available is 2.x, 5.x, 5.6+ or 6.0+. 5.6+ means a version of 5.6 or less than 6.0. 6.0+ means a version of 6.0 or higher, 6.3.2 for example.
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences on how queries are composed.
Currently the versions available are `2.x`, `5.x`, `5.6+`, `6.0+` or `7.0+`. The value `5.6+` means version 5.6 or higher, but lower than 6.0. The value `6.0+` means
version 6.0 or higher, but lower than 7.0. Finally, `7.0+` means version 7.0 or higher, but lower than 8.0.
### Min time interval
A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute.
This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a
number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported:
pkg/tsdb/elasticsearch/client/client.go
+17 -6
View File
@@ -65,7 +65,7 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", "))
switch version {
case 2, 5, 56, 60:
case 2, 5, 56, 60, 70:
return &baseClientImpl{
ctx: ctx,
ds: ds,
@@ -112,12 +112,12 @@ type multiRequest struct {
interval tsdb.Interval
}
func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) {
func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
bytes, err := c.encodeBatchRequests(requests)
if err != nil {
return nil, err
}
return c.executeRequest(http.MethodPost, uriPath, bytes)
return c.executeRequest(http.MethodPost, uriPath, uriQuery, bytes)
}
func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) {
@@ -150,9 +150,10 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
return payload.Bytes(), nil
}
func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) {
func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
u, _ := url.Parse(c.ds.Url)
u.Path = path.Join(u.Path, uriPath)
u.RawQuery = uriQuery
var req *http.Request
var err error
@@ -197,7 +198,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
clientLog.Debug("Executing multisearch", "search requests", len(r.Requests))
multiRequests := c.createMultiSearchRequests(r.Requests)
res, err := c.executeBatchRequest("_msearch", multiRequests)
queryParams := c.getMultiSearchQueryParameters()
res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
if err != nil {
return nil, err
}
@@ -241,7 +243,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
mr.header["search_type"] = "count"
}
if c.version >= 56 {
if c.version >= 56 && c.version < 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256)
mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests
}
@@ -252,6 +254,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
return multiRequests
}
func (c *baseClientImpl) getMultiSearchQueryParameters() string {
if c.version >= 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
}
return ""
}
func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
return NewMultiSearchRequestBuilder(c.GetVersion())
}
pkg/tsdb/elasticsearch/client/client_test.go
+67
View File
@@ -103,6 +103,19 @@ func TestClient(t *testing.T) {
So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 60)
})
Convey("When version 70 should return v7.0 client", func() {
ds := &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"timeField": "@timestamp",
}),
}
c, err := NewClient(context.Background(), ds, nil)
So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 70)
})
})
Convey("Given a fake http client", func() {
@@ -290,6 +303,60 @@ func TestClient(t *testing.T) {
})
})
Convey("and a v7.0 client", func() {
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"maxConcurrentShardRequests": 6,
"timeField": "@timestamp",
"interval": "Daily",
}),
}
c, err := NewClient(context.Background(), &ds, timeRange)
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c)
So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch")
So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
So(responseBuffer, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n')
So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes()
jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil)
jBody, err := simplejson.NewJson(bodyBytes)
So(err, ShouldBeNil)
So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
Convey("and replace $__interval variable", func() {
So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
})
Convey("and replace $__interval_ms variable", func() {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
})
})
})
})
Reset(func() {
newDatasourceHttpClient = currentNewDatasourceHttpClient
})
pkg/tsdb/elasticsearch/client/models.go
+1 -1
View File
@@ -42,7 +42,7 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
// SearchResponseHits represents search response hits
type SearchResponseHits struct {
Hits []map[string]interface{}
Total int64
Total map[string]interface{}
}
// SearchResponse represents a search response
public/app/plugins/datasource/elasticsearch/config_ctrl.ts
+4 -1
View File
@@ -8,7 +8,9 @@ export class ElasticConfigCtrl {
constructor($scope) {
this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp';
this.current.jsonData.esVersion = this.current.jsonData.esVersion || 5;
this.current.jsonData.maxConcurrentShardRequests = this.current.jsonData.maxConcurrentShardRequests || 256;
const defaultMaxConcurrentShardRequests = this.current.jsonData.esVersion >= 70 ? 5 : 256;
this.current.jsonData.maxConcurrentShardRequests =
this.current.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
}
indexPatternTypes = [
@@ -25,6 +27,7 @@ export class ElasticConfigCtrl {
{ name: '5.x', value: 5 },
{ name: '5.6+', value: 56 },
{ name: '6.0+', value: 60 },
{ name: '7.0+', value: 70 },
];
indexPatternTypeChanged() {
public/app/plugins/datasource/elasticsearch/datasource.ts
+24 -5
View File
@@ -234,7 +234,7 @@ export class ElasticDatasource {
ignore_unavailable: true,
index: this.indexPattern.getIndexList(timeFrom, timeTo),
};
if (this.esVersion >= 56) {
if (this.esVersion >= 56 && this.esVersion < 70) {
queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests;
}
return angular.toJson(queryHeader);
@@ -277,12 +277,15 @@ export class ElasticDatasource {
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
payload = this.templateSrv.replace(payload, options.scopedVars);
return this.post('_msearch', payload).then(res => {
const url = this.getMultiSearchUrl();
return this.post(url, payload).then(res => {
return new ElasticResponse(sentTargets, res).getTimeSeries();
});
}
getFields(query) {
const configuredEsVersion = this.esVersion;
return this.get('/_mapping').then(result => {
const typeMap = {
float: 'number',
@@ -347,8 +350,14 @@ export class ElasticDatasource {
const index = result[indexName];
if (index && index.mappings) {
const mappings = index.mappings;
for (const typeName in mappings) {
const properties = mappings[typeName].properties;
if (configuredEsVersion < 70) {
for (const typeName in mappings) {
const properties = mappings[typeName].properties;
getFieldsRecursively(properties);
}
} else {
const properties = mappings.properties;
getFieldsRecursively(properties);
}
}
@@ -371,7 +380,9 @@ export class ElasticDatasource {
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
esQuery = header + '\n' + esQuery + '\n';
return this.post('_msearch?search_type=' + searchType, esQuery).then(res => {
const url = this.getMultiSearchUrl();
return this.post(url, esQuery).then(res => {
if (!res.responses[0].aggregations) {
return [];
}
@@ -386,6 +397,14 @@ export class ElasticDatasource {
});
}
getMultiSearchUrl() {
if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
}
return '_msearch';
}
metricFindQuery(query) {
query = angular.fromJson(query);
if (!query) {
public/app/plugins/datasource/elasticsearch/elastic_response.ts
+4 -2
View File
@@ -184,7 +184,7 @@ export class ElasticResponse {
}
// This is quite complex
// need to recurise down the nested buckets to build series
// need to recurse down the nested buckets to build series
processBuckets(aggs, target, seriesList, table, props, depth) {
let bucket, aggDef, esAgg, aggId;
const maxDepth = target.bucketAggs.length - 1;
@@ -310,11 +310,13 @@ export class ElasticResponse {
}
processHits(hits, seriesList) {
const hitsTotal = typeof hits.total === 'number' ? hits.total : hits.total.value; // <- Works with Elasticsearch 7.0+
const series = {
target: 'docs',
type: 'docs',
datapoints: [],
total: hits.total,
total: hitsTotal,
filterable: true,
};
let propName, hit, doc, i;
public/app/plugins/datasource/elasticsearch/specs/datasource.test.ts
+182 -40
View File
@@ -187,7 +187,7 @@ describe('ElasticDatasource', function(this: any) {
describe('When getting fields', () => {
beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'metricbeat' });
createDatasource({ url: 'http://es.com', index: 'metricbeat', jsonData: { esVersion: 50 } });
ctx.backendSrv.datasourceRequest = jest.fn(options => {
return Promise.resolve({
@@ -236,49 +236,191 @@ describe('ElasticDatasource', function(this: any) {
});
});
it('should return nested fields', () => {
ctx.ds
.getFields({
find: 'fields',
query: '*',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp',
'beat.name.raw',
'beat.name',
'beat.hostname',
'system.cpu.system',
'system.cpu.user',
'system.process.cpu.total',
'system.process.name',
]);
});
it('should return nested fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp',
'beat.name.raw',
'beat.name',
'beat.hostname',
'system.cpu.system',
'system.cpu.user',
'system.process.cpu.total',
'system.process.name',
]);
});
it('should return fields related to query type', () => {
ctx.ds
.getFields({
find: 'fields',
query: '*',
type: 'number',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
});
it('should return number fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'number',
});
ctx.ds
.getFields({
find: 'fields',
query: '*',
type: 'date',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp']);
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp']);
});
});
describe('When getting fields from ES 7.0', () => {
beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'genuine.es7._mapping.response', jsonData: { esVersion: 70 } });
ctx.backendSrv.datasourceRequest = jest.fn(options => {
return Promise.resolve({
data: {
'genuine.es7._mapping.response': {
mappings: {
properties: {
'@timestamp_millis': {
type: 'date',
format: 'epoch_millis',
},
classification_terms: {
type: 'keyword',
},
domains: {
type: 'keyword',
},
ip_address: {
type: 'ip',
},
justification_blob: {
properties: {
criterion: {
type: 'text',
fields: {
keyword: {
type: 'keyword',
ignore_above: 256,
},
},
},
overall_vote_score: {
type: 'float',
},
shallow: {
properties: {
jsi: {
properties: {
sdb: {
properties: {
dsel2: {
properties: {
'bootlegged-gille': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
'uncombed-boris': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
},
},
},
},
},
},
},
},
},
},
overall_vote_score: {
type: 'float',
},
ua_terms_long: {
type: 'keyword',
},
ua_terms_short: {
type: 'keyword',
},
},
},
},
},
});
});
});
it('should return nested fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp_millis',
'classification_terms',
'domains',
'ip_address',
'justification_blob.criterion.keyword',
'justification_blob.criterion',
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
'ua_terms_long',
'ua_terms_short',
]);
});
it('should return number fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'number',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp_millis']);
});
});