working on metic key indexing
This commit is contained in:
Torkel Ödegaard
@@ -8,73 +8,116 @@ function (angular, _, config) {
|
||||
|
||||
var module = angular.module('kibana.controllers');
|
||||
|
||||
module.controller('MetricKeysCtrl', function($scope, $http) {
|
||||
module.controller('MetricKeysCtrl', function($scope, $http, $q) {
|
||||
var elasticSearchUrlForMetricIndex = config.elasticsearch + '/' + config.grafana_index + '/';
|
||||
var loadingDefered;
|
||||
|
||||
$scope.init = function () {
|
||||
$scope.metricPath = "prod.apps.api.boobarella.*";
|
||||
$scope.metricCounter = 0;
|
||||
};
|
||||
|
||||
function deleteIndex()
|
||||
{
|
||||
var deferred = $q.defer();
|
||||
$http.delete(elasticSearchUrlForMetricIndex)
|
||||
.success(function() {
|
||||
deferred.resolve('ok');
|
||||
})
|
||||
.error(function(data, status) {
|
||||
if (status === 404) {
|
||||
deferred.resolve('ok');
|
||||
}
|
||||
else {
|
||||
deferred.reject('elastic search returned unexpected error');
|
||||
}
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
function createIndex()
|
||||
{
|
||||
return $http.put(elasticSearchUrlForMetricIndex, {
|
||||
settings: {
|
||||
analysis: {
|
||||
analyzer: {
|
||||
metric_path_ngram : { tokenizer : "my_ngram_tokenizer" }
|
||||
},
|
||||
tokenizer: {
|
||||
my_ngram_tokenizer : {
|
||||
type : "nGram",
|
||||
min_gram : "3",
|
||||
max_gram : "8",
|
||||
token_chars: [ "letter", "digit", "punctuation", "symbol"]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
mappings: {
|
||||
metricKey: {
|
||||
properties: {
|
||||
metricPath: {
|
||||
/* type: "string",
|
||||
index: "analyzed",
|
||||
index_analyzer: "metric_path_ngram"
|
||||
*/
|
||||
type: "multi_field",
|
||||
fields: {
|
||||
"metricPath": { type: "string", index: "analyzed", index_analyzer: "standard" },
|
||||
"metricPath_ng": { type: "string", index: "analyzed", index_analyzer: "metric_path_ngram" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
$scope.createIndex = function () {
|
||||
$scope.errorText = null;
|
||||
$scope.infoText = null;
|
||||
|
||||
deleteIndex()
|
||||
.then(createIndex)
|
||||
.then(function () {
|
||||
$scope.infoText = "Index created!";
|
||||
})
|
||||
.then(null, function (err) {
|
||||
$scope.errorText = angular.toJson(err);
|
||||
});
|
||||
};
|
||||
|
||||
$scope.loadMetricsFromPath = function () {
|
||||
$scope.infoText = 'loading';
|
||||
loadMetricsRecursive($scope.metricPath)
|
||||
$scope.errorText = null;
|
||||
$scope.infoText = null;
|
||||
$scope.metricCounter = 0;
|
||||
|
||||
/*$http.put(config.elasticsearch + "/grafana-int/", {
|
||||
{
|
||||
"settings": {
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"category_path": {
|
||||
"type": "custom",
|
||||
"tokenizer": "category_path"
|
||||
},
|
||||
"my_ngram_analyzer" : {
|
||||
"tokenizer" : "my_ngram_tokenizer"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"category_path": {
|
||||
"type": "path_hierarchy",
|
||||
"delimiter": "."
|
||||
},
|
||||
"my_ngram_tokenizer" : {
|
||||
"type" : "nGram",
|
||||
"min_gram" : "4",
|
||||
"max_gram" : "8",
|
||||
"token_chars": [ "letter", "digit" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"metricKey": {
|
||||
"properties": {
|
||||
"metricPath": {
|
||||
"type": "string",
|
||||
"index": "analyzed",
|
||||
"index_analyzer": "my_ngram_analyzer"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});*/
|
||||
return loadMetricsRecursive($scope.metricPath)
|
||||
.then(function() {
|
||||
$scope.infoText = "Indexing completed!";
|
||||
}, function(err) {
|
||||
$scope.errorText = "Error: " + err;
|
||||
});
|
||||
};
|
||||
|
||||
function receiveMetric(data) {
|
||||
function receiveMetric(result) {
|
||||
var data = result.data;
|
||||
if (!data || data.length == 0) {
|
||||
console.log('no data');
|
||||
return;
|
||||
}
|
||||
|
||||
_.each(data, function(metric) {
|
||||
var funcs = _.map(data, function(metric) {
|
||||
if (metric.expandable) {
|
||||
console.log('Loading children: ' + metric.id);
|
||||
loadMetricsRecursive(metric.id + ".*");
|
||||
return loadMetricsRecursive(metric.id + ".*");
|
||||
}
|
||||
if (metric.leaf) {
|
||||
saveMetricKey(metric.id);
|
||||
return saveMetricKey(metric.id);
|
||||
}
|
||||
});
|
||||
|
||||
return $q.all(funcs);
|
||||
}
|
||||
|
||||
function saveMetricKey(metricId) {
|
||||
@@ -84,28 +127,27 @@ function (angular, _, config) {
|
||||
metricPath: metricId
|
||||
});
|
||||
|
||||
request.doIndex(
|
||||
return request.doIndex(
|
||||
// Success
|
||||
function(result) {
|
||||
console.log('save metric success', result);
|
||||
$scope.infoText = "Indexing " + metricId;
|
||||
$scope.metricCounter = $scope.metricCounter + 1;
|
||||
},
|
||||
function(error) {
|
||||
console.log('save metric error', error);
|
||||
$scope.errorText = "failed to save metric " + metricId;
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function metricLoadError(data, status, headers, config)
|
||||
{
|
||||
console.log('error: ' + status);
|
||||
$scope.error = "failed to get metrics from graphite";
|
||||
$scope.errorText = "failed to get metric";
|
||||
}
|
||||
|
||||
function loadMetricsRecursive(metricPath, data, callback)
|
||||
function loadMetricsRecursive(metricPath)
|
||||
{
|
||||
$http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} )
|
||||
.success(receiveMetric)
|
||||
.error(metricLoadError);
|
||||
return $http({ method: 'GET', url: config.graphiteUrl + '/metrics/find/?query=' + metricPath} )
|
||||
.then(receiveMetric);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
@@ -21,15 +21,18 @@ function (angular, _, config, $) {
|
||||
};
|
||||
};
|
||||
|
||||
$scope.elasticsearch_dblist = function(query) {
|
||||
var words = query.split(" ");
|
||||
$scope.elasticsearch_dblist = function(queryStr) {
|
||||
var words = queryStr.split(" ");
|
||||
var query = $scope.ejs.BoolQuery();
|
||||
var terms = _.map(words, function(word) {
|
||||
return $scope.ejs.MatchQuery("metricPath", word);
|
||||
return $scope.ejs.MatchQuery("metricPath_ng", word).boost(1.2);
|
||||
});
|
||||
|
||||
console.log("query: ", terms);
|
||||
query.must(terms);
|
||||
var ngramQuery = $scope.ejs.BoolQuery();
|
||||
ngramQuery.must(terms);
|
||||
|
||||
var fieldMatchQuery = $scope.ejs.FieldQuery('metricPath', queryStr + "*").boost(1.2);
|
||||
query.should([ngramQuery, fieldMatchQuery]);
|
||||
|
||||
var request = $scope.ejs.Request().indices(config.grafana_index).types('metricKey');
|
||||
var results = request.query(query).size(20).doSearch();
|
||||
|
||||
@@ -416,7 +416,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
|
||||
restrict: 'A',
|
||||
template: '<div></div>',
|
||||
link: function(scope, elem) {
|
||||
var data = [];
|
||||
var data, plot;
|
||||
|
||||
scope.$on('refresh',function() {
|
||||
scope.get_data();
|
||||
@@ -558,7 +558,7 @@ function (angular, app, $, _, kbn, moment, timeSeries, graphiteSrv, RQ) {
|
||||
console.log('Datapoints[0] count:', data[0].data.length);
|
||||
console.log('Datapoints.Total count:', totalDataPoints);*/
|
||||
|
||||
scope.plot = $.plot(elem, data, options);
|
||||
plot = $.plot(elem, data, options);
|
||||
|
||||
} catch(e) {
|
||||
console.log(e);
|
||||
|
||||
@@ -1,25 +1,33 @@
|
||||
<div ng-controller="MetricKeysCtrl" ng-init="init()">
|
||||
<h5>Load metrics keys into elastic search</h5>
|
||||
|
||||
<div class="row-fluid">
|
||||
<div class="span8">
|
||||
<label class="small">Load all metric keys at once (intensive for graphite web)</label>
|
||||
</div>
|
||||
<div class="span4">
|
||||
<button class="btn btn-danger">Start load all</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row-fluid">
|
||||
<div class="span8">
|
||||
<label class="small">Load metric keys recursively starting at metric key path</label>
|
||||
<div class="span12">
|
||||
<label class="small">Load metrics recursive starting from this metric path</label>
|
||||
<input type="text" class="input-xlarge" ng-model="metricPath"> </input>
|
||||
</div>
|
||||
<div class="span4">
|
||||
<button class="btn btn-danger" style="margin-top: 20px" ng-click="loadMetricsFromPath()">Start load from metric path</button>
|
||||
</div>
|
||||
|
||||
<div class="row-fluid" style="margin-top: 15px;">
|
||||
<div class="span12">
|
||||
<button class="btn btn-success" ng-click="createIndex()">Clear/Create index</button>
|
||||
<button class="btn btn-success" ng-click="loadMetricsFromPath()">Load from metric path</button>
|
||||
<button class="btn btn-danger" ng-click="loadAll()">Load all</button>
|
||||
<tip>Load all will fetch all metrics in one call, can be intensive for graphite and for the browser if you have a lot of metrics</tip>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row-fluid">
|
||||
<div class="span12 alert-message block-message info">
|
||||
|
||||
<div class="row-fluid" style="margin-top: 15px;">
|
||||
<div class="span12" ng-show="infoText" style="padding-top: 10px;">
|
||||
{{infoText}}
|
||||
</div>
|
||||
<div class="span12 alert alert-error" ng-show="errorText">
|
||||
{{errorText}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row-fluid" ng-show="metricCounter">
|
||||
<div class="span12" style="padding-top: 10px;">
|
||||
Metrics indexed: {{metricCounter}}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user