diff --git a/config/setup/common/insight.tpl b/config/setup/common/insight.tpl index 0ed3abd0..0cd60d25 100644 --- a/config/setup/common/insight.tpl +++ b/config/setup/common/insight.tpl @@ -14,7 +14,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1sc28go5i051pl1i0 "formula": "a", "items": [ { - "field": "*", + "field": "id", "name": "a", "statistic": "count" } @@ -51,7 +51,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t2 "formula": "a", "items": [ { - "field": "*", + "field": "id", "name": "a", "statistic": "count" } @@ -94,7 +94,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t1 ], "items": [ { - "field": "*", + "field": "id", "name": "a", "statistic": "count" } @@ -137,7 +137,7 @@ POST $[[SETUP_INDEX_PREFIX]]widget/$[[SETUP_DOC_TYPE]]/cji1ttq8go5i051pl1t0 ], "items": [ { - "field": "*", + "field": "id", "name": "a", "statistic": "count" } @@ -3566,4 +3566,579 @@ POST $[[SETUP_INDEX_PREFIX]]layout/$[[SETUP_DOC_TYPE]]/cicmhbt3q95ich72lrvg }, "type": "workspace", "is_fixed": true +} + +#shard level +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH9 +{ + "id": "bD2jH5QB7KvGccywNCH9", + "name": "Indexing Rate", + "key": "indexing_rate", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH9", + "field": "payload.elasticsearch.shard_stats.indexing.index_total", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH1 +{ + "id": "bD2jH5QB7KvGccywNCH1", + "name": "Shard Storage", + "key": "shard_storage", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH1", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH1", + "field": "payload.elasticsearch.shard_stats.store.size_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "bytes", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH5 +{ + "id": "bD2jH5QB7KvGccywNCH5", + "name": "Document Count", + "key": "doc_count", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH5", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH5", + "field": "payload.elasticsearch.shard_stats.docs.count", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH2 +{ + "id": "bD2jH5QB7KvGccywNCH2", + "name": "Search Rate", + "key": "search_rate", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH2", + "field": "payload.elasticsearch.shard_stats.search.query_total", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH3 +{ + "id": "bD2jH5QB7KvGccywNCH3", + "name": "Indexing Latency", + "key": "indexing_latency", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCx3/bD2jH5QB7KvGccywNCH3", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCx3", + "field": "payload.elasticsearch.shard_stats.indexing.index_total", + "statistic": "rate" + }, + { + "name": "bD2jH5QB7KvGccywNCH3", + "field": "payload.elasticsearch.shard_stats.indexing.index_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH4 +{ + "id": "bD2jH5QB7KvGccywNCH4", + "name": "Search Latency", + "key": "search_latency", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCx4/bD2jH5QB7KvGccywNCH4", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH4", + "field": "payload.elasticsearch.shard_stats.search.query_total", + "statistic": "rate" + }, + { + "name": "bD2jH5QB7KvGccywNCx4", + "field": "payload.elasticsearch.shard_stats.search.query_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH6 +{ + "id": "bD2jH5QB7KvGccywNCH6", + "name": "Segment Count", + "key": "segment_count", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH6", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH6", + "field": "payload.elasticsearch.shard_stats.segments.count", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/bD2jH5QB7KvGccywNCH7 +{ + "id": "bD2jH5QB7KvGccywNCH7", + "name": "Segment memory", + "key": "segment_memory", + "level": "shard", + "formula": "bD2jH5QB7KvGccywNCH7", + "items": [ + { + "name": "bD2jH5QB7KvGccywNCH7", + "field": "payload.elasticsearch.shard_stats.segments.memory_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +#indices level +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH9 +{ + "id": "aD2jH5QB7KvGccywNCH9", + "name": "Indexing Rate", + "key": "indexing_rate", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH9/{{.bucket_size_in_second}}", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH9", + "field": "payload.elasticsearch.index_stats.primaries.indexing.index_total", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH1 +{ + "id": "aD2jH5QB7KvGccywNCH1", + "name": "Index Storage", + "key": "index_storage", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH1", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH1", + "field": "payload.elasticsearch.index_stats.total.store.size_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "bytes", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH5 +{ + "id": "aD2jH5QB7KvGccywNCH5", + "name": "Document Count", + "key": "doc_count", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH5", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH5", + "field": "payload.elasticsearch.index_stats.total.docs.count", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH2 +{ + "id": "aD2jH5QB7KvGccywNCH2", + "name": "Search Rate", + "key": "search_rate", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH2/{{.bucket_size_in_second}}", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH2", + "field": "payload.elasticsearch.index_stats.total.search.query_total", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH3 +{ + "id": "aD2jH5QB7KvGccywNCH3", + "name": "Indexing Latency", + "key": "indexing_latency", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCx3/aD2jH5QB7KvGccywNCH3", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH3", + "field": "payload.elasticsearch.index_stats.primaries.indexing.index_total", + "statistic": "rate" + }, + { + "name": "aD2jH5QB7KvGccywNCx3", + "field": "payload.elasticsearch.index_stats.primaries.indexing.index_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH4 +{ + "id": "aD2jH5QB7KvGccywNCH4", + "name": "Search Latency", + "key": "search_latency", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCx4/aD2jH5QB7KvGccywNCH4", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH4", + "field": "payload.elasticsearch.index_stats.total.search.query_total", + "statistic": "rate" + }, + { + "name": "aD2jH5QB7KvGccywNCx4", + "field": "payload.elasticsearch.index_stats.total.search.query_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH6 +{ + "id": "aD2jH5QB7KvGccywNCH6", + "name": "Segment Count", + "key": "segment_count", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH6", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH6", + "field": "payload.elasticsearch.index_stats.total.segments.count", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/aD2jH5QB7KvGccywNCH7 +{ + "id": "aD2jH5QB7KvGccywNCH7", + "name": "Segment memory", + "key": "segment_memory", + "level": "indices", + "formula": "aD2jH5QB7KvGccywNCH7", + "items": [ + { + "name": "aD2jH5QB7KvGccywNCH7", + "field": "payload.elasticsearch.index_stats.total.segments.memory_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "bytes", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +#node level +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9 +{ + "id": "jD2jH5QB7KvGccywNCH9", + "name": "Indexing Rate", + "key": "indexing_rate", + "level": "node", + "formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}", + "items": [ + { + "name": "jD2jH5QB7KvGccywH9", + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + "statistic": "derivative" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + + +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH4 +{ + "id": "jD2jH5QB7KvGccywNCH4", + "name": "Process CPU Usage", + "key": "process_cpu_used", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCH4", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH4", + "field": "payload.elasticsearch.node_stats.process.cpu.percent", + "statistic": "avg" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "", + "unit": "%", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH3 +{ + "id": "jD2jH5QB7KvGccywNCH3", + "name": "JVM Heap Usage", + "key": "jvm_heap_used", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCH3", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH3", + "field": "payload.elasticsearch.node_stats.jvm.mem.heap_used_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "bytes", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH1 +{ + "id": "jD2jH5QB7KvGccywNCH1", + "name": "Indexing Latency", + "key": "indexing_latency", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCx1/jD2jH5QB7KvGccywNCH1", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH1", + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + "statistic": "rate" + }, + { + "name": "jD2jH5QB7KvGccywNCx1", + "field": "payload.elasticsearch.node_stats.indices.indexing.index_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH0 +{ + "id": "jD2jH5QB7KvGccywNCH0", + "name": "Search Rate", + "key": "search_rate", + "level": "node", + "formula": "jD2jH5QB7KvGccywH0/{{.bucket_size_in_second}}", + "items": [ + { + "name": "jD2jH5QB7KvGccywH0", + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + "statistic": "derivative" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "query/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH9 +{ + "id": "jD2jH5QB7KvGccywNCH9", + "name": "Indexing Rate", + "key": "indexing_rate", + "level": "node", + "formula": "jD2jH5QB7KvGccywH9/{{.bucket_size_in_second}}", + "items": [ + { + "name": "jD2jH5QB7KvGccywH9", + "field": "payload.elasticsearch.node_stats.indices.indexing.index_total", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "doc/s", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH5 +{ + "id": "jD2jH5QB7KvGccywNCH5", + "name": "Search Latency", + "key": "search_latency", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCx5/jD2jH5QB7KvGccywNCH5", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH5", + "field": "payload.elasticsearch.node_stats.indices.search.query_total", + "statistic": "rate" + }, + { + "name": "jD2jH5QB7KvGccywNCx5", + "field": "payload.elasticsearch.node_stats.indices.search.query_time_in_millis", + "statistic": "rate" + } + ], + "statistics": ["rate"], + "format": "number", + "unit": "ms", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} + +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH6 +{ + "id": "jD2jH5QB7KvGccywNCH6", + "name": "Indices Storage", + "key": "indices_storage", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCH6", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH6", + "field": "payload.elasticsearch.node_stats.indices.store.size_in_bytes", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "bytes", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" +} +PUT .infini_metric/_doc/jD2jH5QB7KvGccywNCH7 +{ + "id": "jD2jH5QB7KvGccywNCH7", + "name": "Document Count", + "key": "doc_count", + "level": "node", + "formula": "jD2jH5QB7KvGccywNCH7", + "items": [ + { + "name": "jD2jH5QB7KvGccywNCH7", + "field": "payload.elasticsearch.node_stats.indices.docs.count", + "statistic": "max" + } + ], + "statistics": ["max", "min", "sum", "avg", "p99", "medium"], + "format": "number", + "unit": "", + "builtin": true, + "created": "2025-01-09T14:30:56.63155+08:00", + "updated": "2025-01-09T14:30:56.63155+08:00" } \ No newline at end of file diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 53790b3f..698e7318 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -14,7 +14,9 @@ Information about release notes of INFINI Console is provided here. ### Features - Add allocation to activities if is cluster health change and changed to red. - Add index metrics for segment memory (norms, points, version map, fixed bit set). - +- Support querying top N metrics in the Insight Data Query API +- Add insight metric CURD API for managing custom metrics +- Add built-in metrics templates for common use cases ### Bug fix - Fixed query thread pool metrics when cluster uuid is empty - Fixed unit tests diff --git a/main.go b/main.go index 90c5fa8c..1cfc040a 100644 --- a/main.go +++ b/main.go @@ -158,6 +158,7 @@ func main() { orm.RegisterSchemaWithIndexName(api3.RemoteConfig{}, "configs") orm.RegisterSchemaWithIndexName(model.AuditLog{}, "audit-logs") orm.RegisterSchemaWithIndexName(host.HostInfo{}, "host") + orm.RegisterSchemaWithIndexName(insight.MetricBase{}, "metric") module.Start() diff --git a/model/insight/metric_data.go b/model/insight/metric_data.go index a0e99644..ce7f8dc2 100644 --- a/model/insight/metric_data.go +++ b/model/insight/metric_data.go @@ -29,6 +29,7 @@ package insight import ( "fmt" + "infini.sh/framework/core/orm" "infini.sh/framework/core/util" "regexp" ) @@ -43,11 +44,35 @@ type Metric struct { Sort []GroupSort `json:"sort,omitempty"` ClusterId string `json:"cluster_id,omitempty"` Formula string `json:"formula,omitempty"` + //array of formula for new version + Formulas []string `json:"formulas,omitempty"` Items []MetricItem `json:"items"` - FormatType string `json:"format_type,omitempty"` + FormatType string `json:"format,omitempty"` TimeFilter interface{} `json:"time_filter,omitempty"` TimeBeforeGroup bool `json:"time_before_group,omitempty"` BucketLabel *BucketLabel `json:"bucket_label,omitempty"` + // number of buckets to return, used for aggregation auto_date_histogram when bucket size equals 'auto' + Buckets uint `json:"buckets,omitempty"` + Unit string `json:"unit,omitempty"` +} + +type MetricBase struct { + orm.ORMObjectBase + //display name of the metric + Name string `json:"name"` + //metric identifier + Key string `json:"key"` + //optional values : "node", "indices", "shard" + Level string `json:"level"` + //metric calculation formula + Formula string `json:"formula,omitempty"` + Items []MetricItem `json:"items"` + FormatType string `json:"format,omitempty"` + Unit string `json:"unit,omitempty"` + //determine if this metric is built-in + Builtin bool `json:"builtin"` + //array of supported calculation statistic, eg: "avg", "sum", "min", "max" + Statistics []string `json:"statistics,omitempty"` } type GroupSort struct { @@ -105,12 +130,8 @@ func (m *Metric) ValidateSortKey() error { if !util.StringInArray([]string{"desc", "asc"}, sortItem.Direction){ return fmt.Errorf("unknown sort direction [%s]", sortItem.Direction) } - if v, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){ + if _, ok := mm[sortItem.Key]; !ok && !util.StringInArray([]string{"_key", "_count"}, sortItem.Key){ return fmt.Errorf("unknown sort key [%s]", sortItem.Key) - }else{ - if v != nil && v.Statistic == "derivative" { - return fmt.Errorf("can not sort by pipeline agg [%s]", v.Statistic) - } } } return nil diff --git a/plugin/api/insight/api.go b/plugin/api/insight/api.go index bd9b8dfb..88ce59f6 100644 --- a/plugin/api/insight/api.go +++ b/plugin/api/insight/api.go @@ -56,4 +56,7 @@ func InitAPI() { api.HandleAPIMethod(api.POST, "/elasticsearch/:id/map_label/_render", insight.renderMapLabelTemplate) api.HandleAPIMethod(api.GET, "/insight/widget/:widget_id", insight.getWidget) api.HandleAPIMethod(api.POST, "/insight/widget", insight.RequireLogin(insight.createWidget)) + api.HandleAPIMethod(api.POST, "/insight/metric", insight.createMetric) + api.HandleAPIMethod(api.PUT, "/insight/metric/:metric_id", insight.updateMetric) + api.HandleAPIMethod(api.DELETE, "/insight/metric/:metric_id", insight.deleteMetric) } diff --git a/plugin/api/insight/metadata.go b/plugin/api/insight/metadata.go index 9f4b0057..c3710989 100644 --- a/plugin/api/insight/metadata.go +++ b/plugin/api/insight/metadata.go @@ -248,8 +248,8 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { return nil, err } esClient := elastic.GetClient(metric.ClusterId) - //log.Error(string(util.MustToJSONBytes(query))) - searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, util.MustToJSONBytes(query)) + queryDSL := util.MustToJSONBytes(query) + searchRes, err := esClient.SearchWithRawQueryDSL(metric.IndexPattern, queryDSL) if err != nil { return nil, err } @@ -266,83 +266,101 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { } } timeBeforeGroup := metric.AutoTimeBeforeGroup() - metricData := CollectMetricData(agg, timeBeforeGroup) + metricData, interval := CollectMetricData(agg, timeBeforeGroup) + formula := strings.TrimSpace(metric.Formula) + //support older versions for a single formula. + if metric.Formula != "" && len(metric.Formulas) == 0 { + metric.Formulas = []string{metric.Formula} + } var targetMetricData []insight.MetricData - formula := strings.TrimSpace(metric.Formula) - if len(metric.Items) == 1 && formula == "" { + if len(metric.Items) == 1 && len(metric.Formulas) == 0 { targetMetricData = metricData } else { - tpl, err := template.New("insight_formula").Parse(formula) - if err != nil { - return nil, err - } - msgBuffer := &bytes.Buffer{} params := map[string]interface{}{} if metric.BucketSize != "" { - du, err := util.ParseDuration(metric.BucketSize) - if err != nil { - return nil, err + bucketSize := metric.BucketSize + if metric.BucketSize == "auto" && interval != "" { + bucketSize = interval + } + if interval != "" || bucketSize != "auto" { + du, err := util.ParseDuration(bucketSize) + if err != nil { + return nil, err + } + params["bucket_size_in_second"] = du.Seconds() } - params["bucket_size_in_second"] = du.Seconds() - } - err = tpl.Execute(msgBuffer, params) - if err != nil { - return nil, err } - formula = msgBuffer.String() for _, md := range metricData { targetData := insight.MetricData{ Groups: md.Groups, Data: map[string][]insight.MetricDataItem{}, } - expression, err := govaluate.NewEvaluableExpression(formula) - if err != nil { - return nil, err - } - dataLength := 0 - for _, v := range md.Data { - dataLength = len(v) - break - } - DataLoop: - for i := 0; i < dataLength; i++ { - parameters := map[string]interface{}{} - var timestamp interface{} - hasValidData := false - for k, v := range md.Data { - if len(k) == 20 { - continue - } - if len(v) < dataLength { - continue - } - if _, ok := v[i].Value.(float64); !ok { - continue DataLoop - } - hasValidData = true - parameters[k] = v[i].Value - timestamp = v[i].Timestamp + retMetricDataItem := insight.MetricDataItem{} + for _, formula = range metric.Formulas { + tpl, err := template.New("insight_formula").Parse(formula) + if err != nil { + return nil, err } - //todo return error? - if !hasValidData { - continue + msgBuffer := &bytes.Buffer{} + err = tpl.Execute(msgBuffer, params) + if err != nil { + return nil, err } - result, err := expression.Evaluate(parameters) + resolvedFormula := msgBuffer.String() + expression, err := govaluate.NewEvaluableExpression(resolvedFormula) if err != nil { return nil, err } - if r, ok := result.(float64); ok { - if math.IsNaN(r) || math.IsInf(r, 0) { - //if !isFilterNaN { - // targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()}) - //} + dataLength := 0 + for _, v := range md.Data { + dataLength = len(v) + break + } + DataLoop: + for i := 0; i < dataLength; i++ { + parameters := map[string]interface{}{} + var timestamp interface{} + hasValidData := false + for k, v := range md.Data { + if _, ok := v[i].Value.(float64); !ok { + continue DataLoop + } + hasValidData = true + parameters[k] = v[i].Value + timestamp = v[i].Timestamp + } + //todo return error? + if !hasValidData { + continue + } + result, err := expression.Evaluate(parameters) + if err != nil { + log.Debugf("evaluate formula error: %v", err) continue } + if r, ok := result.(float64); ok { + if math.IsNaN(r) || math.IsInf(r, 0) { + //if !isFilterNaN { + // targetData.Data["result"] = append(targetData.Data["result"], []interface{}{timestamp, math.NaN()}) + //} + continue + } + } + retMetricDataItem.Timestamp = timestamp + if len(metric.Formulas) <= 1 && metric.Formula != ""{ + //support older versions by returning the result for a single formula. + retMetricDataItem.Value = result + } else { + if v, ok := retMetricDataItem.Value.(map[string]interface{}); ok { + v[formula] = result + }else{ + retMetricDataItem.Value = map[string]interface{}{formula: result} + } + } } - - targetData.Data["result"] = append(targetData.Data["result"], insight.MetricDataItem{Timestamp: timestamp, Value: result}) } + targetData.Data["result"] = append(targetData.Data["result"], retMetricDataItem) targetMetricData = append(targetMetricData, targetData) } } @@ -356,7 +374,10 @@ func getMetricData(metric *insight.Metric) (interface{}, error) { } } } - return result, nil + return util.MapStr{ + "data": result, + "request": string(queryDSL), + }, nil } func getMetadataByIndexPattern(clusterID, indexPattern, timeField string, filter interface{}, fieldsFormat map[string]string) (interface{}, error) { diff --git a/plugin/api/insight/metric.go b/plugin/api/insight/metric.go new file mode 100644 index 00000000..add7ad14 --- /dev/null +++ b/plugin/api/insight/metric.go @@ -0,0 +1,166 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Console is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* Copyright © INFINI Ltd. All rights reserved. + * web: https://infinilabs.com + * mail: hello#infini.ltd */ + +package insight + +import ( + "errors" + log "github.com/cihub/seelog" + "infini.sh/console/model/insight" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" + "infini.sh/framework/modules/elastic" + "net/http" +) + +func (h *InsightAPI) createMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + var obj = &insight.MetricBase{} + err := h.DecodeJSON(req, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + err = orm.Create(nil, obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "created", + }, 200) + +} + +func (h *InsightAPI) getMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("metric_id") + + obj := insight.MetricBase{} + obj.ID = id + + _, err := orm.Get(&obj) + if err != nil { + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + h.WriteJSON(w, util.MapStr{ + "found": true, + "_id": id, + "_source": obj, + }, 200) +} + +func (h *InsightAPI) updateMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("metric_id") + obj := insight.MetricBase{} + + obj.ID = id + _, err := orm.Get(&obj) + if err != nil { + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + id = obj.ID + create := obj.Created + obj = insight.MetricBase{} + err = h.DecodeJSON(req, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + //protect + obj.ID = id + obj.Created = create + err = orm.Update(nil, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "updated", + }, 200) +} + +func (h *InsightAPI) deleteMetric(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + id := ps.MustGetParameter("metric_id") + + obj := insight.MetricBase{} + obj.ID = id + + _, err := orm.Get(&obj) + if err != nil { + if errors.Is(err, elastic.ErrNotFound) { + h.WriteJSON(w, util.MapStr{ + "_id": id, + "found": false, + }, http.StatusNotFound) + return + } + h.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if obj.Builtin { + h.WriteError(w, "cannot delete builtin metrics", http.StatusBadRequest) + return + } + + err = orm.Delete(nil, &obj) + if err != nil { + h.WriteError(w, err.Error(), http.StatusInternalServerError) + log.Error(err) + return + } + + h.WriteJSON(w, util.MapStr{ + "_id": obj.ID, + "result": "deleted", + }, 200) +} diff --git a/plugin/api/insight/metric_util.go b/plugin/api/insight/metric_util.go index d972a81a..fce5a25c 100644 --- a/plugin/api/insight/metric_util.go +++ b/plugin/api/insight/metric_util.go @@ -72,7 +72,7 @@ func generateAgg(metricItem *insight.MetricItem, timeField string) map[string]in "includes": []string{field}, } aggValue["sort"] = []util.MapStr{ - util.MapStr{ + { timeField: util.MapStr{ "order": "desc", }, @@ -114,23 +114,46 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { return nil, err } } - verInfo := elastic.GetClient(metric.ClusterId).GetVersion() + var ( + useDateHistogram = false + dateHistogramAgg util.MapStr + dateHistogramAggName string + ) + if metric.BucketSize != "" && metric.TimeField != "" { + useDateHistogram = true + if metric.BucketSize == "auto" { + dateHistogramAggName = "auto_date_histogram" + buckets := metric.Buckets + if buckets == 0 { + buckets = 2 + } + dateHistogramAgg = util.MapStr{ + "field": metric.TimeField, + "buckets": buckets, + } + }else{ + dateHistogramAggName = "date_histogram" + verInfo := elastic.GetClient(metric.ClusterId).GetVersion() - if verInfo.Number == "" { - panic("invalid version") - } + if verInfo.Number == "" { + panic("invalid version") + } - intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize) - if err != nil { - return nil, fmt.Errorf("get interval field error: %w", err) + intervalField, err := elastic.GetDateHistogramIntervalField(verInfo.Distribution, verInfo.Number, metric.BucketSize) + if err != nil { + return nil, fmt.Errorf("get interval field error: %w", err) + } + dateHistogramAgg = util.MapStr{ + "field": metric.TimeField, + intervalField: metric.BucketSize, + } + } } - if metric.BucketSize != "" && !timeBeforeGroup { + + if useDateHistogram && !timeBeforeGroup { basicAggs = util.MapStr{ "time_buckets": util.MapStr{ - "date_histogram": util.MapStr{ - "field": metric.TimeField, - intervalField: metric.BucketSize, - }, + dateHistogramAggName: dateHistogramAgg, "aggs": basicAggs, }, } @@ -138,7 +161,7 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { var rootAggs util.MapStr groups := metric.Groups - err = metric.ValidateSortKey() + err := metric.ValidateSortKey() if err != nil { return nil, err } @@ -156,12 +179,43 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { "field": groups[i].Field, "size": limit, } - if i == grpLength-1 && len(metric.Sort) > 0 { - var termsOrder []interface{} - for _, sortItem := range metric.Sort { - termsOrder = append(termsOrder, util.MapStr{sortItem.Key: sortItem.Direction}) + if i == grpLength - 1 && len(metric.Sort) > 0 { + //use bucket sort instead of terms order when time after group + if !timeBeforeGroup && len(metric.Sort) > 0 { + basicAggs["sort_field"] = util.MapStr{ + "max_bucket": util.MapStr{ + "buckets_path": fmt.Sprintf("time_buckets>%s", metric.Sort[0].Key), + }, + } + //using 65536 as a workaround for the terms aggregation limit; the actual limit is enforced in the bucket sort step + termsCfg["size"] = 65536 + basicAggs["bucket_sorter"] = util.MapStr{ + "bucket_sort": util.MapStr{ + "size": limit, + "sort": []util.MapStr{ + {"sort_field": util.MapStr{"order": metric.Sort[0].Direction}}, + }, + }, + } + }else{ + var termsOrder []interface{} + percentAggs := []string{"p99", "p95", "p90", "p80", "p50"} + for _, sortItem := range metric.Sort { + var percent string + for _, item := range metric.Items { + lowerCaseStatistic := strings.ToLower(item.Statistic) + if item.Name == sortItem.Key && util.StringInArray(percentAggs, lowerCaseStatistic) { + percent = lowerCaseStatistic[1:] + } + } + sortKey := sortItem.Key + if percent != "" { + sortKey = fmt.Sprintf("%s[%s]", sortItem.Key, percent) + } + termsOrder = append(termsOrder, util.MapStr{sortKey: sortItem.Direction}) + } + termsCfg["order"] = termsOrder } - termsCfg["order"] = termsOrder } groupAgg := util.MapStr{ "terms": termsCfg, @@ -176,32 +230,26 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { } lastGroupAgg = groupAgg } - if metric.BucketSize == "" || (metric.BucketSize != "" && !timeBeforeGroup) { - rootAggs = util.MapStr{ - util.GetUUID(): lastGroupAgg, - } - } else { + if useDateHistogram && timeBeforeGroup { rootAggs = util.MapStr{ "time_buckets": util.MapStr{ - "date_histogram": util.MapStr{ - "field": metric.TimeField, - intervalField: metric.BucketSize, - }, + dateHistogramAggName: dateHistogramAgg, "aggs": util.MapStr{ util.GetUUID(): lastGroupAgg, }, }, } + } else { + rootAggs = util.MapStr{ + util.GetUUID(): lastGroupAgg, + } } } else { if metric.BucketSize != "" && timeBeforeGroup { basicAggs = util.MapStr{ "time_buckets": util.MapStr{ - "date_histogram": util.MapStr{ - "field": metric.TimeField, - intervalField: metric.BucketSize, - }, + dateHistogramAggName: dateHistogramAgg, "aggs": basicAggs, }, } @@ -228,20 +276,22 @@ func GenerateQuery(metric *insight.Metric) (interface{}, error) { return queryDsl, nil } -func CollectMetricData(agg interface{}, timeBeforeGroup bool) []insight.MetricData { +func CollectMetricData(agg interface{}, timeBeforeGroup bool) ([]insight.MetricData, string) { metricData := []insight.MetricData{} + var interval string if timeBeforeGroup { - collectMetricDataOther(agg, nil, &metricData, nil) + interval = collectMetricDataOther(agg, nil, &metricData, nil) } else { - collectMetricData(agg, nil, &metricData) + interval = collectMetricData(agg, nil, &metricData) } - return metricData + return metricData, interval } // timeBeforeGroup => false -func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) { +func collectMetricData(agg interface{}, groupValues []string, metricData *[]insight.MetricData) (interval string){ if aggM, ok := agg.(map[string]interface{}); ok { if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok { + interval, _ = timeBks["interval"].(string) if bks, ok := timeBks["buckets"].([]interface{}); ok { md := insight.MetricData{ Data: map[string][]insight.MetricDataItem{}, @@ -255,7 +305,7 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi continue } - if vm, ok := v.(map[string]interface{}); ok && len(k) < 5 { + if vm, ok := v.(map[string]interface{}); ok { collectMetricDataItem(k, vm, &md, bkM["key"]) } @@ -283,14 +333,12 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi newGroupValues := make([]string, 0, len(groupValues)+1) newGroupValues = append(newGroupValues, groupValues...) newGroupValues = append(newGroupValues, currentGroup) - collectMetricData(bk, newGroupValues, metricData) + interval = collectMetricData(bk, newGroupValues, metricData) } } } else { //non time series metric data - if len(k) < 5 { - collectMetricDataItem(k, vm, &md, nil) - } + collectMetricDataItem(k, vm, &md, nil) } } } @@ -299,12 +347,14 @@ func collectMetricData(agg interface{}, groupValues []string, metricData *[]insi } } } + return } // timeBeforeGroup => true -func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) { +func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[]insight.MetricData, timeKey interface{}) (interval string){ if aggM, ok := agg.(map[string]interface{}); ok { if timeBks, ok := aggM["time_buckets"].(map[string]interface{}); ok { + interval, _ = timeBks["interval"].(string) if bks, ok := timeBks["buckets"].([]interface{}); ok { md := insight.MetricData{ Data: map[string][]insight.MetricDataItem{}, @@ -318,7 +368,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[ } if vm, ok := v.(map[string]interface{}); ok { if vm["buckets"] != nil { - collectMetricDataOther(vm, groupValues, metricData, bkM["key"]) + interval = collectMetricDataOther(vm, groupValues, metricData, bkM["key"]) } else { collectMetricDataItem(k, vm, &md, bkM["key"]) } @@ -346,7 +396,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[ newGroupValues := make([]string, 0, len(groupValues)+1) newGroupValues = append(newGroupValues, groupValues...) newGroupValues = append(newGroupValues, currentGroup) - collectMetricDataOther(bk, newGroupValues, metricData, timeKey) + interval = collectMetricDataOther(bk, newGroupValues, metricData, timeKey) } } } else { @@ -354,7 +404,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[ for k, v := range aggM { if vm, ok := v.(map[string]interface{}); ok { if vm["buckets"] != nil { - collectMetricDataOther(vm, groupValues, metricData, timeKey) + interval = collectMetricDataOther(vm, groupValues, metricData, timeKey) } else { collectMetricDataItem(k, vm, &md, timeKey) } @@ -367,6 +417,7 @@ func collectMetricDataOther(agg interface{}, groupValues []string, metricData *[ } } } + return } func collectMetricDataItem(key string, vm map[string]interface{}, metricData *insight.MetricData, timeKey interface{}) { diff --git a/plugin/api/platform/domain.go b/plugin/api/platform/domain.go index 8c43a919..38a99997 100644 --- a/plugin/api/platform/domain.go +++ b/plugin/api/platform/domain.go @@ -31,6 +31,7 @@ import ( "infini.sh/console/core/security/enum" consoleModel "infini.sh/console/model" "infini.sh/console/model/alerting" + "infini.sh/console/model/insight" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" "infini.sh/framework/core/model" @@ -211,6 +212,10 @@ func GetCollectionMetas() map[string]CollectionMeta { }, MatchObject: &alerting.Rule{}, }, + "metric": { + Name: "metric", + MatchObject: &insight.MetricBase{}, + }, } }) return collectionMetas