Skip to content

Commit

Permalink
feat: add rollup template to setup, add rollup permission
Browse files Browse the repository at this point in the history
  • Loading branch information
@luohoufu
luohoufu committed Jan 8, 2025
1 parent 814ac64 commit e76229a
Show file tree
Hide file tree
Showing 4 changed files with 331 additions and 8 deletions.
21 changes: 16 additions & 5 deletions config/permission.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,9 +281,6 @@
{"name": "indices.field_caps", "methods":["get", "post"],
"path": "/:index_name/_field_caps"
},
{"name": "indices.exists_template", "methods":["get"],
"path": "/_template/:name"
},
{"name": "indices.field_usage_stats", "methods":["get"],
"path": "/:index_name/_field_usage_stats"
},
Expand Down Expand Up @@ -367,7 +364,6 @@
"path": "/_nodes/hot_threads"
}
],

"reindex": [
{"name": "reindex.*", "methods": ["*"],
"path": "/_reindex/*"
Expand All @@ -379,7 +375,22 @@
"path": "/_reindex/:rid/_rethrottle"
}
],

"template": [
{"name": "template.*", "methods": ["*"],
"path": "/_template/*"
},
{"name": "template", "methods": ["put", "get", "delete"],
"path": "/_template/:name"
}
],
"rollup": [
{ "name": "rollup.*","methods": ["*"],
"path": "/_rollup/*"
},
{ "name": "rollup","methods": ["put", "get", "delete"],
"path": "/_rollup/jobs/:name/_explain"
}
],
"render_search_template": [
{"name": "render_search_template.*", "methods": ["*"],
"path": "/_render/template"
Expand Down
304 changes: 304 additions & 0 deletions config/setup/easysearch/template_rollup.tpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,304 @@
PUT /.easysearch-ilm-config/_settings
{
"index": {
"mapping": {
"nested_fields": {
"limit": 1000
},
"nested_objects": {
"limit": 20000
},
"total_fields": {
"limit": 30000
}
}
}
}

DELETE _rollup/jobs/rollup_index_stats
PUT _rollup/jobs/rollup_index_stats?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_index_stats_{{ctx.source_index}}",
"timestamp": "timestamp",
"continuous": true,
"page_size": 100,
"cron": "*/10 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.labels.index_id",
"metadata.labels.index_name",
"metadata.category",
"payload.elasticsearch.index_stats.index_info.health"
],
"attributes": [
"agent.*",
"metadata.*"
],
"metrics": [
"payload.elasticsearch.index_stats.*"
],
"filter": {
"metadata.name": "index_stats"
}
}
}

DELETE _rollup/jobs/rollup_index_health
PUT _rollup/jobs/rollup_index_health?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_index_health_{{ctx.source_index}}",
"timestamp": "timestamp",
"continuous": true,
"page_size": 100,
"cron": "*/10 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.labels.index_name",
"metadata.category",
"payload.elasticsearch.index_health.status"
],
"attributes": [
"agent.*",
"metadata.*"
],
"metrics": [
"payload.elasticsearch.index_health.*"
],
"filter": {
"metadata.name": "index_health"
}
}
}

DELETE _rollup/jobs/rollup_cluster_stats
PUT _rollup/jobs/rollup_cluster_stats?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_cluster_stats_{{ctx.source_index}}",
"page_size": 100,
"continuous": true,
"cron": "*/10 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"timestamp": "timestamp",
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.category",
"payload.elasticsearch.cluster_stats.status"
],
"attributes": [
"agent.*",
"metadata.*"
],
"metrics": [
"payload.elasticsearch.cluster_stats.indices.*",
"payload.elasticsearch.cluster_stats.nodes.*"
],
"filter": {
"metadata.name": "cluster_stats"
}
}
}

DELETE _rollup/jobs/rollup_cluster_health
PUT _rollup/jobs/rollup_cluster_health?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_cluster_health_{{ctx.source_index}}",
"continuous": true,
"page_size": 100,
"cron": "*/10 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"timestamp": "timestamp",
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.category",
"payload.elasticsearch.cluster_health.status"
],
"attributes": [
"agent.*",
"metadata.*"
],
"filter": {
"metadata.name": "cluster_health"
},
"metrics": [
"payload.elasticsearch.cluster_health.*"
]
}
}

# 高级 节点
DELETE _rollup/jobs/rollup_node_stats
PUT _rollup/jobs/rollup_node_stats?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_node_stats_{{ctx.source_index}}",
"timestamp": "timestamp",
"continuous": true,
"page_size": 100,
"cron": "*/10 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"min": {}
},
{
"value_count": {}
}
],
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.category",
"metadata.labels.node_id",
"metadata.labels.transport_address"
],
"attributes": [
"agent.*",
"metadata.*"
],
"filter": {
"metadata.name": "node_stats"
},
"exclude": [
"payload.elasticsearch.node_stats.http.routes.*"
],
"metrics": [
"payload.elasticsearch.node_stats.*"
]
}
}

DELETE _rollup/jobs/rollup_shard_stats_metrics
PUT _rollup/jobs/rollup_shard_stats_metrics?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_shard_stats_metrics_{{ctx.source_index}}",
"timestamp": "timestamp",
"continuous": true,
"page_size": 100,
"cron": "*/5 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.labels.index_id",
"metadata.labels.index_name",
"metadata.category",
"metadata.labels.shard_id"
],
"attributes": [
"agent.*",
"metadata.*"
],
"metrics": [
"payload.elasticsearch.shard_stats.*"
],
"filter": {
"metadata.name": "shard_stats",
"payload.elasticsearch.shard_stats.routing.primary": true
}
}
}

DELETE _rollup/jobs/rollup_shard_stats_state
PUT _rollup/jobs/rollup_shard_stats_state?replace
{
"rollup": {
"source_index": ".infini_metrics",
"target_index": "rollup_shard_stats_state_{{ctx.source_index}}",
"timestamp": "timestamp",
"continuous": true,
"page_size": 100,
"cron": "*/5 1-23 * * *",
"timezone": "UTC",
"stats": [
{
"max": {}
},
{
"value_count": {}
}
],
"interval": "1m",
"identity": [
"metadata.labels.cluster_id",
"metadata.labels.cluster_uuid",
"metadata.labels.index_name",
"metadata.category",
"payload.elasticsearch.shard_stats.routing.state",
"metadata.labels.node_id"
],
"attributes": [
"agent.*",
"metadata.*"
],
"filter": {
"metadata.name": "shard_stats"
}
}
}
11 changes: 8 additions & 3 deletions plugin/setup/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@ import (
"crypto/md5"
"encoding/hex"
"fmt"
"infini.sh/console/core/security"
"infini.sh/framework/lib/go-ucfg"
elastic2 "infini.sh/framework/modules/elastic"
"io"
"io/ioutil"
"net/http"
Expand All @@ -42,6 +39,10 @@ import (
"strings"
"time"

"infini.sh/console/core/security"
"infini.sh/framework/lib/go-ucfg"
elastic2 "infini.sh/framework/modules/elastic"

log "github.com/cihub/seelog"
"github.com/valyala/fasttemplate"
"golang.org/x/crypto/bcrypt"
Expand Down Expand Up @@ -704,6 +705,10 @@ func (module *Module) initializeTemplate(w http.ResponseWriter, r *http.Request,
useCommon = false
dslTplFileName = "template_ilm.tpl"
elastic2.InitTemplate(true)
case "rollup":
useCommon = false
dslTplFileName = "template_rollup.tpl"
elastic2.InitTemplate(true)
case "alerting":
dslTplFileName = "alerting.tpl"
case "insight":
Expand Down
3 changes: 3 additions & 0 deletions web/src/pages/Guide/Initialization/components/Initialization/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ const errorReason = {
const initialTasks = [{
name: "template_ilm",
desc: "Initialize template and ilm"
},{
name: "rollup",
desc: "Initialize rollup template"
},{
name: "insight",
desc: "Initialize dashboard template and chart template"
Expand Down

0 comments on commit e76229a

Please sign in to comment.