feat: 支持waf统计器

feat: 支持服务级别waf feat: 支持动态waf级别切换 feat: 移除节点级别waf feat: 调整负载逻辑规则 feat: 优化插件开发流程 feat: 调整同步器作为插件引入 feat: 优化代码格式规范 feat: 事务日程调整 fix : 修复管理台修改配置不生效bug fix : 修复waf过滤条件判断错误
tl-open-source · Jul 7, 2022 · 82c7967 · 82c7967
1 parent beb17a6
commit 82c7967
Show file tree

Hide file tree

Showing 128 changed files with 2,986 additions and 4,248 deletions.
diff --git a/README.md b/README.md
@@ -63,7 +63,6 @@ qq交流群 : 624214498，欢迎有兴趣的童鞋提交PR, 持续更新中 ....
 - [x] 支持熔断策略调整
 
 
-
 - [x] 支持配置数据持久
 
 - [x] 支持动态节点扩展
@@ -76,6 +75,10 @@ qq交流群 : 624214498，欢迎有兴趣的童鞋提交PR, 持续更新中 ....
 
 - [x] 支持多级日志生成
 
+- [x] 支持插件二次开发
+
+- [ ] 支持安装部署脚本
+
 
 
 - [x] 支持定制WAF策略
@@ -94,27 +97,19 @@ qq交流群 : 624214498，欢迎有兴趣的童鞋提交PR, 持续更新中 ....
 
 
 
-- [ ] 支持配置灰度发布
-
-- [ ] 支持权限身份控制
-
-- [x] 支持插件二次开发
-
-- [x] 支持版本迭代数据同步
-
-- [ ] 支持集群部署数据同步
-
-- [ ] 支持安装部署脚本
+- [x] 【插件】支持版本迭代数据同步
 
-- [ ] 支持多语言管理界面
+- [ ] 【插件】支持配置灰度发布
 
+- [ ] 【插件】支持权限身份控制
 
+- [ ] 【插件】支持集群部署数据同步
 
-- [ ] 支持健康检查日志分析
+- [ ] 【插件】支持健康检查日志分析
 
-- [ ] 支持熔断限流日志分析
+- [ ] 【插件】支持熔断限流日志分析
 
-- [ ] 支持路由负载日志分析
+- [ ] 【插件】支持路由负载日志分析
 
 
 # 更新日志

diff --git a/api/router/tl_ops_api_get_balance.lua b/api/router/tl_ops_api_get_balance.lua
@@ -4,15 +4,14 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
-cjson.encode_empty_table_as_object(false)
-
-local snowflake = require("lib.snowflake");
-local cache = require("cache.tl_ops_cache"):new("tl-ops-balance");
-local tl_ops_constant_balance = require("constant.tl_ops_constant_balance");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
 
+local snowflake               = require("lib.snowflake")
+local cache                   = require("cache.tl_ops_cache_core"):new("tl-ops-balance")
+local tl_ops_constant_balance = require("constant.tl_ops_constant_balance")
+local tl_ops_rt               = require("constant.tl_ops_constant_comm").tl_ops_rt
+local tl_ops_utils_func       = require("utils.tl_ops_utils_func")
+local cjson                   = require("cjson.safe");
+cjson.encode_empty_table_as_object(false)
 
 local Router = function() 
    local code_str = cache:get(tl_ops_constant_balance.cache_key.options)

diff --git a/api/router/tl_ops_api_get_balance_api.lua b/api/router/tl_ops_api_get_balance_api.lua
@@ -4,15 +4,14 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
+local snowflake                     = require("lib.snowflake");
+local cache                         = require("cache.tl_ops_cache_core"):new("tl-ops-balance-api");
+local tl_ops_rt                     = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func             = require("utils.tl_ops_utils_func");
+local tl_ops_constant_balance_api   = require("constant.tl_ops_constant_balance_api")
+local cjson                         = require("cjson.safe"); 
 cjson.encode_empty_table_as_object(false)
 
-local snowflake = require("lib.snowflake");
-local cache = require("cache.tl_ops_cache"):new("tl-ops-balance-api");
-local tl_ops_constant_balance_api = require("constant.tl_ops_constant_balance_api");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
 local Router = function() 
     local rule, _ = cache:get(tl_ops_constant_balance_api.cache_key.rule);
     if not rule or rule == nil then

diff --git a/api/router/tl_ops_api_get_balance_cookie.lua b/api/router/tl_ops_api_get_balance_cookie.lua
@@ -4,15 +4,14 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
+local snowflake                         = require("lib.snowflake");
+local cache                             = require("cache.tl_ops_cache_core"):new("tl-ops-balance-cookie");
+local tl_ops_rt                         = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func                 = require("utils.tl_ops_utils_func");
+local tl_ops_constant_balance_cookie    = require("constant.tl_ops_constant_balance_cookie")
+local cjson                             = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
 
-local snowflake = require("lib.snowflake");
-local cache = require("cache.tl_ops_cache"):new("tl-ops-balance-cookie");
-local tl_ops_constant_balance_cookie = require("constant.tl_ops_constant_balance_cookie");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
 local Router = function() 
     local rule, _ = cache:get(tl_ops_constant_balance_cookie.cache_key.rule);
     if not rule or rule == nil then

diff --git a/api/router/tl_ops_api_get_balance_header.lua b/api/router/tl_ops_api_get_balance_header.lua
@@ -4,16 +4,14 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
+local snowflake                         = require("lib.snowflake");
+local cache                             = require("cache.tl_ops_cache_core"):new("tl-ops-balance-header");
+local tl_ops_rt                         = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func                 = require("utils.tl_ops_utils_func");
+local tl_ops_constant_balance_header    = require("constant.tl_ops_constant_balance_header")
+local cjson                             = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
 
-local snowflake = require("lib.snowflake");
-local cache = require("cache.tl_ops_cache"):new("tl-ops-balance-header");
-local tl_ops_constant_balance_header = require("constant.tl_ops_constant_balance_header");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
-
 local Router = function() 
     local rule, _ = cache:get(tl_ops_constant_balance_header.cache_key.rule);
     if not rule or rule == nil then

diff --git a/api/router/tl_ops_api_get_balance_param.lua b/api/router/tl_ops_api_get_balance_param.lua
@@ -4,15 +4,14 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
+local snowflake                     = require("lib.snowflake");
+local tl_ops_rt                     = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local cache                         = require("cache.tl_ops_cache_core"):new("tl-ops-balance-param");
+local tl_ops_constant_balance_param = require("constant.tl_ops_constant_balance_param")
+local tl_ops_utils_func             = require("utils.tl_ops_utils_func");
+local cjson                         = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
 
-local snowflake = require("lib.snowflake");
-local cache = require("cache.tl_ops_cache"):new("tl-ops-balance-param");
-local tl_ops_constant_balance_param = require("constant.tl_ops_constant_balance_param");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
 local Router = function() 
     local rule, _ = cache:get(tl_ops_constant_balance_param.cache_key.rule);
     if not rule or rule == nil then

diff --git a/api/router/tl_ops_api_get_health.lua b/api/router/tl_ops_api_get_health.lua
@@ -4,14 +4,13 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
+local cache                     = require("cache.tl_ops_cache_core"):new("tl-ops-health");
+local tl_ops_constant_health    = require("constant.tl_ops_constant_health")
+local tl_ops_rt                 = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func         = require("utils.tl_ops_utils_func");
+local cjson                     = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
 
-local cache = require("cache.tl_ops_cache"):new("tl-ops-health");
-local tl_ops_constant_health = require("constant.tl_ops_constant_health");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
 
 local Router = function() 
     local list_str, _ = cache:get(tl_ops_constant_health.cache_key.options_list);

diff --git a/api/router/tl_ops_api_get_limit.lua b/api/router/tl_ops_api_get_limit.lua
@@ -4,14 +4,13 @@
 -- @author iamtsm
 -- @email [email protected]
 
-local cjson = require("cjson");
-cjson.encode_empty_table_as_object(false)
 
-local cache = require("cache.tl_ops_cache"):new("tl-ops-limit");
+local cache                 = require("cache.tl_ops_cache_core"):new("tl-ops-limit");
 local tl_ops_constant_limit = require("constant.tl_ops_constant_limit");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
+local tl_ops_rt             = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func     = require("utils.tl_ops_utils_func");
+local cjson                 = require("cjson.safe");
+cjson.encode_empty_table_as_object(false)
 
 local Router = function() 
     local fuse_list_str, _ = cache:get(tl_ops_constant_limit.fuse.cache_key.options_list);

diff --git a/api/router/tl_ops_api_get_service.lua b/api/router/tl_ops_api_get_service.lua
@@ -5,13 +5,12 @@
 -- @email [email protected]
 
 
-local cjson = require("cjson");
+local cache                     = require("cache.tl_ops_cache_core"):new("tl-ops-service");
+local tl_ops_constant_service   = require("constant.tl_ops_constant_service");
+local tl_ops_rt                 = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func         = require("utils.tl_ops_utils_func");
+local cjson                     = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
-local cache = require("cache.tl_ops_cache"):new("tl-ops-service");
-local tl_ops_constant_service = require("constant.tl_ops_constant_service");
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-
 
 local Router = function() 
     local rule, _ = cache:get(tl_ops_constant_service.cache_key.service_rule);

diff --git a/api/router/tl_ops_api_get_state.lua b/api/router/tl_ops_api_get_state.lua
@@ -4,31 +4,27 @@
 -- @author iamtsm
 -- @email [email protected]
 
-
-local cjson = require("cjson");
+local tl_ops_constant_waf       = require("constant.tl_ops_constant_waf");
+local tl_ops_constant_balance   = require("constant.tl_ops_constant_balance");
+local tl_ops_constant_service   = require("constant.tl_ops_constant_service");
+local tl_ops_constant_health    = require("constant.tl_ops_constant_health")
+local tl_ops_constant_limit     = require("constant.tl_ops_constant_limit");
+local tl_ops_limit              = require("limit.tl_ops_limit");
+local cache_service             = require("cache.tl_ops_cache_core"):new("tl-ops-service");
+local cache_limit               = require("cache.tl_ops_cache_core"):new("tl-ops-limit");
+local cache_health              = require("cache.tl_ops_cache_core"):new("tl-ops-health");
+local tl_ops_rt                 = require("constant.tl_ops_constant_comm").tl_ops_rt;
+local tl_ops_utils_func         = require("utils.tl_ops_utils_func");
+local shared                    = ngx.shared.tlopsbalance
+local cjson                     = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
-local tl_ops_constant_balance = require("constant.tl_ops_constant_balance");
-local tl_ops_constant_service = require("constant.tl_ops_constant_service");
-local tl_ops_constant_health = require("constant.tl_ops_constant_health")
-local tl_ops_constant_limit = require("constant.tl_ops_constant_limit");
-
-local tl_ops_limit = require("limit.tl_ops_limit");
-
-local cache_service = require("cache.tl_ops_cache"):new("tl-ops-service");
-local cache_limit = require("cache.tl_ops_cache"):new("tl-ops-limit");
-local cache_health = require("cache.tl_ops_cache"):new("tl-ops-health");
-
-local tl_ops_rt = require("constant.tl_ops_constant_comm").tl_ops_rt;
-local tl_ops_utils_func = require("utils.tl_ops_utils_func");
-local shared = ngx.shared.tlopsbalance
 
 local Router = function() 
     --返回的cache state
     local cache_state = {
-        service = {}, health = {}, limit = {}, balance = {}, other = {}
+        service = {}, health = {}, limit = {}, balance = {}, waf = {}, other = {}
     }
 
-
     -- 服务相关状态
     local list_str, _ = cache_service:get(tl_ops_constant_service.cache_key.service_list);
     if not list_str or list_str == nil then
@@ -63,12 +59,21 @@ local Router = function()
             limit_version_cache = 0 --"version cache nil"
         end
 
+        -- waf统计
+        local waf_count_name_service = "tl-ops-waf-count-" .. tl_ops_constant_waf.count.interval;
+        local waf_cache_count_service = require("cache.tl_ops_cache_core"):new(waf_count_name_service);
+        local waf_success_cache_service = waf_cache_count_service:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf.cache_key.waf_interval_success, service_name, nil)) 
+        if not waf_success_cache_service then
+            waf_success_cache_service = "{}"
+        end
+
         cache_state.service[service_name] = {
             health_lock = health_lock_cache,
             health_version = health_version_cache,
             health_uncheck = health_uncheck_cache,
             limit_state = limit_state_cache,
             limit_version = limit_version_cache,
+            waf_success = waf_success_cache_service
         }
         cache_state.service[service_name].nodes = { }
 
@@ -151,13 +156,20 @@ local Router = function()
                 end
             end
 
-            local count_name = "tl-ops-balance-count-" .. tl_ops_constant_balance.count.interval;
-            local cache_balance_count = require("cache.tl_ops_cache"):new(count_name);
-            local balance_success_cache = cache_balance_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_balance.cache_key.balance_interval_success, node.service, node_id)) 
+            local balance_count_name = "tl-ops-balance-count-" .. tl_ops_constant_balance.count.interval;
+            local balance_cache_count = require("cache.tl_ops_cache_core"):new(balance_count_name);
+            local balance_success_cache = balance_cache_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_balance.cache_key.balance_interval_success, node.service, node_id)) 
             if not balance_success_cache then
                 balance_success_cache = "{}"
             end
 
+            local waf_count_name_node = "tl-ops-waf-count-" .. tl_ops_constant_waf.count.interval;
+            local waf_cache_count_node = require("cache.tl_ops_cache_core"):new(waf_count_name_node);
+            local waf_success_cache_node = waf_cache_count_node:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf.cache_key.waf_interval_success, node.service, node_id)) 
+            if not waf_success_cache_node then
+                waf_success_cache_node = "{}"
+            end
+
             cache_state.service[service_name].nodes[node.name] = {
                 health_state = health_node_state_cache,
                 health_failed = health_node_failed_cache,
@@ -171,6 +183,7 @@ local Router = function()
                 limit_pre_time = limit_pre_time,
                 limit_bucket = limit_bucket,
                 balance_success =  cjson.decode(balance_success_cache),
+                waf_success =  cjson.decode(waf_success_cache_node),
             }
         end
     end
@@ -189,7 +202,6 @@ local Router = function()
         health_timers_str = "{}" --"timers nil"
     end
     local health_timer_list = cjson.decode(health_timers_str)
-
     cache_state.health['timer_list'] = health_timer_list
     cache_state.health['options_list'] = health_options_list    
 
@@ -207,36 +219,24 @@ local Router = function()
         limit_timers_str = "{}" --"timers nil"
     end
     local limit_timer_list = cjson.decode(limit_timers_str)
-
-    -- local pre_time = shared:get(tl_ops_constant_limit.global_token.cache_key.pre_time)
-    -- if not pre_time then
-    --     pre_time = "nil" --"pre_time nil"
-    -- end
-    -- local token_bucket = shared:get(tl_ops_constant_limit.global_token.cache_key.token_bucket)
-    -- if not token_bucket then
-    --     token_bucket = "nil" --"token_bucket nil"
-    -- end
-    -- local warm = shared:get(tl_ops_constant_limit.global_token.cache_key.warm)
-    -- if not warm then
-    --     warm = "nil" --"warm nil"
-    -- end
-    -- local lock = shared:get(tl_ops_constant_limit.global_token.cache_key.lock)
-    -- if not lock then
-    --     lock = "nil" --"lock nil"
-    -- end
-    -- cache_state.limit['pre_time'] = pre_time
-    -- cache_state.limit['token_bucket'] = token_bucket
-    -- cache_state.limit['warm'] = warm
-    -- cache_state.limit['lock'] = lock
     cache_state.health['timer_list'] = limit_timer_list
     cache_state.limit['option_list'] = limit_options_list
 
 
-
     -- 路由相关
     cache_state.balance['count_interval'] = tl_ops_constant_balance.count.interval
 
 
+    -- waf相关
+    local waf_count_name_global = "tl-ops-waf-count-" .. tl_ops_constant_waf.count.interval;
+    local waf_cache_count_global = require("cache.tl_ops_cache_core"):new(waf_count_name_global);
+    local waf_success_cache_global = waf_cache_count_global:get001(tl_ops_constant_waf.cache_key.waf_interval_success) 
+    if not waf_success_cache_global then
+        waf_success_cache_global = "{}"
+    end
+    cache_state.waf['waf_success'] = cjson.decode(waf_success_cache_global) 
+
+
     -- 其他
     -- cache_state.other['dict_keys'] = shared:get_keys(1024)