From f913dbf073512d80a0587fd441cbd329f60b189e Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:34:26 +0800 Subject: [PATCH 01/11] test: ci unit test --- cmd/anomalyzer/anomalyze_test.go | 4 +- common/flow.go | 54 ++++++++++++------------ pipeline/index_diff/index_diff.go | 2 +- proxy/filters/security/ldap/ldap_test.go | 12 +++--- proxy/output/http/http.go | 4 +- 5 files changed, 40 insertions(+), 36 deletions(-) diff --git a/cmd/anomalyzer/anomalyze_test.go b/cmd/anomalyzer/anomalyze_test.go index b7a417d0..72cb46bd 100644 --- a/cmd/anomalyzer/anomalyze_test.go +++ b/cmd/anomalyzer/anomalyze_test.go @@ -64,9 +64,9 @@ func TestAnomalyzer(t *testing.T) { anomalyzer, err := NewAnomalyzer(conf, data) assert.Equal(t, nil, err, "Error initializing new anomalyzer") - prob := anomalyzer.Push(1.6) + prob := anomalyzer.Push(8.0) fmt.Println(prob) - assert.Equal(t, prob > 0.5,true, "Anomalyzer returned a probability that was too small") + assert.Equal(t, prob > 0.5, true, "Anomalyzer returned a probability that was too small") } func Example() { diff --git a/common/flow.go b/common/flow.go index 9c89ea17..d869c74f 100644 --- a/common/flow.go +++ b/common/flow.go @@ -24,14 +24,15 @@ package common import ( + "strings" + "sync" + log "github.com/cihub/seelog" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "infini.sh/framework/core/pipeline" "infini.sh/framework/lib/fasthttp" - "strings" - "sync" ) type FilterFlow struct { @@ -40,8 +41,8 @@ type FilterFlow struct { } func (flow *FilterFlow) JoinFilter(filter pipeline.Filter) *FilterFlow { - if filter==nil||filter.Filter==nil{ - panic("invalid filer") + if filter == nil { + panic("invalid filter") } flow.Filters = append(flow.Filters, filter) return flow @@ -62,7 +63,7 @@ func (flow *FilterFlow) ToString() string { func (flow *FilterFlow) Process(ctx *fasthttp.RequestCtx) { for _, v := range flow.Filters { - if v==nil{ + if v == nil { panic("invalid filter") } if !ctx.ShouldContinue() { @@ -79,22 +80,23 @@ func (flow *FilterFlow) Process(ctx *fasthttp.RequestCtx) { v.Filter(ctx) } } -var nilIDFlowError=errors.New("flow id can't be nil") -func GetFlow(flowID string) (FilterFlow,error) { +var nilIDFlowError = errors.New("flow id can't be nil") + +func GetFlow(flowID string) (FilterFlow, error) { v := FilterFlow{} - if flowID==""{ - return v,nilIDFlowError + if flowID == "" { + return v, nilIDFlowError } - v1,ok:=flows.Load(flowID) - if ok{ - return v1.(FilterFlow),nil + v1, ok := flows.Load(flowID) + if ok { + return v1.(FilterFlow), nil } - cfg,err := GetFlowConfig(flowID) - if err!=nil{ - return v,err + cfg, err := GetFlowConfig(flowID) + if err != nil { + return v, err } if global.Env().IsDebug { @@ -103,20 +105,20 @@ func GetFlow(flowID string) (FilterFlow,error) { if len(cfg.Filters) > 0 { flow1, err := pipeline.NewFilter(cfg.GetConfig()) - if flow1==nil||err != nil { - return v,err + if flow1 == nil || err != nil { + return v, err } v.JoinFilter(flow1) } flows.Store(flowID, v) - return v,nil + return v, nil } func MustGetFlow(flowID string) FilterFlow { - flow,err:=GetFlow(flowID) - if err!=nil{ + flow, err := GetFlow(flowID) + if err != nil { panic(err) } return flow @@ -137,10 +139,10 @@ func ClearFlowCache(flow string) { flows.Delete(flow) } -func GetAllFlows()map[string]FilterFlow{ - data:=map[string]FilterFlow{} +func GetAllFlows() map[string]FilterFlow { + data := map[string]FilterFlow{} flows.Range(func(key, value any) bool { - data[key.(string)]=value.(FilterFlow) + data[key.(string)] = value.(FilterFlow) return true }) return data @@ -173,10 +175,10 @@ func GetRouter(name string) RouterConfig { return v } -func GetFlowConfig(id string) (FlowConfig,error) { +func GetFlowConfig(id string) (FlowConfig, error) { v, ok := flowConfigs[id] if !ok { - return v,errors.Errorf("flow [%s] not found", id) + return v, errors.Errorf("flow [%s] not found", id) } - return v,nil + return v, nil } diff --git a/pipeline/index_diff/index_diff.go b/pipeline/index_diff/index_diff.go index fa9b7f89..db85ae0d 100644 --- a/pipeline/index_diff/index_diff.go +++ b/pipeline/index_diff/index_diff.go @@ -539,7 +539,7 @@ func (processor *IndexDiffProcessor) generateTextReport() { } if leftBuilder.Len() == 0 && rightBuilder.Len() == 0 && bothBuilder.Len() == 0 { - fmt.Println("Congratulations, the two clusters are consistent!\n") + fmt.Println("Congratulations, the two clusters are consistent.") } } diff --git a/proxy/filters/security/ldap/ldap_test.go b/proxy/filters/security/ldap/ldap_test.go index 5bb83ad8..7b2a5830 100644 --- a/proxy/filters/security/ldap/ldap_test.go +++ b/proxy/filters/security/ldap/ldap_test.go @@ -26,23 +26,26 @@ package ldap import ( "context" "fmt" - "infini.sh/framework/lib/fasthttp" - "infini.sh/gateway/lib/guardian/auth/strategies/ldap" "testing" + + "infini.sh/framework/lib/fasthttp" + "infini.sh/framework/lib/guardian/auth/strategies/ldap" ) func TestLDAPFunctions(t *testing.T) { + t.Skip() + cfg := ldap.Config{ BaseDN: "dc=example,dc=com", BindDN: "cn=read-only-admin,dc=example,dc=com", Port: 389, Host: "ldap.forumsys.com", BindPassword: "password", - UserFilter: "(uid=%s)", + UserFilter: "(uid=%s)", } - r:=fasthttp.AcquireRequest() + r := &fasthttp.Request{} r.SetBasicAuth("galieleo", "password") user, err := ldap.New(&cfg).Authenticate(context.Background(), r) @@ -53,5 +56,4 @@ func TestLDAPFunctions(t *testing.T) { fmt.Println(user.GetGroups()) fmt.Println(user.GetExtensions()) - } diff --git a/proxy/output/http/http.go b/proxy/output/http/http.go index 979ad2a3..ad4df13a 100644 --- a/proxy/output/http/http.go +++ b/proxy/output/http/http.go @@ -253,7 +253,7 @@ func (filter *HTTPFilter) forward(host string, ctx *fasthttp.RequestCtx) (err er if ok { client, ok := c.(*fasthttp.Client) if !ok { - return errors.Errorf("invalid host client:", host) + return errors.Errorf("invalid host client: %s", host) } if filter.FollowRedirects { @@ -280,7 +280,7 @@ func (filter *HTTPFilter) forward(host string, ctx *fasthttp.RequestCtx) (err er } } else { - err = errors.Errorf("invalid host client:", host) + err = errors.Errorf("invalid host client: %s", host) log.Warn(err) } From 28e7d512402182e71ddc6c03b601b722b9f78588 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:34:52 +0800 Subject: [PATCH 02/11] test: ci unit test yaml --- .github/workflows/unit_test.yml | 99 +++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/unit_test.yml diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml new file mode 100644 index 00000000..b0a1557b --- /dev/null +++ b/.github/workflows/unit_test.yml @@ -0,0 +1,99 @@ +name: Gateway Go Test + +on: + pull_request: + branches: [ "main" ] + +defaults: + run: + shell: bash + +jobs: + build: + runs-on: ubuntu-latest + env: + GO_VERSION: 1.23.4 + NODEJS_VERSION: 16.20.2 + steps: + - name: Checkout current repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: gateway + + - name: Checkout framework repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + repository: infinilabs/framework + path: framework + + + - name: Checkout framework-vendor + uses: actions/checkout@v4 + with: + ref: main + fetch-depth: 0 + repository: infinilabs/framework-vendor + path: vendor + + - name: Set up nodejs toolchain + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODEJS_VERSION }} + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + node_modules + key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }} + restore-keys: | + ${{ runner.os }}-cnpm- + + - name: Check nodejs toolchain + run: | + if ! command -v cnpm >/dev/null 2>&1; then + npm install -g rimraf --quiet --no-progress + npm install -g cnpm@9.2.0 --quiet --no-progress + fi + node -v && npm -v && cnpm -v + + - name: Set up go toolchain + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: false + cache: true + + - name: Check go toolchain + run: go version + + - name: Cache Build Output + uses: actions/cache@v4 + with: + path: | + .public + key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}- + ${{ runner.os }}-build- + + - name: Test gateway code + run: | + echo Home path is $HOME + export WORKBASE=$HOME/go/src/infini.sh + export WORK=$WORKBASE/gateway + + # for test workspace + mkdir -p $HOME/go/src/ + ln -s $GITHUB_WORKSPACE $WORKBASE + + # check work folder + ls -lrt $WORKBASE/ + ls -alrt $WORK + + # for unit test + cd $WORK + echo Testing code at $PWD ... + make test \ No newline at end of file From 537ee851a61a773f0f923674b67b1fa7f91d2e3a Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:36:51 +0800 Subject: [PATCH 03/11] test: unit test need update-plugins --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index b0a1557b..0064a8d6 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -96,4 +96,4 @@ jobs: # for unit test cd $WORK echo Testing code at $PWD ... - make test \ No newline at end of file + make update-plugins test \ No newline at end of file From 5b4efe28b4a175042131877a56ff3f5903a86451 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:47:26 +0800 Subject: [PATCH 04/11] test: unit test need config --- .github/workflows/unit_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 0064a8d6..181efb89 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -96,4 +96,4 @@ jobs: # for unit test cd $WORK echo Testing code at $PWD ... - make update-plugins test \ No newline at end of file + make config test \ No newline at end of file From 8352cf7d2ba7a1baf6c049373daac33e847b2bd9 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:52:31 +0800 Subject: [PATCH 05/11] test: for typo --- .github/workflows/unit_test.yml | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 181efb89..d303978d 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -1,4 +1,4 @@ -name: Gateway Go Test +name: Unit Test on: pull_request: @@ -13,7 +13,6 @@ jobs: runs-on: ubuntu-latest env: GO_VERSION: 1.23.4 - NODEJS_VERSION: 16.20.2 steps: - name: Checkout current repository uses: actions/checkout@v4 @@ -37,28 +36,6 @@ jobs: repository: infinilabs/framework-vendor path: vendor - - name: Set up nodejs toolchain - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODEJS_VERSION }} - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: | - node_modules - key: ${{ runner.os }}-cnpm-${{ hashFiles('**/package.json') }} - restore-keys: | - ${{ runner.os }}-cnpm- - - - name: Check nodejs toolchain - run: | - if ! command -v cnpm >/dev/null 2>&1; then - npm install -g rimraf --quiet --no-progress - npm install -g cnpm@9.2.0 --quiet --no-progress - fi - node -v && npm -v && cnpm -v - - name: Set up go toolchain uses: actions/setup-go@v5 with: @@ -79,7 +56,7 @@ jobs: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}- ${{ runner.os }}-build- - - name: Test gateway code + - name: Unit test run: | echo Home path is $HOME export WORKBASE=$HOME/go/src/infini.sh From b7f73e02af9aac59d277c91ca191338cc6454e30 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 10:56:21 +0800 Subject: [PATCH 06/11] test: remove no use step --- .github/workflows/unit_test.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index d303978d..49f6a49a 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -46,16 +46,6 @@ jobs: - name: Check go toolchain run: go version - - name: Cache Build Output - uses: actions/cache@v4 - with: - path: | - .public - key: ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-build-${{ hashFiles('**/package.json') }}- - ${{ runner.os }}-build- - - name: Unit test run: | echo Home path is $HOME From 07a61c5a88a82a6287e89debe074f4aeacf9f895 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 11:00:06 +0800 Subject: [PATCH 07/11] test: remove some blank line --- .github/workflows/unit_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 49f6a49a..929b5c26 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -27,7 +27,6 @@ jobs: repository: infinilabs/framework path: framework - - name: Checkout framework-vendor uses: actions/checkout@v4 with: From 9d0c330a4d32ed87e484d5a960d287a45a888a66 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 15:30:35 +0800 Subject: [PATCH 08/11] test: fix ldap test --- proxy/filters/security/ldap/ldap_test.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/proxy/filters/security/ldap/ldap_test.go b/proxy/filters/security/ldap/ldap_test.go index 7b2a5830..06e16522 100644 --- a/proxy/filters/security/ldap/ldap_test.go +++ b/proxy/filters/security/ldap/ldap_test.go @@ -25,17 +25,15 @@ package ldap import ( "context" - "fmt" "testing" + "github.com/stretchr/testify/assert" + "infini.sh/framework/lib/fasthttp" "infini.sh/framework/lib/guardian/auth/strategies/ldap" ) func TestLDAPFunctions(t *testing.T) { - - t.Skip() - cfg := ldap.Config{ BaseDN: "dc=example,dc=com", BindDN: "cn=read-only-admin,dc=example,dc=com", @@ -48,12 +46,15 @@ func TestLDAPFunctions(t *testing.T) { r := &fasthttp.Request{} r.SetBasicAuth("galieleo", "password") - user, err := ldap.New(&cfg).Authenticate(context.Background(), r) + ldapClient := ldap.New(&cfg) + user, err := ldapClient.Authenticate(context.Background(), r) - fmt.Println(err) - fmt.Println(user.GetUserName()) - fmt.Println(user.GetID()) - fmt.Println(user.GetGroups()) - fmt.Println(user.GetExtensions()) + if err != nil { + t.Fatalf("failed to authenticate: %v", err) + } + assert.Equal(t, "galieleo", user.GetUserName(), "unexpected username") + assert.Equal(t, "expected-id", user.GetID(), "unexpected user ID") + assert.Equal(t, []string{"expected-group"}, user.GetGroups(), "unexpected user groups") + assert.Equal(t, map[string]string{"key": "value"}, user.GetExtensions(), "unexpected user extensions") } From 13b2e82e9eac917033973c7d4368722cc49ff2ef Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 15:49:01 +0800 Subject: [PATCH 09/11] test: add env flag --- .github/workflows/unit_test.yml | 4 +++- config/generated.go | 10 +++++----- proxy/filters/security/ldap/ldap_test.go | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 929b5c26..7bea004b 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -46,6 +46,8 @@ jobs: run: go version - name: Unit test + env: + GOFLAGS: -tags=ci run: | echo Home path is $HOME export WORKBASE=$HOME/go/src/infini.sh @@ -62,4 +64,4 @@ jobs: # for unit test cd $WORK echo Testing code at $PWD ... - make config test \ No newline at end of file + make test \ No newline at end of file diff --git a/config/generated.go b/config/generated.go index f52e02d3..5ec7f41f 100755 --- a/config/generated.go +++ b/config/generated.go @@ -1,10 +1,10 @@ package config -const LastCommitLog = "N/A" -const BuildDate = "N/A" +const LastCommitLog = "9d0c330a4d32ed87e484d5a960d287a45a888a66" +const BuildDate = "2025-01-06T07:45:20Z" -const EOLDate = "N/A" +const EOLDate = "2025-12-31T10:10:10Z" -const Version = "0.0.1-SNAPSHOT" +const Version = "1.0.0_SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" diff --git a/proxy/filters/security/ldap/ldap_test.go b/proxy/filters/security/ldap/ldap_test.go index 06e16522..3fa878e7 100644 --- a/proxy/filters/security/ldap/ldap_test.go +++ b/proxy/filters/security/ldap/ldap_test.go @@ -21,6 +21,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +//go:build !ci + package ldap import ( From 0f1c34d83c7129652db1a72ce2cb66087ac48dfb Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 6 Jan 2025 16:25:41 +0800 Subject: [PATCH 10/11] test: restore-generated-file --- config/generated.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/generated.go b/config/generated.go index 5ec7f41f..f52e02d3 100755 --- a/config/generated.go +++ b/config/generated.go @@ -1,10 +1,10 @@ package config -const LastCommitLog = "9d0c330a4d32ed87e484d5a960d287a45a888a66" -const BuildDate = "2025-01-06T07:45:20Z" +const LastCommitLog = "N/A" +const BuildDate = "N/A" -const EOLDate = "2025-12-31T10:10:10Z" +const EOLDate = "N/A" -const Version = "1.0.0_SNAPSHOT" +const Version = "0.0.1-SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" From 0359ee235611f15248ec911771bc59a6a68fcb1b Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 10 Jan 2025 14:41:51 +0800 Subject: [PATCH 11/11] chore: add new line after end for Println --- pipeline/index_diff/index_diff.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pipeline/index_diff/index_diff.go b/pipeline/index_diff/index_diff.go index db85ae0d..ad63335f 100644 --- a/pipeline/index_diff/index_diff.go +++ b/pipeline/index_diff/index_diff.go @@ -540,6 +540,7 @@ func (processor *IndexDiffProcessor) generateTextReport() { if leftBuilder.Len() == 0 && rightBuilder.Len() == 0 && bothBuilder.Len() == 0 { fmt.Println("Congratulations, the two clusters are consistent.") + fmt.Println() } }