Update purge audit to purge both audit_log_ext and audit_log

Signed-off-by: stonezdj <[email protected]>

src/jobservice/job/impl/purge/purge.go

@@ -21,6 +21,7 @@ import (
 	"github.com/goharbor/harbor/src/common"
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/pkg/audit"
+	"github.com/goharbor/harbor/src/pkg/auditext"
 )
 
 // Job defines the purge job
@@ -29,6 +30,7 @@ type Job struct {
 	includeOperations []string
 	dryRun            bool
 	auditMgr          audit.Manager
+	auditExtMgr       auditext.Manager
 }
 
 // MaxFails is implementation of same method in Interface.
@@ -106,17 +108,26 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error {
 	if j.retentionHour > common.MaxAuditRetentionHour {
 		j.retentionHour = common.MaxAuditRetentionHour
 	}
+	// TODO: Remove the previous purge logic when the audit_log is not used anymore
 	n, err := j.auditMgr.Purge(ormCtx, j.retentionHour, j.includeOperations, j.dryRun)
 	if err != nil {
 		logger.Errorf("failed to purge audit log, error: %v", err)
 		return err
 	}
+	// purge the audit_log_ext table
+	n2, err2 := j.auditExtMgr.Purge(ormCtx, j.retentionHour, j.includeOperations, j.dryRun)
+	if err2 != nil {
+		logger.Errorf("failed to purge audit log ext, error: %v", err2)
+		return err2
+	}
 	logger.Infof("Purge operation parameter, retention_hour=%v, include_operations:%v, dry_run:%v",
 		j.retentionHour, j.includeOperations, j.dryRun)
 	if j.dryRun {
-		logger.Infof("[DRYRUN]Purged %d rows of audit logs", n)
+		logger.Infof("[DRYRUN]Purged %d rows of audit_logs", n)
+		logger.Infof("[DRYRUN]Purged %d rows of audit_log_exts", n2)
 	} else {
-		logger.Infof("Purged %d rows of audit logs", n)
+		logger.Infof("Purged %d rows of audit_logs", n)
+		logger.Infof("Purged %d rows of audit_log_exts", n2)
 	}
 
 	// Successfully exit

src/jobservice/job/impl/purge/purge_test.go

@@ -27,6 +27,7 @@ import (
 	mockjobservice "github.com/goharbor/harbor/src/testing/jobservice"
 	"github.com/goharbor/harbor/src/testing/mock"
 	mockAudit "github.com/goharbor/harbor/src/testing/pkg/audit"
+	mockauditext "github.com/goharbor/harbor/src/testing/pkg/auditext"
 )
 
 type PurgeJobTestSuite struct {
@@ -67,14 +68,16 @@ func (suite *PurgeJobTestSuite) TestRun() {
 	ctx.On("GetLogger").Return(logger)
 	ctx.On("OPCommand").Return(job.NilCommand, true)
 	auditManager := &mockAudit.Manager{}
+	auditExtManager := &mockauditext.Manager{}
 	auditManager.On("Purge", mock.Anything, 128, []string{}, true).Return(int64(100), nil)
-	j := &Job{auditMgr: auditManager}
+	auditExtManager.On("Purge", mock.Anything, 128, []string{}, true).Return(int64(100), nil)
+	j := &Job{auditMgr: auditManager, auditExtMgr: auditExtManager}
 	param := job.Parameters{common.PurgeAuditRetentionHour: 128, common.PurgeAuditDryRun: true}
 	ret := j.Run(ctx, param)
 	suite.Require().Nil(ret)
 
 	auditManager.On("Purge", mock.Anything, 24, []string{}, false).Return(int64(0), fmt.Errorf("failed to connect database"))
-	j2 := &Job{auditMgr: auditManager}
+	j2 := &Job{auditMgr: auditManager, auditExtMgr: auditExtManager}
 	param2 := job.Parameters{common.PurgeAuditRetentionHour: 24, common.PurgeAuditDryRun: false}
 	ret2 := j2.Run(ctx, param2)
 	suite.Require().NotNil(ret2)
@@ -85,7 +88,8 @@ func (suite *PurgeJobTestSuite) TestStop() {
 	ctx.On("GetLogger").Return(logger)
 	ctx.On("OPCommand").Return(job.StopCommand, true)
 	auditManager := &mockAudit.Manager{}
-	j := &Job{auditMgr: auditManager}
+	auditExtManager := &mockauditext.Manager{}
+	j := &Job{auditMgr: auditManager, auditExtMgr: auditExtManager}
 	suite.True(j.shouldStop(ctx))
 }
 

src/pkg/auditext/dao/dao.go

@@ -199,6 +199,8 @@ func permitEventTypes(includeEventTypes []string) []string {
 		event := strings.ToLower(e)
 		if utils.StringInSlice(event, model.EventTypes) {
 			filterEvents = append(filterEvents, e)
+		} else if event == model.OtherEvents { // include all other events
+			filterEvents = append(filterEvents, model.OtherEventTypes...)
 		}
 	}
 	return filterEvents

src/pkg/auditext/dao/dao_test.go

@@ -185,4 +185,9 @@ func TestPermitEventTypes(t *testing.T) {
 		t.Errorf("permitEventTypes failed")
 	}
 
+	// test other event types
+	otherEventTypes := permitEventTypes([]string{"create_artifact", "delete_artifact", "pull_artifact", "other_events"})
+	if len(otherEventTypes) != len(model.EventTypes) {
+		t.Errorf("permitOtherEventTypes failed, it should include all event types")
+	}
 }

src/pkg/auditext/event/basic.go

@@ -46,7 +46,7 @@ type Resolver struct {
 	SucceedCodes   []int
 	// SensitiveAttributes is the attributes that need to be redacted
 	SensitiveAttributes []string
-	// HasResourceName indicates if the resource has name, if true, need to resolve the resource name before delete
+	// ShouldResolveName indicates if the resource name should be resolved before delete, if true, need to resolve the resource name before delete
 	ShouldResolveName bool
 	// IDToNameFunc is used to resolve the resource name from resource id
 	IDToNameFunc ResolveIDToNameFunc

src/pkg/auditext/model/model.go

@@ -20,6 +20,8 @@ import (
 	beego_orm "github.com/beego/beego/v2/client/orm"
 )
 
+const OtherEvents = "other_events"
+
 func init() {
 	beego_orm.RegisterModel(&AuditLogExt{})
 }
@@ -43,7 +45,7 @@ func (a *AuditLogExt) TableName() string {
 	return "audit_log_ext"
 }
 
-// EventTypes defines the types of audit log event
+// EventTypes defines the types of audit log event, new event types should be added at the end of the list
 var EventTypes = []string{
 	"create_artifact",
 	"delete_artifact",
@@ -60,3 +62,6 @@ var EventTypes = []string{
 	"delete_robot",
 	"update_configure",
 }
+
+// OtherEventTypes defines the types of other audit log event types excludes previous EventTypes: create_artifact, delete_artifact, pull_artifact
+var OtherEventTypes = EventTypes[3:]