Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: RLS #1162

Merged
merged 3 commits into from
Dec 26, 2024
Merged

feat: RLS #1162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9d96c71
feat: row level security
adityathebe Oct 28, 2024
082e4e1
feat: benchmarks
adityathebe Dec 23, 2024
b77320c
feat: improve rls performance
adityathebe Dec 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ ginkgo:
test: ginkgo
ginkgo -r -v

.PHONY: bench
bench:
go test -bench=. -benchtime=10s -timeout 30m github.com/flanksource/duty/bench

fmt:
go fmt ./...

Expand Down
1 change: 1 addition & 0 deletions api/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type Config struct {
LogLevel string
LogName string

EnableRLS bool // Enable Row-level security
RunMigrations bool
SkipMigrations bool
SkipMigrationFiles []string
Expand Down
129 changes: 129 additions & 0 deletions bench/bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
package bench_test

import (
"fmt"
"os"
"testing"

"github.com/flanksource/commons/logger"
"github.com/flanksource/duty/context"
"github.com/flanksource/duty/shutdown"
"github.com/flanksource/duty/tests/setup"
)

type DistinctBenchConfig struct {
// view/table name
relation string

// optional column to fetch.
// when left empty all columns are fetched (this is left empty for views with single column)
column string
}

var benchConfigs = []DistinctBenchConfig{
{"catalog_changes", "change_type"},
{"config_changes", "change_type"},
{"config_detail", "type"},
{"config_names", "type"},
{"config_summary", "type"},
{"configs", "type"},

// These are single column views
{"analysis_types", ""},
{"analyzer_types", ""},
{"change_types", ""},
{"config_classes", ""},
{"config_types", ""},
}

var (
testCtx context.Context
connUrl string

// number of total configs in the database
testSizes = []int{10_000, 25_000, 50_000, 100_000}
)

func setupTestDB(dbPath string) error {
logger.Infof("using %q as the pg data dir", dbPath)
os.Setenv(setup.DUTY_DB_DATA_DIR, dbPath)

shutdown.AddHookWithPriority("delete data dir", shutdown.PriorityCritical+1, func() {
if err := os.RemoveAll(dbPath); err != nil {
logger.Errorf("failed to delete data dir: %v", err)
}
})

var err error
testCtx, err = setup.SetupDB("test",
setup.WithoutDummyData, // we generate the dummy data
setup.WithoutRLS, // start without RLS
)
if err != nil {
return fmt.Errorf("failed to setup db: %v", err)
}
connUrl = testCtx.Value("db_url").(string)
return nil
}

func TestMain(m *testing.M) {
shutdown.WaitForSignal()

// Create a fixed postgres data directory
dbDataPath, err := os.CreateTemp("", "bench-pg-dir-*")
if err != nil {
shutdown.ShutdownAndExit(1, "failed to create temp dir for db")
}

if err := setupTestDB(dbDataPath.Name()); err != nil {
shutdown.ShutdownAndExit(1, err.Error())
}

result := m.Run()
shutdown.ShutdownAndExit(result, "exiting ...")
}

func BenchmarkMain(b *testing.B) {
for _, size := range testSizes {
resetPG(b, false)
_, err := setupConfigsForSize(testCtx, size)
if err != nil {
b.Fatalf("failed to setup configs for size %d: %v", size, err)
}

b.Run(fmt.Sprintf("Sample-%d", size), func(b *testing.B) {
for _, config := range benchConfigs {
runBenchmark(b, config)
}
})
}
}

func runBenchmark(b *testing.B, config DistinctBenchConfig) {
b.Run(config.relation, func(b *testing.B) {
for _, rls := range []bool{false, true} {
resetPG(b, rls)
name := "Without RLS"
if rls {
name = "With RLS"
}

b.Run(name, func(b *testing.B) {
for i := 0; i < b.N; i++ {
if rls {
b.StopTimer()
tags := sampleTags[i%len(sampleTags)]
if err := setupRLSPayload(testCtx, tags); err != nil {
b.Fatalf("failed to setup rls payload with tag(%v): %v", tags, err)
}
b.StartTimer()
}

if err := fetchView(testCtx, config.relation, config.column); err != nil {
b.Fatalf("%v", err)
}
}
})
}
})
}
235 changes: 235 additions & 0 deletions bench/benchmark.md
View file
Open in desktop

Large diffs are not rendered by default.

92 changes: 92 additions & 0 deletions bench/new.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
goos: linux
goarch: amd64
pkg: github.com/flanksource/duty/bench
cpu: Intel(R) Core(TM) i9-14900K
BenchmarkMain/Sample-10000/catalog_changes/Without_RLS-32 6649 1620014 ns/op
BenchmarkMain/Sample-10000/catalog_changes/With_RLS-32 3399 3547842 ns/op
BenchmarkMain/Sample-10000/config_changes/Without_RLS-32 7155 1628757 ns/op
BenchmarkMain/Sample-10000/config_changes/With_RLS-32 3273 3569723 ns/op
BenchmarkMain/Sample-10000/config_detail/Without_RLS-32 9540 1220023 ns/op
BenchmarkMain/Sample-10000/config_detail/With_RLS-32 4063 2900145 ns/op
BenchmarkMain/Sample-10000/config_names/Without_RLS-32 1663 7124439 ns/op
BenchmarkMain/Sample-10000/config_names/With_RLS-32 4093 2914901 ns/op
BenchmarkMain/Sample-10000/config_summary/Without_RLS-32 687 17482952 ns/op
BenchmarkMain/Sample-10000/config_summary/With_RLS-32 5929 1908932 ns/op
BenchmarkMain/Sample-10000/configs/Without_RLS-32 5686 2078947 ns/op
BenchmarkMain/Sample-10000/configs/With_RLS-32 4071 2906051 ns/op
BenchmarkMain/Sample-10000/analysis_types/Without_RLS-32 9468 1266709 ns/op
BenchmarkMain/Sample-10000/analysis_types/With_RLS-32 9462 1251793 ns/op
BenchmarkMain/Sample-10000/analyzer_types/Without_RLS-32 9788 1199055 ns/op
BenchmarkMain/Sample-10000/analyzer_types/With_RLS-32 9859 1214105 ns/op
BenchmarkMain/Sample-10000/change_types/Without_RLS-32 7276 1601141 ns/op
BenchmarkMain/Sample-10000/change_types/With_RLS-32 7311 1619463 ns/op
BenchmarkMain/Sample-10000/config_classes/Without_RLS-32 10000 1045498 ns/op
BenchmarkMain/Sample-10000/config_classes/With_RLS-32 4072 2904409 ns/op
BenchmarkMain/Sample-10000/config_types/Without_RLS-32 9136 1223087 ns/op
BenchmarkMain/Sample-10000/config_types/With_RLS-32 4093 2904356 ns/op
BenchmarkMain/Sample-25000/catalog_changes/Without_RLS-32 3142 3764216 ns/op
BenchmarkMain/Sample-25000/catalog_changes/With_RLS-32 1412 8327931 ns/op
BenchmarkMain/Sample-25000/config_changes/Without_RLS-32 3159 3766311 ns/op
BenchmarkMain/Sample-25000/config_changes/With_RLS-32 1400 8388122 ns/op
BenchmarkMain/Sample-25000/config_detail/Without_RLS-32 3972 2967181 ns/op
BenchmarkMain/Sample-25000/config_detail/With_RLS-32 1696 7008540 ns/op
BenchmarkMain/Sample-25000/config_names/Without_RLS-32 709 17180999 ns/op
BenchmarkMain/Sample-25000/config_names/With_RLS-32 1700 6991508 ns/op
BenchmarkMain/Sample-25000/config_summary/Without_RLS-32 264 45680070 ns/op
BenchmarkMain/Sample-25000/config_summary/With_RLS-32 2690 4537575 ns/op
BenchmarkMain/Sample-25000/configs/Without_RLS-32 2382 5012024 ns/op
BenchmarkMain/Sample-25000/configs/With_RLS-32 1699 6932345 ns/op
BenchmarkMain/Sample-25000/analysis_types/Without_RLS-32 3981 2994821 ns/op
BenchmarkMain/Sample-25000/analysis_types/With_RLS-32 4100 2963487 ns/op
BenchmarkMain/Sample-25000/analyzer_types/Without_RLS-32 4102 2872676 ns/op
BenchmarkMain/Sample-25000/analyzer_types/With_RLS-32 4158 2865456 ns/op
BenchmarkMain/Sample-25000/change_types/Without_RLS-32 3058 3953717 ns/op
BenchmarkMain/Sample-25000/change_types/With_RLS-32 3061 3909598 ns/op
BenchmarkMain/Sample-25000/config_classes/Without_RLS-32 4725 2566520 ns/op
BenchmarkMain/Sample-25000/config_classes/With_RLS-32 1682 6972777 ns/op
BenchmarkMain/Sample-25000/config_types/Without_RLS-32 3924 2963325 ns/op
BenchmarkMain/Sample-25000/config_types/With_RLS-32 1708 7065202 ns/op
BenchmarkMain/Sample-50000/catalog_changes/Without_RLS-32 1478 8000063 ns/op
BenchmarkMain/Sample-50000/catalog_changes/With_RLS-32 674 18089184 ns/op
BenchmarkMain/Sample-50000/config_changes/Without_RLS-32 1530 8402061 ns/op
BenchmarkMain/Sample-50000/config_changes/With_RLS-32 669 17876571 ns/op
BenchmarkMain/Sample-50000/config_detail/Without_RLS-32 2131 5745608 ns/op
BenchmarkMain/Sample-50000/config_detail/With_RLS-32 866 13684545 ns/op
BenchmarkMain/Sample-50000/config_names/Without_RLS-32 366 32851181 ns/op
BenchmarkMain/Sample-50000/config_names/With_RLS-32 868 13829836 ns/op
BenchmarkMain/Sample-50000/config_summary/Without_RLS-32 124 94852697 ns/op
BenchmarkMain/Sample-50000/config_summary/With_RLS-32 1329 8875333 ns/op
BenchmarkMain/Sample-50000/configs/Without_RLS-32 1190 9905524 ns/op
BenchmarkMain/Sample-50000/configs/With_RLS-32 870 13808263 ns/op
BenchmarkMain/Sample-50000/analysis_types/Without_RLS-32 1952 6000208 ns/op
BenchmarkMain/Sample-50000/analysis_types/With_RLS-32 2071 5783666 ns/op
BenchmarkMain/Sample-50000/analyzer_types/Without_RLS-32 2136 5642676 ns/op
BenchmarkMain/Sample-50000/analyzer_types/With_RLS-32 2142 5594963 ns/op
BenchmarkMain/Sample-50000/change_types/Without_RLS-32 1528 7845923 ns/op
BenchmarkMain/Sample-50000/change_types/With_RLS-32 1544 7853844 ns/op
BenchmarkMain/Sample-50000/config_classes/Without_RLS-32 2418 4906577 ns/op
BenchmarkMain/Sample-50000/config_classes/With_RLS-32 871 13753221 ns/op
BenchmarkMain/Sample-50000/config_types/Without_RLS-32 2029 5746793 ns/op
BenchmarkMain/Sample-50000/config_types/With_RLS-32 867 13773012 ns/op
BenchmarkMain/Sample-100000/catalog_changes/Without_RLS-32 640 16708249 ns/op
BenchmarkMain/Sample-100000/catalog_changes/With_RLS-32 309 37262982 ns/op
BenchmarkMain/Sample-100000/config_changes/Without_RLS-32 637 16886999 ns/op
BenchmarkMain/Sample-100000/config_changes/With_RLS-32 319 36727055 ns/op
BenchmarkMain/Sample-100000/config_detail/Without_RLS-32 1014 11893316 ns/op
BenchmarkMain/Sample-100000/config_detail/With_RLS-32 426 28133108 ns/op
BenchmarkMain/Sample-100000/config_names/Without_RLS-32 169 71342338 ns/op
BenchmarkMain/Sample-100000/config_names/With_RLS-32 428 28080877 ns/op
BenchmarkMain/Sample-100000/config_summary/Without_RLS-32 67 170440224 ns/op
BenchmarkMain/Sample-100000/config_summary/With_RLS-32 652 18252059 ns/op
BenchmarkMain/Sample-100000/configs/Without_RLS-32 573 20778969 ns/op
BenchmarkMain/Sample-100000/configs/With_RLS-32 423 28208192 ns/op
BenchmarkMain/Sample-100000/analysis_types/Without_RLS-32 974 12216983 ns/op
BenchmarkMain/Sample-100000/analysis_types/With_RLS-32 1047 11827838 ns/op
BenchmarkMain/Sample-100000/analyzer_types/Without_RLS-32 1076 11213405 ns/op
BenchmarkMain/Sample-100000/analyzer_types/With_RLS-32 1057 11392111 ns/op
BenchmarkMain/Sample-100000/change_types/Without_RLS-32 639 17009622 ns/op
BenchmarkMain/Sample-100000/change_types/With_RLS-32 627 16996126 ns/op
BenchmarkMain/Sample-100000/config_classes/Without_RLS-32 1158 9950993 ns/op
BenchmarkMain/Sample-100000/config_classes/With_RLS-32 433 27732173 ns/op
BenchmarkMain/Sample-100000/config_types/Without_RLS-32 990 11939862 ns/op
BenchmarkMain/Sample-100000/config_types/With_RLS-32 434 27360176 ns/op
Loading
Loading