Skip to content

Commit

Permalink
feat: improve rls performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe committed Dec 26, 2024
1 parent 082e4e1 commit b77320c
Show file tree
Hide file tree
Showing 6 changed files with 409 additions and 105 deletions.
310 changes: 217 additions & 93 deletions bench/benchmark.md
View file

Large diffs are not rendered by default.

92 changes: 92 additions & 0 deletions bench/new.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
goos: linux
goarch: amd64
pkg: github.com/flanksource/duty/bench
cpu: Intel(R) Core(TM) i9-14900K
BenchmarkMain/Sample-10000/catalog_changes/Without_RLS-32 6649 1620014 ns/op
BenchmarkMain/Sample-10000/catalog_changes/With_RLS-32 3399 3547842 ns/op
BenchmarkMain/Sample-10000/config_changes/Without_RLS-32 7155 1628757 ns/op
BenchmarkMain/Sample-10000/config_changes/With_RLS-32 3273 3569723 ns/op
BenchmarkMain/Sample-10000/config_detail/Without_RLS-32 9540 1220023 ns/op
BenchmarkMain/Sample-10000/config_detail/With_RLS-32 4063 2900145 ns/op
BenchmarkMain/Sample-10000/config_names/Without_RLS-32 1663 7124439 ns/op
BenchmarkMain/Sample-10000/config_names/With_RLS-32 4093 2914901 ns/op
BenchmarkMain/Sample-10000/config_summary/Without_RLS-32 687 17482952 ns/op
BenchmarkMain/Sample-10000/config_summary/With_RLS-32 5929 1908932 ns/op
BenchmarkMain/Sample-10000/configs/Without_RLS-32 5686 2078947 ns/op
BenchmarkMain/Sample-10000/configs/With_RLS-32 4071 2906051 ns/op
BenchmarkMain/Sample-10000/analysis_types/Without_RLS-32 9468 1266709 ns/op
BenchmarkMain/Sample-10000/analysis_types/With_RLS-32 9462 1251793 ns/op
BenchmarkMain/Sample-10000/analyzer_types/Without_RLS-32 9788 1199055 ns/op
BenchmarkMain/Sample-10000/analyzer_types/With_RLS-32 9859 1214105 ns/op
BenchmarkMain/Sample-10000/change_types/Without_RLS-32 7276 1601141 ns/op
BenchmarkMain/Sample-10000/change_types/With_RLS-32 7311 1619463 ns/op
BenchmarkMain/Sample-10000/config_classes/Without_RLS-32 10000 1045498 ns/op
BenchmarkMain/Sample-10000/config_classes/With_RLS-32 4072 2904409 ns/op
BenchmarkMain/Sample-10000/config_types/Without_RLS-32 9136 1223087 ns/op
BenchmarkMain/Sample-10000/config_types/With_RLS-32 4093 2904356 ns/op
BenchmarkMain/Sample-25000/catalog_changes/Without_RLS-32 3142 3764216 ns/op
BenchmarkMain/Sample-25000/catalog_changes/With_RLS-32 1412 8327931 ns/op
BenchmarkMain/Sample-25000/config_changes/Without_RLS-32 3159 3766311 ns/op
BenchmarkMain/Sample-25000/config_changes/With_RLS-32 1400 8388122 ns/op
BenchmarkMain/Sample-25000/config_detail/Without_RLS-32 3972 2967181 ns/op
BenchmarkMain/Sample-25000/config_detail/With_RLS-32 1696 7008540 ns/op
BenchmarkMain/Sample-25000/config_names/Without_RLS-32 709 17180999 ns/op
BenchmarkMain/Sample-25000/config_names/With_RLS-32 1700 6991508 ns/op
BenchmarkMain/Sample-25000/config_summary/Without_RLS-32 264 45680070 ns/op
BenchmarkMain/Sample-25000/config_summary/With_RLS-32 2690 4537575 ns/op
BenchmarkMain/Sample-25000/configs/Without_RLS-32 2382 5012024 ns/op
BenchmarkMain/Sample-25000/configs/With_RLS-32 1699 6932345 ns/op
BenchmarkMain/Sample-25000/analysis_types/Without_RLS-32 3981 2994821 ns/op
BenchmarkMain/Sample-25000/analysis_types/With_RLS-32 4100 2963487 ns/op
BenchmarkMain/Sample-25000/analyzer_types/Without_RLS-32 4102 2872676 ns/op
BenchmarkMain/Sample-25000/analyzer_types/With_RLS-32 4158 2865456 ns/op
BenchmarkMain/Sample-25000/change_types/Without_RLS-32 3058 3953717 ns/op
BenchmarkMain/Sample-25000/change_types/With_RLS-32 3061 3909598 ns/op
BenchmarkMain/Sample-25000/config_classes/Without_RLS-32 4725 2566520 ns/op
BenchmarkMain/Sample-25000/config_classes/With_RLS-32 1682 6972777 ns/op
BenchmarkMain/Sample-25000/config_types/Without_RLS-32 3924 2963325 ns/op
BenchmarkMain/Sample-25000/config_types/With_RLS-32 1708 7065202 ns/op
BenchmarkMain/Sample-50000/catalog_changes/Without_RLS-32 1478 8000063 ns/op
BenchmarkMain/Sample-50000/catalog_changes/With_RLS-32 674 18089184 ns/op
BenchmarkMain/Sample-50000/config_changes/Without_RLS-32 1530 8402061 ns/op
BenchmarkMain/Sample-50000/config_changes/With_RLS-32 669 17876571 ns/op
BenchmarkMain/Sample-50000/config_detail/Without_RLS-32 2131 5745608 ns/op
BenchmarkMain/Sample-50000/config_detail/With_RLS-32 866 13684545 ns/op
BenchmarkMain/Sample-50000/config_names/Without_RLS-32 366 32851181 ns/op
BenchmarkMain/Sample-50000/config_names/With_RLS-32 868 13829836 ns/op
BenchmarkMain/Sample-50000/config_summary/Without_RLS-32 124 94852697 ns/op
BenchmarkMain/Sample-50000/config_summary/With_RLS-32 1329 8875333 ns/op
BenchmarkMain/Sample-50000/configs/Without_RLS-32 1190 9905524 ns/op
BenchmarkMain/Sample-50000/configs/With_RLS-32 870 13808263 ns/op
BenchmarkMain/Sample-50000/analysis_types/Without_RLS-32 1952 6000208 ns/op
BenchmarkMain/Sample-50000/analysis_types/With_RLS-32 2071 5783666 ns/op
BenchmarkMain/Sample-50000/analyzer_types/Without_RLS-32 2136 5642676 ns/op
BenchmarkMain/Sample-50000/analyzer_types/With_RLS-32 2142 5594963 ns/op
BenchmarkMain/Sample-50000/change_types/Without_RLS-32 1528 7845923 ns/op
BenchmarkMain/Sample-50000/change_types/With_RLS-32 1544 7853844 ns/op
BenchmarkMain/Sample-50000/config_classes/Without_RLS-32 2418 4906577 ns/op
BenchmarkMain/Sample-50000/config_classes/With_RLS-32 871 13753221 ns/op
BenchmarkMain/Sample-50000/config_types/Without_RLS-32 2029 5746793 ns/op
BenchmarkMain/Sample-50000/config_types/With_RLS-32 867 13773012 ns/op
BenchmarkMain/Sample-100000/catalog_changes/Without_RLS-32 640 16708249 ns/op
BenchmarkMain/Sample-100000/catalog_changes/With_RLS-32 309 37262982 ns/op
BenchmarkMain/Sample-100000/config_changes/Without_RLS-32 637 16886999 ns/op
BenchmarkMain/Sample-100000/config_changes/With_RLS-32 319 36727055 ns/op
BenchmarkMain/Sample-100000/config_detail/Without_RLS-32 1014 11893316 ns/op
BenchmarkMain/Sample-100000/config_detail/With_RLS-32 426 28133108 ns/op
BenchmarkMain/Sample-100000/config_names/Without_RLS-32 169 71342338 ns/op
BenchmarkMain/Sample-100000/config_names/With_RLS-32 428 28080877 ns/op
BenchmarkMain/Sample-100000/config_summary/Without_RLS-32 67 170440224 ns/op
BenchmarkMain/Sample-100000/config_summary/With_RLS-32 652 18252059 ns/op
BenchmarkMain/Sample-100000/configs/Without_RLS-32 573 20778969 ns/op
BenchmarkMain/Sample-100000/configs/With_RLS-32 423 28208192 ns/op
BenchmarkMain/Sample-100000/analysis_types/Without_RLS-32 974 12216983 ns/op
BenchmarkMain/Sample-100000/analysis_types/With_RLS-32 1047 11827838 ns/op
BenchmarkMain/Sample-100000/analyzer_types/Without_RLS-32 1076 11213405 ns/op
BenchmarkMain/Sample-100000/analyzer_types/With_RLS-32 1057 11392111 ns/op
BenchmarkMain/Sample-100000/change_types/Without_RLS-32 639 17009622 ns/op
BenchmarkMain/Sample-100000/change_types/With_RLS-32 627 16996126 ns/op
BenchmarkMain/Sample-100000/config_classes/Without_RLS-32 1158 9950993 ns/op
BenchmarkMain/Sample-100000/config_classes/With_RLS-32 433 27732173 ns/op
BenchmarkMain/Sample-100000/config_types/Without_RLS-32 990 11939862 ns/op
BenchmarkMain/Sample-100000/config_types/With_RLS-32 434 27360176 ns/op
92 changes: 92 additions & 0 deletions bench/old.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
goos: linux
goarch: amd64
pkg: github.com/flanksource/duty/bench
cpu: Intel(R) Core(TM) i9-14900K
BenchmarkMain/Sample-10000/catalog_changes/Without_RLS-32 7210 1618398 ns/op
BenchmarkMain/Sample-10000/catalog_changes/With_RLS-32 468 25696464 ns/op
BenchmarkMain/Sample-10000/config_changes/Without_RLS-32 7225 1662744 ns/op
BenchmarkMain/Sample-10000/config_changes/With_RLS-32 472 25400088 ns/op
BenchmarkMain/Sample-10000/config_detail/Without_RLS-32 9204 1258744 ns/op
BenchmarkMain/Sample-10000/config_detail/With_RLS-32 1110 10922483 ns/op
BenchmarkMain/Sample-10000/config_names/Without_RLS-32 1630 7167192 ns/op
BenchmarkMain/Sample-10000/config_names/With_RLS-32 1068 11294074 ns/op
BenchmarkMain/Sample-10000/config_summary/Without_RLS-32 691 17307363 ns/op
BenchmarkMain/Sample-10000/config_summary/With_RLS-32 134 88561638 ns/op
BenchmarkMain/Sample-10000/configs/Without_RLS-32 5647 2071318 ns/op
BenchmarkMain/Sample-10000/configs/With_RLS-32 1111 10701505 ns/op
BenchmarkMain/Sample-10000/analysis_types/Without_RLS-32 9230 1267578 ns/op
BenchmarkMain/Sample-10000/analysis_types/With_RLS-32 9554 1249278 ns/op
BenchmarkMain/Sample-10000/analyzer_types/Without_RLS-32 9843 1186921 ns/op
BenchmarkMain/Sample-10000/analyzer_types/With_RLS-32 9657 1208368 ns/op
BenchmarkMain/Sample-10000/change_types/Without_RLS-32 7399 1602853 ns/op
BenchmarkMain/Sample-10000/change_types/With_RLS-32 7578 1618275 ns/op
BenchmarkMain/Sample-10000/config_classes/Without_RLS-32 9602 1053916 ns/op
BenchmarkMain/Sample-10000/config_classes/With_RLS-32 1102 10601675 ns/op
BenchmarkMain/Sample-10000/config_types/Without_RLS-32 9938 1220556 ns/op
BenchmarkMain/Sample-10000/config_types/With_RLS-32 1132 10687988 ns/op
BenchmarkMain/Sample-25000/catalog_changes/Without_RLS-32 3189 3777448 ns/op
BenchmarkMain/Sample-25000/catalog_changes/With_RLS-32 199 59728301 ns/op
BenchmarkMain/Sample-25000/config_changes/Without_RLS-32 3106 3796288 ns/op
BenchmarkMain/Sample-25000/config_changes/With_RLS-32 202 59201120 ns/op
BenchmarkMain/Sample-25000/config_detail/Without_RLS-32 3986 2825246 ns/op
BenchmarkMain/Sample-25000/config_detail/With_RLS-32 472 25408187 ns/op
BenchmarkMain/Sample-25000/config_names/Without_RLS-32 712 16344633 ns/op
BenchmarkMain/Sample-25000/config_names/With_RLS-32 447 26696849 ns/op
BenchmarkMain/Sample-25000/config_summary/Without_RLS-32 274 43747108 ns/op
BenchmarkMain/Sample-25000/config_summary/With_RLS-32 45 242723303 ns/op
BenchmarkMain/Sample-25000/configs/Without_RLS-32 2466 4885648 ns/op
BenchmarkMain/Sample-25000/configs/With_RLS-32 470 25306394 ns/op
BenchmarkMain/Sample-25000/analysis_types/Without_RLS-32 4042 2932464 ns/op
BenchmarkMain/Sample-25000/analysis_types/With_RLS-32 4096 2936163 ns/op
BenchmarkMain/Sample-25000/analyzer_types/Without_RLS-32 4293 2776851 ns/op
BenchmarkMain/Sample-25000/analyzer_types/With_RLS-32 4180 2812037 ns/op
BenchmarkMain/Sample-25000/change_types/Without_RLS-32 3093 3864532 ns/op
BenchmarkMain/Sample-25000/change_types/With_RLS-32 3123 3806187 ns/op
BenchmarkMain/Sample-25000/config_classes/Without_RLS-32 4693 2435089 ns/op
BenchmarkMain/Sample-25000/config_classes/With_RLS-32 476 25211551 ns/op
BenchmarkMain/Sample-25000/config_types/Without_RLS-32 4164 2861676 ns/op
BenchmarkMain/Sample-25000/config_types/With_RLS-32 476 25352067 ns/op
BenchmarkMain/Sample-50000/catalog_changes/Without_RLS-32 1560 7545395 ns/op
BenchmarkMain/Sample-50000/catalog_changes/With_RLS-32 100 117274979 ns/op
BenchmarkMain/Sample-50000/config_changes/Without_RLS-32 1573 7551748 ns/op
BenchmarkMain/Sample-50000/config_changes/With_RLS-32 99 117770448 ns/op
BenchmarkMain/Sample-50000/config_detail/Without_RLS-32 2101 5593338 ns/op
BenchmarkMain/Sample-50000/config_detail/With_RLS-32 242 49418844 ns/op
BenchmarkMain/Sample-50000/config_names/Without_RLS-32 378 31770900 ns/op
BenchmarkMain/Sample-50000/config_names/With_RLS-32 226 52552379 ns/op
BenchmarkMain/Sample-50000/config_summary/Without_RLS-32 128 90894472 ns/op
BenchmarkMain/Sample-50000/config_summary/With_RLS-32 25 473002784 ns/op
BenchmarkMain/Sample-50000/configs/Without_RLS-32 1251 9464835 ns/op
BenchmarkMain/Sample-50000/configs/With_RLS-32 238 49838197 ns/op
BenchmarkMain/Sample-50000/analysis_types/Without_RLS-32 2052 5801409 ns/op
BenchmarkMain/Sample-50000/analysis_types/With_RLS-32 2121 5712487 ns/op
BenchmarkMain/Sample-50000/analyzer_types/Without_RLS-32 2216 5442149 ns/op
BenchmarkMain/Sample-50000/analyzer_types/With_RLS-32 2169 5515249 ns/op
BenchmarkMain/Sample-50000/change_types/Without_RLS-32 1592 7552502 ns/op
BenchmarkMain/Sample-50000/change_types/With_RLS-32 1521 7634041 ns/op
BenchmarkMain/Sample-50000/config_classes/Without_RLS-32 2442 4780004 ns/op
BenchmarkMain/Sample-50000/config_classes/With_RLS-32 241 49653432 ns/op
BenchmarkMain/Sample-50000/config_types/Without_RLS-32 2145 5558880 ns/op
BenchmarkMain/Sample-50000/config_types/With_RLS-32 241 49518770 ns/op
BenchmarkMain/Sample-100000/catalog_changes/Without_RLS-32 668 15792969 ns/op
BenchmarkMain/Sample-100000/catalog_changes/With_RLS-32 50 236585972 ns/op
BenchmarkMain/Sample-100000/config_changes/Without_RLS-32 670 15857288 ns/op
BenchmarkMain/Sample-100000/config_changes/With_RLS-32 49 237727030 ns/op
BenchmarkMain/Sample-100000/config_detail/Without_RLS-32 1060 11282955 ns/op
BenchmarkMain/Sample-100000/config_detail/With_RLS-32 121 98802558 ns/op
BenchmarkMain/Sample-100000/config_names/Without_RLS-32 175 68280940 ns/op
BenchmarkMain/Sample-100000/config_names/With_RLS-32 100 105502052 ns/op
BenchmarkMain/Sample-100000/config_summary/Without_RLS-32 67 169628955 ns/op
BenchmarkMain/Sample-100000/config_summary/With_RLS-32 12 984132710 ns/op
BenchmarkMain/Sample-100000/configs/Without_RLS-32 609 19589287 ns/op
BenchmarkMain/Sample-100000/configs/With_RLS-32 120 99833450 ns/op
BenchmarkMain/Sample-100000/analysis_types/Without_RLS-32 1039 11434234 ns/op
BenchmarkMain/Sample-100000/analysis_types/With_RLS-32 1064 11451964 ns/op
BenchmarkMain/Sample-100000/analyzer_types/Without_RLS-32 1110 10675073 ns/op
BenchmarkMain/Sample-100000/analyzer_types/With_RLS-32 1114 10854744 ns/op
BenchmarkMain/Sample-100000/change_types/Without_RLS-32 669 15856671 ns/op
BenchmarkMain/Sample-100000/change_types/With_RLS-32 668 16162332 ns/op
BenchmarkMain/Sample-100000/config_classes/Without_RLS-32 1261 9487116 ns/op
BenchmarkMain/Sample-100000/config_classes/With_RLS-32 121 98950319 ns/op
BenchmarkMain/Sample-100000/config_types/Without_RLS-32 1060 11280585 ns/op
BenchmarkMain/Sample-100000/config_types/With_RLS-32 121 99579524 ns/op
6 changes: 0 additions & 6 deletions bench/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"github.com/flanksource/duty/api"
"github.com/flanksource/duty/context"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/shutdown"
pkgGenerator "github.com/flanksource/duty/tests/generator"
"github.com/flanksource/duty/tests/setup"
)
Expand Down Expand Up @@ -178,9 +177,4 @@ func resetPG(b *testing.B, rlsEnable bool) {
b.Fatalf("failed to enable rls: %v", err)
}
}

// This is required due to a bug in how we handle rls_enable / disable scripts.
if err := testCtx.DB().Exec("DELETE FROM migration_logs").Error; err != nil {
shutdown.ShutdownAndExit(1, fmt.Sprintf("failed to delete migration logs: %v", err))
}
}
2 changes: 2 additions & 0 deletions migrate/migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,10 @@ func RunMigrations(pool *sql.DB, config api.Config) error {

if config.EnableRLS {
config.SkipMigrationFiles = append(config.SkipMigrationFiles, "035_rls_disable.sql")
config.MustRun = append(config.MustRun, "034_rls_enable.sql")
} else {
config.SkipMigrationFiles = append(config.SkipMigrationFiles, "034_rls_enable.sql")
config.MustRun = append(config.MustRun, "035_rls_disable.sql")
}

row := pool.QueryRow("SELECT current_database();")
Expand Down
Loading

0 comments on commit b77320c

Please sign in to comment.