From b75e6d83dfef61fc739b5903ec27bdddeeb6bec4 Mon Sep 17 00:00:00 2001 From: Moshe Immerman Date: Wed, 20 Dec 2023 21:34:20 +0200 Subject: [PATCH] fix: remove time window heuristics --- go.mod | 3 +- go.sum | 21 +++- pkg/api.go | 31 ++--- pkg/api/api.go | 53 +++----- pkg/cache/cache.go | 6 +- pkg/cache/postgres.go | 12 +- pkg/cache/postgres_query.go | 203 ++++--------------------------- pkg/cache/postgres_query_test.go | 101 --------------- pkg/topology/run_test.go | 8 +- 9 files changed, 88 insertions(+), 350 deletions(-) delete mode 100644 pkg/cache/postgres_query_test.go diff --git a/go.mod b/go.mod index 78a1e5fe4..e2ca1e240 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/fergusstrange/embedded-postgres v1.24.0 github.com/flanksource/artifacts v1.0.0 github.com/flanksource/commons v1.19.2 - github.com/flanksource/duty v1.0.245-0.20231220161640-a0c01f94c2ba + github.com/flanksource/duty v1.0.245 github.com/flanksource/gomplate/v3 v3.20.26 github.com/flanksource/is-healthy v0.0.0-20231003215854-76c51e3a3ff7 github.com/flanksource/kommons v0.31.4 @@ -215,6 +215,7 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/samber/lo v1.39.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect github.com/sethvargo/go-retry v0.2.4 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/go.sum b/go.sum index 5496bb8e1..a9bb5a09a 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybI github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.0.0 h1:yfJe15aSwEQ6Oo6J+gdfdulPNoZ3TEhmbhLIoxZcA+U= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v0.8.0 h1:T028gtTPiYt/RMUfs8nVsAL7FDQrfLlrm/NnRG/zcC4= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.1.0 h1:HCc0+LpPfpCKs6LGGLAhwBARt9632unrVcI6i8s/8os= @@ -626,6 +627,7 @@ github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy86 github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -663,6 +665,8 @@ github.com/antonmedv/expr v1.15.5/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4J github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -739,6 +743,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -825,8 +830,8 @@ github.com/flanksource/artifacts v1.0.0 h1:Fer3firlsI1L0YoOoUOw77w7j4PgtMkNTsQDY github.com/flanksource/artifacts v1.0.0/go.mod h1:KWGcGNGTJe+wb/Pv31thiWsNSdhu7iWqRO/hApLflyA= github.com/flanksource/commons v1.19.2 h1:6JAnKzBBk5iUROrR79dm7Bzbg9beAiFTehkTeZyI2/g= github.com/flanksource/commons v1.19.2/go.mod h1:k+3B7McXUOS+TirYFR9h0pPk6mHNG3dqVUEY9gKI3/U= -github.com/flanksource/duty v1.0.245-0.20231220161640-a0c01f94c2ba h1:4593zVyJ/Si1tXUiAtkNUoDgorTwVAeVS1CGwf6f4hA= -github.com/flanksource/duty v1.0.245-0.20231220161640-a0c01f94c2ba/go.mod h1:tT+MmnSlSJMeBrFGzWDES/vKOW26g+y3i907/IJrd1s= +github.com/flanksource/duty v1.0.245 h1:tfyrbQ//WFxni6lFQKuYNbPgoPvzKOU/0fBh0pXCp2g= +github.com/flanksource/duty v1.0.245/go.mod h1:nhvoArC48DByxrRvvcCKwRfbAfn9wUbo4YWietT9t7I= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.20.26 h1:85lUzlKgZjb1uIkzoa4zN03OcdOnFPG+oWxshZTYqz4= github.com/flanksource/gomplate/v3 v3.20.26/go.mod h1:GKmptFMdr2LbOuqwQZrmo9a/UygyZ0pbXffks8MuYhE= @@ -942,6 +947,7 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -1121,6 +1127,8 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -1130,10 +1138,12 @@ github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf h1:I1sbT4ZbI github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf/go.mod h1:jDHmWDKZY6MIIYltYYfW4Rs7hQ50oS4qf/6spSiZAxY= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce h1:cVkYhlWAxwuS2/Yp6qPtcl0fGpcWxuZNonywHZ6/I+s= github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce/go.mod h1:7TyiGlHI+IO+iJbqRZ82QbFtvgj/AIcFm5qc9DLn7Kc= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-getter v1.7.3 h1:bN2+Fw9XPFvOCjB0UOevFIMICZ7G2XSQHzfvLUyOM5E= github.com/hashicorp/go-getter v1.7.3/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= @@ -1183,6 +1193,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -1434,6 +1445,9 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec= @@ -1447,6 +1461,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= @@ -1526,6 +1541,7 @@ github.com/xhit/go-str2duration v1.2.0/go.mod h1:3cPSlfZlUHVlneIVfePFWcJZsuwf+P1 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= @@ -1545,6 +1561,7 @@ github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zclconf/go-cty v1.14.1 h1:t9fyA35fwjjUMcmL5hLER+e/rEPqrbCK1/OSE4SI9KA= github.com/zclconf/go-cty v1.14.1/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= diff --git a/pkg/api.go b/pkg/api.go index 83c2e3e3e..dc1d58201 100644 --- a/pkg/api.go +++ b/pkg/api.go @@ -115,7 +115,9 @@ type Timeseries struct { Error string `json:"error,omitempty"` Duration int `json:"duration"` // Count is the number of times the check has been run in the specified time window - Count int `json:"count,omitempty"` + Count int `json:"count,omitempty"` + Passed int `json:"passed,omitempty"` + Failed int `json:"failed,omitempty"` } type Canary struct { @@ -138,15 +140,15 @@ func (c Canary) GetCheckID(checkName string) string { } func (c Canary) ToV1() (*v1.Canary, error) { + annotations := c.Annotations + annotations["source"] = c.Source canary := v1.Canary{ ObjectMeta: metav1.ObjectMeta{ - Name: c.Name, - Namespace: c.Namespace, - Annotations: map[string]string{ - "source": c.Source, - }, - Labels: c.Labels, - UID: k8stypes.UID(c.ID.String()), + Name: c.Name, + Namespace: c.Namespace, + Annotations: annotations, + Labels: c.Labels, + UID: k8stypes.UID(c.ID.String()), }, } var deletionTimestamp metav1.Time @@ -181,12 +183,13 @@ func CanaryFromV1(canary v1.Canary) (Canary, error) { checks = canary.Status.Checks } return Canary{ - Spec: spec, - Labels: types.JSONStringMap(canary.Labels), - Name: canary.Name, - Namespace: canary.Namespace, - Source: canary.Annotations["source"], - Checks: types.JSONStringMap(checks), + Spec: spec, + Labels: types.JSONStringMap(canary.Labels), + Annotations: types.JSONStringMap(canary.Annotations), + Name: canary.Name, + Namespace: canary.Namespace, + Source: canary.Annotations["source"], + Checks: types.JSONStringMap(checks), }, nil } diff --git a/pkg/api/api.go b/pkg/api/api.go index 7004e4731..3662a7e0e 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -8,6 +8,7 @@ import ( "github.com/flanksource/canary-checker/pkg/runner" "github.com/flanksource/duty/context" "github.com/flanksource/duty/models" + "github.com/flanksource/duty/query" "github.com/labstack/echo/v4" "github.com/flanksource/canary-checker/pkg" @@ -66,21 +67,21 @@ func CheckDetails(c echo.Context) error { start := time.Now() - summary, err := cache.PostgresCache.Query(*q) - if err != nil { - return errorResonse(c, err, http.StatusInternalServerError) - } - if len(summary) == 0 { - return c.JSON(http.StatusOK, DetailResponse{}) - } - - checkSummary := summary[0] - totalChecks := checkSummary.TotalRuns + end := q.GetEndTime() + since := q.GetStartTime() + timeRange := end.Sub(*since) - rangeDuration := checkSummary.LatestRuntime.Sub(*checkSummary.EarliestRuntime) - q.WindowDuration = getBestPartitioner(totalChecks, rangeDuration) + if timeRange <= time.Hour*2 { + q.WindowDuration = time.Minute + } else if timeRange >= time.Hour*24 { + q.WindowDuration = time.Minute * 15 + } else if timeRange >= time.Hour*24*7 { + q.WindowDuration = time.Minute * 60 + } else { + q.WindowDuration = time.Hour * 4 + } - results, err := cache.PostgresCache.QueryStatus(c.Request().Context(), *q) + results, uptime, latency, err := cache.PostgresCache.QueryStatus(c.Request().Context(), *q) if err != nil { return errorResonse(c, err, http.StatusInternalServerError) } @@ -89,33 +90,13 @@ func CheckDetails(c echo.Context) error { RunnerName: runner.RunnerName, Status: results, Duration: int(time.Since(start).Milliseconds()), - Latency: checkSummary.Latency, - Uptime: checkSummary.Uptime, + Latency: latency, + Uptime: uptime, } return c.JSON(http.StatusOK, apiResponse) } -func CheckSummary(c echo.Context) error { - q, err := cache.ParseQuery(c) - if err != nil { - return errorResonse(c, err, http.StatusBadRequest) - } - - start := time.Now() - results, err := cache.PostgresCache.Query(*q) - if err != nil { - return errorResonse(c, err, http.StatusInternalServerError) - } - - apiResponse := &Response{ - RunnerName: runner.RunnerName, - Checks: results, - Duration: int(time.Since(start).Milliseconds()), - } - return c.JSON(http.StatusOK, apiResponse) -} - func HealthSummary(c echo.Context) error { ctx := c.Request().Context().(context.Context) @@ -125,7 +106,7 @@ func HealthSummary(c echo.Context) error { } start := time.Now() - results, err := cache.PostgresCache.QuerySummary(ctx, queryOpt) + results, err := query.CheckSummary(ctx, query.CheckSummaryOptions(queryOpt)) if err != nil { return errorResonse(c, err, http.StatusInternalServerError) } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 699e7de16..755d63f09 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -9,6 +9,7 @@ import ( "github.com/flanksource/canary-checker/pkg" "github.com/labstack/echo/v4" "github.com/pkg/errors" + "github.com/samber/lo" ) var DefaultCacheCount int @@ -57,9 +58,12 @@ func (q QueryParams) GetStartTime() *time.Time { } func (q QueryParams) GetEndTime() *time.Time { - if q._end != nil || q.End == "" { + if q._end != nil { return q._end } + if q.End == "" { + q._end = lo.ToPtr(time.Now()) + } q._end, _ = timeV(q.End) return q._end } diff --git a/pkg/cache/postgres.go b/pkg/cache/postgres.go index 1610a8b1c..cd16b84b1 100644 --- a/pkg/cache/postgres.go +++ b/pkg/cache/postgres.go @@ -8,8 +8,6 @@ import ( "github.com/flanksource/canary-checker/pkg" "github.com/flanksource/canary-checker/pkg/db" "github.com/flanksource/commons/logger" - "github.com/flanksource/duty/context" - "github.com/flanksource/duty/models" "github.com/flanksource/duty/query" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgxpool" @@ -112,15 +110,7 @@ func (c *postgresCache) AddCheckStatus(check pkg.Check, status pkg.CheckStatus) } } -func (c *postgresCache) Query(q QueryParams) (pkg.Checks, error) { - return q.ExecuteSummary(db.Pool) -} - -func (c *postgresCache) QuerySummary(ctx context.Context, opt SummaryOptions) ([]models.CheckSummary, error) { - return query.CheckSummary(ctx, query.CheckSummaryOptions(opt)) -} - -func (c *postgresCache) QueryStatus(ctx gocontext.Context, q QueryParams) ([]pkg.Timeseries, error) { +func (c *postgresCache) QueryStatus(ctx gocontext.Context, q QueryParams) ([]pkg.Timeseries, pkg.Uptime, pkg.Latency, error) { return q.ExecuteDetails(ctx, db.Pool) } diff --git a/pkg/cache/postgres_query.go b/pkg/cache/postgres_query.go index dbf31acb4..6a7c2fdc8 100644 --- a/pkg/cache/postgres_query.go +++ b/pkg/cache/postgres_query.go @@ -6,10 +6,9 @@ import ( "strings" "time" + "github.com/asecurityteam/rolling" "github.com/flanksource/canary-checker/pkg" "github.com/flanksource/commons/duration" - "github.com/flanksource/commons/logger" - "github.com/google/uuid" "github.com/jackc/pgx/v5" ) @@ -79,7 +78,7 @@ func (q QueryParams) GetWhereClause() (string, map[string]interface{}, error) { return strings.TrimSpace(clause), args, nil } -func (q QueryParams) ExecuteDetails(ctx context.Context, db Querier) ([]pkg.Timeseries, error) { +func (q QueryParams) ExecuteDetails(ctx context.Context, db Querier) ([]pkg.Timeseries, pkg.Uptime, pkg.Latency, error) { start := q.GetStartTime().Format(time.RFC3339) end := q.GetEndTime().Format(time.RFC3339) @@ -88,6 +87,8 @@ With grouped_by_window AS ( SELECT duration, status, + CASE WHEN check_statuses.status = TRUE THEN 1 ELSE 0 END AS passed, + CASE WHEN check_statuses.status = FALSE THEN 1 ELSE 0 END AS failed, to_timestamp(floor((extract(epoch FROM time) + $1) / $2) * $2) AS time FROM check_statuses WHERE @@ -95,11 +96,13 @@ With grouped_by_window AS ( time <= $4 AND check_id = $5 ) -SELECT +SELECT time, bool_and(status), - AVG(duration)::integer as duration -FROM + AVG(duration)::integer as duration, + sum(passed) as passed, + sum(failed) as failed +FROM grouped_by_window GROUP BY time ORDER BY time @@ -107,13 +110,19 @@ ORDER BY time args := []any{q.WindowDuration.Seconds() / 2, q.WindowDuration.Seconds(), start, end, q.Check} if q.WindowDuration == 0 { - query = `SELECT time, status, duration FROM check_statuses WHERE time >= $1 AND time <= $2 AND check_id = $3` + // FIXME + query = `SELECT time, status, duration, + CASE WHEN check_statuses.status = TRUE THEN 1 ELSE 0 END AS passed, + CASE WHEN check_statuses.status = FALSE THEN 1 ELSE 0 END AS failed + FROM check_statuses WHERE time >= $1 AND time <= $2 AND check_id = $3` args = []any{start, end, q.Check} } + uptime := pkg.Uptime{} + latency := rolling.NewPointPolicy(rolling.NewWindow(100)) rows, err := db.Query(ctx, query, args...) if err != nil { - return nil, err + return nil, uptime, pkg.Latency{}, err } defer rows.Close() @@ -121,181 +130,15 @@ ORDER BY time for rows.Next() { var datapoint pkg.Timeseries var ts time.Time - if err := rows.Scan(&ts, &datapoint.Status, &datapoint.Duration); err != nil { - return nil, err + if err := rows.Scan(&ts, &datapoint.Status, &datapoint.Duration, &datapoint.Passed, &datapoint.Failed); err != nil { + return nil, uptime, pkg.Latency{}, err } - + uptime.Failed += datapoint.Failed + uptime.Passed += datapoint.Passed + latency.Append(float64(datapoint.Duration)) datapoint.Time = ts.Format(time.RFC3339) results = append(results, datapoint) } - return results, nil -} - -func exec(db Querier, q QueryParams, sql string, namedArgs map[string]interface{}) (pgx.Rows, error) { - if q.Trace { - sqlDebug := ConvertNamedParamsDebug(sql, namedArgs) - logger.Tracef(sqlDebug) - } - - positionalSQL, args := ConvertNamedParams(sql, namedArgs) - - rows, err := db.Query(context.Background(), positionalSQL, args...) - - if err != nil { - logger.Debugf("Error executing query: %v\n%s\n args=%v", err, positionalSQL, args) - } - return rows, err -} - -func (q QueryParams) ExecuteSummary(db Querier) (pkg.Checks, error) { - clause, namedArgs, err := q.GetWhereClause() - if err != nil { - return nil, err - } - var checkClause string - if q.CanaryID != "" { - checkClause += " AND checks.canary_id = :canary_id " - namedArgs["canary_id"] = q.CanaryID - } - if _, exists := namedArgs["check_key"]; exists { - checkClause += " AND checks.id = :check_key " - } - - statusColumns := "" - if q.IncludeMessages { - statusColumns += ", 'message', message, 'error', error" - } - sql := fmt.Sprintf(` -WITH filtered_check_status AS ( - SELECT * FROM check_statuses - WHERE %s -) -SELECT - checks.id::text, - canary_id::text, - stats.passed, - stats.failed, - stats.p99, stats.p97, stats.p95, - statii, - type, - checks.icon, - checks.name, - checks.description, - canaries.namespace, - canaries.name as canaryName, - canaries.labels || checks.labels as labels, - severity, - owner, - last_runtime, - checks.created_at, - checks.updated_at, - checks.deleted_at, - status, - stats.max_time, - stats.min_time, - stats.total_checks -FROM checks checks -RIGHT JOIN ( - SELECT - check_id, - percentile_disc(0.99) within group (order by filtered_check_status.duration) as p99, - percentile_disc(0.97) within group (order by filtered_check_status.duration) as p97, - percentile_disc(0.05) within group (order by filtered_check_status.duration) as p95, - COUNT(*) FILTER (WHERE filtered_check_status.status = TRUE) as passed, - COUNT(*) FILTER (WHERE filtered_check_status.status = FALSE) as failed, - COUNT(*) total_checks, - MIN(filtered_check_status.time) as min_time, - MAX(filtered_check_status.time) as max_time - FROM - filtered_check_status - GROUP BY check_id -) as stats ON stats.check_id = checks.id - -INNER JOIN canaries on checks.canary_id = canaries.id - -RIGHT JOIN ( - SELECT check_id, json_agg(json_build_object('status',status,'duration',duration,'time',time %s)) as statii - FROM ( - SELECT check_id, - status, - time, - duration, - message, - error, - rank() OVER ( - PARTITION BY check_id - ORDER BY time DESC - ) - FROM filtered_check_status - ) check_statuses - WHERE rank <= :count - GROUP by check_id -) as statuses ON statuses.check_id = checks.id -WHERE (stats.passed > 0 OR stats.failed > 0) %s - `, clause, statusColumns, checkClause) - - if q.StatusCount == 0 { - q.StatusCount = 5 - } - namedArgs["count"] = q.StatusCount - - rows, err := exec(db, q, sql, namedArgs) - if err != nil { - return nil, err - } - defer rows.Close() - - checks := pkg.Checks{} - for rows.Next() { - var check = pkg.Check{} - vals, err := rows.Values() - if err != nil { - return nil, err - } - check.ID, _ = uuid.Parse(vals[0].(string)) - check.CanaryID, _ = uuid.Parse(vals[1].(string)) - check.Uptime.Passed = intV(vals[2]) - check.Uptime.Failed = intV(vals[3]) - check.Latency.Percentile99 = float64V(vals[4]) - check.Latency.Percentile97 = float64V(vals[5]) - check.Latency.Percentile95 = float64V(vals[6]) - check.Type = vals[8].(string) - check.Icon = vals[9].(string) - check.Name = vals[10].(string) - check.Description = vals[11].(string) - check.Namespace = vals[12].(string) - check.CanaryName = vals[13].(string) - check.Labels = mapStringString(vals[14]) - check.Severity = vals[15].(string) - check.Owner = vals[16].(string) - check.LastRuntime, _ = timeV(vals[17]) - check.CreatedAt, _ = timeV(vals[18]) - check.UpdatedAt, _ = timeV(vals[19]) - if vals[20] != nil { - check.DeletedAt, _ = timeV(vals[20]) - } - check.Status = vals[21].(string) - check.LatestRuntime, _ = timeV(vals[22]) - check.EarliestRuntime, _ = timeV(vals[23]) - check.TotalRuns = intV(vals[24]) - - if vals[7] != nil { - for _, status := range vals[7].([]interface{}) { - s := status.(map[string]interface{}) - check.Statuses = append(check.Statuses, pkg.CheckStatus{ - Status: s["status"].(bool), - Time: s["time"].(string), - Duration: intV(s["duration"]), - Message: stringV(s["message"]), - Error: stringV(s["error"]), - }) - } - } - if q.Trace { - logger.Infof("%+v", check) - } - checks = append(checks, &check) - } - return checks, err + return results, uptime, pkg.Latency{Percentile95: latency.Reduce(rolling.Percentile(95))}, nil } diff --git a/pkg/cache/postgres_query_test.go b/pkg/cache/postgres_query_test.go deleted file mode 100644 index 9577f34d5..000000000 --- a/pkg/cache/postgres_query_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package cache - -import ( - "fmt" - "testing" - - "github.com/flanksource/canary-checker/pkg/db" - "github.com/flanksource/duty" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" -) - -var cases = []struct { - fixture QueryParams - returnErr bool - args map[string]interface{} - clause string -}{ - { - fixture: QueryParams{ - Start: "1h", - Trace: true, - StatusCount: 5, - }, - args: map[string]interface{}{ - "start": float64(60), - }, - returnErr: false, - clause: "time > (NOW() at TIME ZONE 'utc' - Interval '1 minute' * :start)", - }, - // { - // fixture: QueryParams{ - // End: "1h", - // }, - // args: map[string]interface{}{ - // "end": float64(60), - // }, - // returnErr: false, - // clause: "time < (NOW() - Interval '1 minute' * :end)", - // }, - // { - // fixture: QueryParams{ - // Start: "2h", - // End: "1h", - // }, - // args: map[string]interface{}{ - // "start": float64(120), - // "end": float64(60), - // }, - // returnErr: false, - // clause: "time BETWEEN (NOW() - Interval '1 minute' * :start) AND (NOW() - Interval '1 minute' * :end)", - // }, -} - -func TestQueries(t *testing.T) { - t.Skip("Skipping TestQueries due to DB object creation failing. TODO: Fix me") - if err := db.Init(); err != nil { - t.Fatalf("Failed to init db: %v", err) - } - var err error - if db.Gorm, db.Pool, err = duty.SetupDB("", nil); err != nil { - t.Fatalf("Failed to setup database: %v", err) - } - psql := NewPostgresCache(db.Pool) - _cache := psql - t.Run(fmt.Sprintf("%T", psql), func(t *testing.T) { - for _, tc := range cases { - t.Run(tc.fixture.String(), func(t *testing.T) { - results, err := _cache.Query(tc.fixture) - if err != nil { - t.Errorf("Expected no error, got: %v", err) - } - g := NewWithT(t) - g.Expect(len(results)).To(BeNumerically(">", 1)) - check := results[0] - t.Log(*check) - g.Expect(*check).To((MatchFields(IgnoreExtras, Fields{ - "Name": Not(BeEmpty()), - "Namespace": Not(BeEmpty()), - "Type": Not(BeEmpty()), - "Key": Not(BeEmpty()), - "RunnerName": Not(BeEmpty()), - "Statuses": HaveLen(tc.fixture.StatusCount), - }))) - }) - } - }) -} - -func TestDurations(t *testing.T) { - for _, tc := range cases { - t.Run(tc.fixture.String(), func(t *testing.T) { - clause, args, err := tc.fixture.GetWhereClause() - returnedErr := err != nil - g := NewWithT(t) - g.Expect(returnedErr).To(Equal(tc.returnErr)) - g.Expect(args).To(Equal(tc.args)) - g.Expect(clause).To(Equal(tc.clause)) - }) - } -} diff --git a/pkg/topology/run_test.go b/pkg/topology/run_test.go index 22244ec6f..ec93b8349 100644 --- a/pkg/topology/run_test.go +++ b/pkg/topology/run_test.go @@ -5,11 +5,11 @@ import ( v1 "github.com/flanksource/canary-checker/api/v1" "github.com/flanksource/canary-checker/pkg/db" - "github.com/flanksource/canary-checker/pkg/utils" "github.com/flanksource/duty/models" "github.com/flanksource/duty/types" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" @@ -47,13 +47,13 @@ var _ = ginkgo.Describe("Test topology run", ginkgo.Ordered, func() { } ci := models.ConfigItem{ - Name: utils.Ptr("config-item"), + Name: lo.ToPtr("config-item"), Tags: &types.JSONStringMap{ "tag-1": "a", "tag-2": "b", }, - Config: utils.Ptr(`{"spec": {"container": {"name": "hello", "version": "v3"}}}`), - Type: utils.Ptr("Config::Dummy"), + Config: lo.ToPtr(`{"spec": {"container": {"name": "hello", "version": "v3"}}}`), + Type: lo.ToPtr("Config::Dummy"), ConfigClass: "Dummy", }