diff --git a/cmd/lotus-shed/finality.go b/cmd/lotus-shed/finality.go index 4d6fff0a488..715117de785 100644 --- a/cmd/lotus-shed/finality.go +++ b/cmd/lotus-shed/finality.go @@ -7,11 +7,10 @@ import ( "os" "strconv" - "github.com/dreading/gospecfunc/bessel" "github.com/filecoin-project/lotus/build" + skellampmf "github.com/rvagg/go-skellam-pmf" "github.com/urfave/cli/v2" "golang.org/x/exp/constraints" - "gonum.org/v1/gonum/stat/distuv" ) var finalityCmd = &cli.Command{ @@ -25,6 +24,10 @@ var finalityCmd = &cli.Command{ &cli.StringFlag{ Name: "input", }, + &cli.IntFlag{ + Name: "target", + Usage: "target epoch for which finality is calculated", + }, }, ArgsUsage: "[inputFile]", Action: func(cctx *cli.Context) error { @@ -49,14 +52,15 @@ var finalityCmd = &cli.Command{ return err } - blocksPerEpoch := 5.0 // Expected number of blocks per epoch - byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network - currentEpoch := len(chain) - 1 // Current epoch (end of history) - targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated + blocksPerEpoch := 5.0 // Expected number of blocks per epoch + byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network + currentEpoch := len(chain) - 1 // Current epoch (end of history) + // targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated + targetEpoch := cctx.Int("target") finality := FinalityCalcValidator(chain, blocksPerEpoch, byzantineFraction, currentEpoch, targetEpoch) - fmt.Fprintf(cctx.App.Writer, "Finality probability: %f\n", finality) + fmt.Fprintf(cctx.App.Writer, "Finality=%v @ %d for chain len=%d\n", finality, targetEpoch, currentEpoch) return nil }, @@ -88,15 +92,14 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio sumExpectedAdversarialBlocksI += rateMaliciousBlocks sumChainBlocksI += chain[i-1] // Poisson(k=k, lambda=sum(f*e)) - prLi := distuv.Poisson{Lambda: sumExpectedAdversarialBlocksI}.Prob(float64(k + sumChainBlocksI)) + prLi := poissonProb(sumExpectedAdversarialBlocksI, float64(k+sumChainBlocksI)) prL[k] = math.Max(prL[k], prLi) - // Break if prL[k] becomes negligible - if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] { - maxKL = k - prL = prL[:k+1] - break - } + } + if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] { + maxKL = k + prL = prL[:k+1] + break } } @@ -108,7 +111,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio // Calculate Pr(B=k) for each value of k for k := 0; k <= maxKB; k++ { - prB[k] = distuv.Poisson{Lambda: float64(currentEpoch-targetEpoch) * rateMaliciousBlocks}.Prob(float64(k)) + prB[k] = poissonProb(float64(currentEpoch-targetEpoch)*rateMaliciousBlocks, float64(k)) // Break if prB[k] becomes negligible if k > 1 && prB[k] < negligibleThreshold && prB[k] < prB[k-1] { @@ -119,11 +122,11 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio } // Compute M - prHgt0 := 1 - distuv.Poisson{Lambda: rateHonestBlocks}.Prob(0) + prHgt0 := 1 - poissonProb(rateHonestBlocks, 0) expZ := 0.0 for k := 0; k < int(4*blocksPerEpoch); k++ { - pmf := distuv.Poisson{Lambda: rateMaliciousBlocks}.Prob(float64(k)) + pmf := poissonProb(rateMaliciousBlocks, float64(k)) expZ += ((rateHonestBlocks + float64(k)) / math.Pow(2, float64(k))) * pmf } @@ -132,7 +135,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio prM := make([]float64, maxKM+1) for k := 0; k <= maxKM; k++ { for i := maxIM; i > 0; i-- { - probMI := SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain) + probMI := skellampmf.SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain) // Break if probMI becomes negligible if probMI < negligibleThreshold && probMI < prM[k] { @@ -185,6 +188,17 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio return math.Min(prError, 1.0) } +func poissonProb(lambda float64, x float64) float64 { + return math.Exp(poissonLogProb(lambda, x)) +} + +func poissonLogProb(lambda float64, x float64) float64 { + if x < 0 || math.Floor(x) != x { + return math.Inf(-1) + } + lg, _ := math.Lgamma(math.Floor(x) + 1) + return x*math.Log(lambda) - lambda - lg +} func sum[T constraints.Integer | constraints.Float](s []T) T { var total T @@ -210,46 +224,3 @@ func min(a, b int) int { } return b } - -// SkellamPMF calculates the probability mass function (PMF) of a Skellam distribution. -// -// The Skellam distribution is the probability distribution of the difference -// of two independent Poisson random variables. -// -// Arguments: -// * k - The difference of two Poisson random variables. -// * mu1 - The expected value of the first Poisson distribution. -// * mu2 - The expected value of the second Poisson distribution. -// -// Returns: -// * A float64 representing the PMF of the Skellam distribution at k. -func SkellamPMF(k int, mu1 float64, mu2 float64) float64 { - // Based on https://github.com/jsoares/rusty-skellam/blob/main/src/lib.rs - - // Return NaN if parameters outside range - if math.IsNaN(mu1) || mu1 <= 0 || math.IsNaN(mu2) || mu2 <= 0 { - return math.NaN() - } - - // Parameterise and compute the Modified Bessel function of the first kind - nu := float64(k) - z := complex(2.0*math.Sqrt(mu1*mu2), 0) - besselResult := bessel.I(nu, z) - - // Compute the pmf - return math.Exp(-(mu1 + mu2)) * math.Pow(mu1/mu2, nu/2.0) * real(besselResult) -} - -/* -func main() { - seed := rand.NewSource(1) - random := rand.New(seed) - chain := make([]int, 1000) - for i := range chain { - chain[i] = random.Intn(5) + 1 - } - - errorProbability := FinalityCalcValidator(chain, 5.0, 0.3, 1000, 900) - fmt.Printf("Error probability: %f\n", errorProbability) -} -*/ diff --git a/go.mod b/go.mod index 63803d58ebc..cbcbeae8fcb 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/lotus -go 1.21 +go 1.21.11 retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead. @@ -42,7 +42,7 @@ require ( github.com/filecoin-project/go-jsonrpc v0.3.2 github.com/filecoin-project/go-padreader v0.0.1 github.com/filecoin-project/go-paramfetch v0.0.4 - github.com/filecoin-project/go-state-types v0.14.0-dev + github.com/filecoin-project/go-state-types v0.14.0-rc1 github.com/filecoin-project/go-statemachine v1.0.3 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/go-storedcounter v0.1.0 @@ -123,6 +123,7 @@ require ( github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.3.0 + github.com/rvagg/go-skellam-pmf v0.0.0 github.com/samber/lo v1.39.0 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -146,6 +147,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.23.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.25.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 @@ -181,7 +183,6 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/drand/kyber-bls12381 v0.3.1 // indirect - github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 // indirect github.com/elastic/go-windows v1.0.0 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect @@ -318,7 +319,6 @@ require ( go.uber.org/dig v1.17.1 // indirect go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.15.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index 1683a09490c..7f6657312f5 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,6 @@ github.com/drand/kyber v1.3.0 h1:TVd7+xoRgKQ4Ck1viNLPFy6IWhuZM36Bq6zDXD8Asls= github.com/drand/kyber v1.3.0/go.mod h1:f+mNHjiGT++CuueBrpeMhFNdKZAsy0tu03bKq9D5LPA= github.com/drand/kyber-bls12381 v0.3.1 h1:KWb8l/zYTP5yrvKTgvhOrk2eNPscbMiUOIeWBnmUxGo= github.com/drand/kyber-bls12381 v0.3.1/go.mod h1:H4y9bLPu7KZA/1efDg+jtJ7emKx+ro3PU7/jWUVt140= -github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 h1:oOp1la+wHlyd3ODqW2CbFj8w6Lod4gPMzHFbD0rbp88= -github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3/go.mod h1:lkytgpljbGOM3VZj4Fm7FkGy/oUInQFklbkHBVAvJEg= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -293,9 +291,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.13.1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY= -github.com/filecoin-project/go-state-types v0.14.0-dev h1:bDwq1S28D7EC/uDmKU8vvNcdFw/YDsNq09pe3zeV5h4= -github.com/filecoin-project/go-state-types v0.14.0-dev/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY= +github.com/filecoin-project/go-state-types v0.14.0-rc1 h1:kWBGX/uqZmYotYMNmw+R/fIuot/k0KMcEtB7PKFy1SQ= +github.com/filecoin-project/go-state-types v0.14.0-rc1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY= github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk= github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= @@ -1192,6 +1189,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rvagg/go-skellam-pmf v0.0.0 h1:117SdhgSlbIWHtteOQHIm4TdxlmM3niE9mJhY6iJ37E= +github.com/rvagg/go-skellam-pmf v0.0.0/go.mod h1:8vgJzn0nYi+wCYggFMFtpn70AgqEO889eatpNoxM/kI= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= @@ -1757,7 +1756,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191022213345-0bbdf54effa2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=