From 30fbab6f2555b82d2034ccf47396b4ac45df2cba Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 19 Jul 2024 12:57:56 -1000 Subject: [PATCH 01/59] wip --- blockstore/badger/blockstore.go | 20 +-- blockstore/badger/versions/.badger.go.swo | Bin 0 -> 12288 bytes blockstore/badger/versions/.badger.go.swp | Bin 0 -> 12288 bytes .../badger/versions/.badger_interface.go.swp | Bin 0 -> 12288 bytes blockstore/badger/versions/.badger_v2.go.swp | Bin 0 -> 12288 bytes blockstore/badger/versions/.badger_v4.go.swp | Bin 0 -> 12288 bytes blockstore/badger/versions/badger.go | 39 +++++ .../badger/versions/badger_interface.go | 58 ++++++++ blockstore/badger/versions/badger_v2.go | 133 +++++++++++++++++ blockstore/badger/versions/badger_v4.go | 138 ++++++++++++++++++ 10 files changed, 379 insertions(+), 9 deletions(-) create mode 100644 blockstore/badger/versions/.badger.go.swo create mode 100644 blockstore/badger/versions/.badger.go.swp create mode 100644 blockstore/badger/versions/.badger_interface.go.swp create mode 100644 blockstore/badger/versions/.badger_v2.go.swp create mode 100644 blockstore/badger/versions/.badger_v4.go.swp create mode 100644 blockstore/badger/versions/badger.go create mode 100644 blockstore/badger/versions/badger_interface.go create mode 100644 blockstore/badger/versions/badger_v2.go create mode 100644 blockstore/badger/versions/badger_v4.go diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index e6612ccf46a..bbdfd36f599 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -10,9 +10,8 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/options" - "github.com/dgraph-io/badger/v2/pb" + "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/pb" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -22,6 +21,8 @@ import ( "go.uber.org/zap" "golang.org/x/xerrors" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" + "github.com/filecoin-project/lotus/blockstore" ) @@ -114,8 +115,9 @@ type Blockstore struct { moveState bsMoveState rlock int - db *badger.DB - dbNext *badger.DB // when moving + //change this + db badger.BadgerDB + dbNext badger.BadgerDB // when moving opts Options prefixing bool @@ -391,7 +393,7 @@ func symlink(path, linkTo string) error { // doCopy copies a badger blockstore to another, with an optional filter; if the filter // is not nil, then only cids that satisfy the filter will be copied. -func (b *Blockstore) doCopy(from, to *badger.DB) error { +func (b *Blockstore) doCopy(from, to *badger.BadgerDB) error { workers := runtime.NumCPU() / 2 if workers < 2 { workers = 2 @@ -745,7 +747,7 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error { defer KeyPool.Put(k) } - put := func(db *badger.DB) error { + put := func(db *badger.BadgerDB) error { // Check if we have it before writing it. switch err := db.View(func(txn *badger.Txn) error { _, err := txn.Get(k) @@ -832,7 +834,7 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { return err } - put := func(db *badger.DB) error { + put := func(db *badger.BadgerDB) error { batch := db.NewWriteBatch() defer batch.Cancel() @@ -1094,6 +1096,6 @@ func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte { // this method is added for lotus-shed needs // WARNING: THIS IS COMPLETELY UNSAFE; DONT USE THIS IN PRODUCTION CODE -func (b *Blockstore) DB() *badger.DB { +func (b *Blockstore) DB() *badger.BadgerDB { return b.db } diff --git a/blockstore/badger/versions/.badger.go.swo b/blockstore/badger/versions/.badger.go.swo new file mode 100644 index 0000000000000000000000000000000000000000..8457c6c9391a7781342fc023ad91078e845ec20a GIT binary patch literal 12288 zcmeI2O>5LZ7{{j`lm**Ly_abe-AbF(7=>1fPs-+dhn`bYc z`~o6)^Bed9Jn2!qiTVL5`k!nPv}{{?RGz?(NoJmTd47{Ugq>iiv$4sS?Inim9AmSe zUv{5o57@iYj1A(j?~5cCaUzsyKjhl1;udfFnU<+I(BVfDDiU zGC&5%02v?yWPl8i0W$Cp8t@~=HqJ8k6PW-1>+k>X(~NxsU%)5u5xfD;GWG$y1+Ty!cmy)A3~qqy;3Aj-)8JRN&V931 zy2t<-AOmE843GgbKnBPF86X2EF|f;8O6WqRJPH*8~Nu@=^N2ir}b$Z#FXRSzd{UXsTn8(c3iUPjM*UCf9>uc>x zqq~l~69pZ27xDGvtM%ljOUEr&hMuDdINA%da%;>uabt?e<2x zDF#wKMDMt|t(y4C9|NwB7P^&bbMCsG`MQsxLq8YX8oJgo33w&}@9+e?0||Iz6Y!b@ zJo|(Rpu_Tp#aK2+;61Aeb&m;kExe>w ZnM7`}cwk;f-p1#(AKv$SqF!0fegoCUKiL2P literal 0 HcmV?d00001 diff --git a/blockstore/badger/versions/.badger.go.swp b/blockstore/badger/versions/.badger.go.swp new file mode 100644 index 0000000000000000000000000000000000000000..62ff887059f13f6b1c75ac7f994c341579c9ab31 GIT binary patch literal 12288 zcmeI2J#W)M7{{+nSdZK2bK0sDJ_0*m z03jhh03!<$0~1Vu!~z2YEC|HL{}Lx6kWd(b$I?&dyL;~C`Q24&RfqGZmE;BdUx7#!sOIM2FSp41Ep>?&g@=3<(=&R zs2{bD@B{mor^izd86X2>fDDiUGC&5%02v?y|C|Ax&#>FbYDbk>y((vZD61+JWPl8i z0Wv@a$N(8217v^4^4gw481fTaX_8vS2_rM*{1=qn@a2nLX40yMjv1i}`xC_>S1}oqcmSp>0AFKHj}}9>2TCq9qh2b}po{*!tZ-QL8fx!B+m$ zh_MD##_ECD;<4e6cUm7^)pJN`-2An{n9m zwh%vDd~vwA8PfFnJHyDq0!`1knv1L^b6uD*y}(Frdi=zwANnoJ^gEZbCDD~?1EZ7b zwd&#v-%WUNu+giz#?Fgc9_S9Hjsjc^yXV>4B;cC_{OJk!KP2E!Ou!!{;5&bq00#Uj z0pBE0tR*_?3{R(ju*RG@Yans*1CgWiVXZQYxPTMJ1*|?;(EHJ|E Yr-HYnYc3_PF_s6>YS0$dfiLddT-K8Qd4VM@NbkC3qlhm0pV7YSEo`yI+w-EU=+Wm;xlB14uO z(D&*s6Z#RK_%WsqrASvYB`@<(`-#*=?gyA+lxwNDAFyb^l|SJs7cxat^?XUBnyZ+F z+#ARPma-Cwl|FZQtiTLr8FRspMkRlq7>6|f3e z1*`&hxdJ*{B5xqZd+In(*Y(nEb^T~7tAJI&Dqt0`3RnfK0#*U5fK|XMU=^?mSOxxv z3b2Tfw;galoX79~d*A>6c$knI;5Tp${0u$=?|~E~U<953OW@Ck2>Atk4?YBMfy>|p za1tB`_kvH35b`d#1TKKrz@y-=2MM_jz6BqHH^Egf2WNl_9Ps-Cg!~A;17CtKz~?}N zi{LcqfaBm8cpUtM*uDW@gO9-b;2luL+y~p>W$+BJ#VTMGunJfOtO8a6tAJHtzXJ0) znG+`{Vv4fsb&EL0{ZMB#*%Y~+qtuQ6wcbA&J-buRN%Sn^)T~2ut%^|7*YTOJw3;6D z49x5wdi6pu(;V$ADYzS_G%nK6HP$xeN=a2$#={NG$F#Yx3diAJWs7($Ql3{1 zgmOHV=|Z>CU4fA~Y?M*0Semm?;|{FxZcXGNQ<2*yP7h;+hG*5<#98AB*W4ZPS?z1n z(&xG~cj$yAx3{gEw#Mw1K6dJOOeC*N7d@;3DNTpQvEY0aj#1mh@uCWc9DB;(aj2&> zl(^YD)n3ok3$CY;Tjt!d$%BOky(tGTDTq0Bv#Kg{M5F^qgMiN!<8|q#<=;}?cAZSs zhWEANY~SV>_v*GogTq>w1Iy8fX;$vmVt5;A#HT%m7`Hf_3Uh4JvO-QPuC?@PBXPRr zjpu*SuU`ats6=p}SREP$CL2c+x@=BEty6~@X^?K7=iQR5#MxrgF4N(#FLpWntFwL2 z+LJ|EW>;8`ipI zjOJRoIi_YN*bmKuKHq)$ literal 0 HcmV?d00001 diff --git a/blockstore/badger/versions/.badger_v2.go.swp b/blockstore/badger/versions/.badger_v2.go.swp new file mode 100644 index 0000000000000000000000000000000000000000..9ef2c83fc54317d8d44dac43e7617700083cb99b GIT binary patch literal 12288 zcmeI2U1%It6vuCsHeI#a55ZPnZnq`O#$=|(w%S6eZXz+HsikQnsIkt>-r0Tfe3;{iw{br(sxB$Dk}O~wc?ASh~E|wAAHf5;>Vl+J71ewW@GyzC^zt9cIKXY z?mho=@435VjBV57lXRE5jo@!BA?%lztsSS^&8dmFG!5H$9R6r_Fpg_p2q1)GxTZeCnw)&RxX1Zbh;Q~YXNGc!|kP1izqyka_ zsen{KDj*g3zbe3+E6C#r?usP9r<3c7u4{5DFH!-ifK)&#AQg}bNCl(j?Q3{0zPUpM%f98Sor92A&0bKtH$xe zi`Noz0ela>1Mh%S;1zHZxL_1i!JVKFe6fa*x4~I(2Al@-U<@c=D=34_;Nmred<;GU z=fN}J2$%s6fZd=UYz14upH~y|E%+LI1>OKJfaBnK&=3B)ijY6R@8CUf0=xuX1owem z;5M)mYyu_F2UdYEON4v~-UsKvG0*}W1mF&^15j{1SP#~LPgWE1I(QAd3XX%Pz?0w! za5t!c)nFA^2|n$EM&JxM4ITv#fd|11*bZ(2o4`hJ5$|#Q2rhshz&Y?1coUoj0k{_& z28Y1SU;|hO)`Fj~F3tn7K28GpCl!zi{7VWfl9JDO=yzXTYID|5$9p{&jnnZ9Q%Z$c zp=(k_^q>RP=KGQLTuL-uWte>cz-alh3aMsJmwC;pV zvD{_nlh{x(WhM4kq0IL^Ke;5{2jaKDR1pT@3r$9s21Wz!hbB*KdD~SFfl)g!b-$Jc zg6ADQU{Il%MLveq!HCyr*nFA(sX;vBVke`_VVo&*Y#|k*U>=!GqG>+(Xi2yab9<}5?Dbx3ORb~=rqli1j>Lc1)f>A9S>xjGVG3l(;4 zhm@=aR}c8u9*(P&C&wr5DpyEJ8=!ml?xh1-vbhQdoMLWGF+vMd2`!6tE#N@i=hYAR z%(B~b4{b&tmh;6kb0szwy4Y?#l`v5EhmA20zFcu#)>UMwM0ON$r7qGe#ERt;5kD); zg`tjC)p57R+9NuWG08A5Gst#{(Md&x7MRfi-$p2mb5BkB<~L5{zbf7b{b&CLW|!uF zDl`Jqw~c5O4N}t>Ome~kRaMhvm5U|UrF%IE7jY4rYtey;>3xVXSz;Z}6s~hMt|0J9 zycI16`eR>4p`jz;6|XY=^1gv}c2L}fAA9J$zYCe-je zr$R;bi3`*swtRdNyVP=(+z7nwisqy;MdMA|Y+i|uFf`J1U%_2xzVVryn?l?R1BkOv zbnJBK^b^PPWzObs6!zuMC+&9$GfINbbcxGFCVqUY^i)1=PY&_;wqP@i2P2Lbu+9;l z)34JS<*%TB)Vp})CQZ4en&<}j+4QRjZ4FbbH{H+!M!84r zM$=&p=JF*rqgAoVm~ZJOQ}J7ml*)CR&xVF-dJU~shs)U^+tacVS?*;Ke;?9X!)5$5 eE#u4*mdP1cCK%iEIaLU0>gJqYXEc4Cp8O3--sK_y literal 0 HcmV?d00001 diff --git a/blockstore/badger/versions/.badger_v4.go.swp b/blockstore/badger/versions/.badger_v4.go.swp new file mode 100644 index 0000000000000000000000000000000000000000..0d5e1d36435f93965fa8003eb75de79d06e0a10d GIT binary patch literal 12288 zcmeI2O^6&t6vr!yAEV|21w~L(yOOnMveO-Pkwn>pvjfSn*%-3PsEBA!PxWlOJ>5fh z_3Z2}8}Et)j3(+;iHKg!5>QWGje0XNV05DwK@krgqUgW+Yqp1;i8(4Y@S{KK)vH(k zSM|D=?Z%#ZZI13!_YnMc5OVV8cg($?_sF@M329r#g3fH8SvK=T`;gyUY*#+0cevY8 zZP(E49NXg=hgy?$H50B4*K*1o&wYs*T(eys_*w%l7Jcq|Ol#=P7W1?&^L)#7&=g+3 z(4A6SZq69;mQ+A0kfT7r&GO!@_)g zhrk5*qkuVsD_{eB1WtiBz)A2TI0_DfF|ZwM0e{_!xWFZF5nKSLK>&QdsCcrN6C%)y_0N;bl;1loxcpsbruYxtO3Rb`qU?;dA z+y{Qg+V~b+0N;QQ!3iMNOcThTR6r^)LV-0>@E8xg!J|Vh&f02i*nQDhk3M`-io^^Y zgDRp2Emgv=pz2dKToq|*Lj%j%qLhJh}aV(2Ryxvw5{TZcVnT0J^VJK@PFUpzHeB4eSq<+PI2 z`0Amcz2Ai|M;u4G3N4k;j-s3-E|Tm&F{C(la4My_{&-h>raZ{eI%zkX|&znuPI@BEb;H)WP?lDKLmTl4zW zPSeuY7P{o=9iQ@r Date: Fri, 19 Jul 2024 12:59:29 -1000 Subject: [PATCH 02/59] delete vim swaps --- blockstore/badger/versions/.badger.go.swo | Bin 12288 -> 0 bytes blockstore/badger/versions/.badger.go.swp | Bin 12288 -> 0 bytes .../badger/versions/.badger_interface.go.swp | Bin 12288 -> 0 bytes blockstore/badger/versions/.badger_v2.go.swp | Bin 12288 -> 0 bytes blockstore/badger/versions/.badger_v4.go.swp | Bin 12288 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 blockstore/badger/versions/.badger.go.swo delete mode 100644 blockstore/badger/versions/.badger.go.swp delete mode 100644 blockstore/badger/versions/.badger_interface.go.swp delete mode 100644 blockstore/badger/versions/.badger_v2.go.swp delete mode 100644 blockstore/badger/versions/.badger_v4.go.swp diff --git a/blockstore/badger/versions/.badger.go.swo b/blockstore/badger/versions/.badger.go.swo deleted file mode 100644 index 8457c6c9391a7781342fc023ad91078e845ec20a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O>5LZ7{{j`lm**Ly_abe-AbF(7=>1fPs-+dhn`bYc z`~o6)^Bed9Jn2!qiTVL5`k!nPv}{{?RGz?(NoJmTd47{Ugq>iiv$4sS?Inim9AmSe zUv{5o57@iYj1A(j?~5cCaUzsyKjhl1;udfFnU<+I(BVfDDiU zGC&5%02v?yWPl8i0W$Cp8t@~=HqJ8k6PW-1>+k>X(~NxsU%)5u5xfD;GWG$y1+Ty!cmy)A3~qqy;3Aj-)8JRN&V931 zy2t<-AOmE843GgbKnBPF86X2EF|f;8O6WqRJPH*8~Nu@=^N2ir}b$Z#FXRSzd{UXsTn8(c3iUPjM*UCf9>uc>x zqq~l~69pZ27xDGvtM%ljOUEr&hMuDdINA%da%;>uabt?e<2x zDF#wKMDMt|t(y4C9|NwB7P^&bbMCsG`MQsxLq8YX8oJgo33w&}@9+e?0||Iz6Y!b@ zJo|(Rpu_Tp#aK2+;61Aeb&m;kExe>w ZnM7`}cwk;f-p1#(AKv$SqF!0fegoCUKiL2P diff --git a/blockstore/badger/versions/.badger.go.swp b/blockstore/badger/versions/.badger.go.swp deleted file mode 100644 index 62ff887059f13f6b1c75ac7f994c341579c9ab31..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2J#W)M7{{+nSdZK2bK0sDJ_0*m z03jhh03!<$0~1Vu!~z2YEC|HL{}Lx6kWd(b$I?&dyL;~C`Q24&RfqGZmE;BdUx7#!sOIM2FSp41Ep>?&g@=3<(=&R zs2{bD@B{mor^izd86X2>fDDiUGC&5%02v?y|C|Ax&#>FbYDbk>y((vZD61+JWPl8i z0Wv@a$N(8217v^4^4gw481fTaX_8vS2_rM*{1=qn@a2nLX40yMjv1i}`xC_>S1}oqcmSp>0AFKHj}}9>2TCq9qh2b}po{*!tZ-QL8fx!B+m$ zh_MD##_ECD;<4e6cUm7^)pJN`-2An{n9m zwh%vDd~vwA8PfFnJHyDq0!`1knv1L^b6uD*y}(Frdi=zwANnoJ^gEZbCDD~?1EZ7b zwd&#v-%WUNu+giz#?Fgc9_S9Hjsjc^yXV>4B;cC_{OJk!KP2E!Ou!!{;5&bq00#Uj z0pBE0tR*_?3{R(ju*RG@Yans*1CgWiVXZQYxPTMJ1*|?;(EHJ|E Yr-HYnYc3_PF_s6>YS0$dfiLddT-K8Qd4VM@NbkC3qlhm0pV7YSEo`yI+w-EU=+Wm;xlB14uO z(D&*s6Z#RK_%WsqrASvYB`@<(`-#*=?gyA+lxwNDAFyb^l|SJs7cxat^?XUBnyZ+F z+#ARPma-Cwl|FZQtiTLr8FRspMkRlq7>6|f3e z1*`&hxdJ*{B5xqZd+In(*Y(nEb^T~7tAJI&Dqt0`3RnfK0#*U5fK|XMU=^?mSOxxv z3b2Tfw;galoX79~d*A>6c$knI;5Tp${0u$=?|~E~U<953OW@Ck2>Atk4?YBMfy>|p za1tB`_kvH35b`d#1TKKrz@y-=2MM_jz6BqHH^Egf2WNl_9Ps-Cg!~A;17CtKz~?}N zi{LcqfaBm8cpUtM*uDW@gO9-b;2luL+y~p>W$+BJ#VTMGunJfOtO8a6tAJHtzXJ0) znG+`{Vv4fsb&EL0{ZMB#*%Y~+qtuQ6wcbA&J-buRN%Sn^)T~2ut%^|7*YTOJw3;6D z49x5wdi6pu(;V$ADYzS_G%nK6HP$xeN=a2$#={NG$F#Yx3diAJWs7($Ql3{1 zgmOHV=|Z>CU4fA~Y?M*0Semm?;|{FxZcXGNQ<2*yP7h;+hG*5<#98AB*W4ZPS?z1n z(&xG~cj$yAx3{gEw#Mw1K6dJOOeC*N7d@;3DNTpQvEY0aj#1mh@uCWc9DB;(aj2&> zl(^YD)n3ok3$CY;Tjt!d$%BOky(tGTDTq0Bv#Kg{M5F^qgMiN!<8|q#<=;}?cAZSs zhWEANY~SV>_v*GogTq>w1Iy8fX;$vmVt5;A#HT%m7`Hf_3Uh4JvO-QPuC?@PBXPRr zjpu*SuU`ats6=p}SREP$CL2c+x@=BEty6~@X^?K7=iQR5#MxrgF4N(#FLpWntFwL2 z+LJ|EW>;8`ipI zjOJRoIi_YN*bmKuKHq)$ diff --git a/blockstore/badger/versions/.badger_v2.go.swp b/blockstore/badger/versions/.badger_v2.go.swp deleted file mode 100644 index 9ef2c83fc54317d8d44dac43e7617700083cb99b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2U1%It6vuCsHeI#a55ZPnZnq`O#$=|(w%S6eZXz+HsikQnsIkt>-r0Tfe3;{iw{br(sxB$Dk}O~wc?ASh~E|wAAHf5;>Vl+J71ewW@GyzC^zt9cIKXY z?mho=@435VjBV57lXRE5jo@!BA?%lztsSS^&8dmFG!5H$9R6r_Fpg_p2q1)GxTZeCnw)&RxX1Zbh;Q~YXNGc!|kP1izqyka_ zsen{KDj*g3zbe3+E6C#r?usP9r<3c7u4{5DFH!-ifK)&#AQg}bNCl(j?Q3{0zPUpM%f98Sor92A&0bKtH$xe zi`Noz0ela>1Mh%S;1zHZxL_1i!JVKFe6fa*x4~I(2Al@-U<@c=D=34_;Nmred<;GU z=fN}J2$%s6fZd=UYz14upH~y|E%+LI1>OKJfaBnK&=3B)ijY6R@8CUf0=xuX1owem z;5M)mYyu_F2UdYEON4v~-UsKvG0*}W1mF&^15j{1SP#~LPgWE1I(QAd3XX%Pz?0w! za5t!c)nFA^2|n$EM&JxM4ITv#fd|11*bZ(2o4`hJ5$|#Q2rhshz&Y?1coUoj0k{_& z28Y1SU;|hO)`Fj~F3tn7K28GpCl!zi{7VWfl9JDO=yzXTYID|5$9p{&jnnZ9Q%Z$c zp=(k_^q>RP=KGQLTuL-uWte>cz-alh3aMsJmwC;pV zvD{_nlh{x(WhM4kq0IL^Ke;5{2jaKDR1pT@3r$9s21Wz!hbB*KdD~SFfl)g!b-$Jc zg6ADQU{Il%MLveq!HCyr*nFA(sX;vBVke`_VVo&*Y#|k*U>=!GqG>+(Xi2yab9<}5?Dbx3ORb~=rqli1j>Lc1)f>A9S>xjGVG3l(;4 zhm@=aR}c8u9*(P&C&wr5DpyEJ8=!ml?xh1-vbhQdoMLWGF+vMd2`!6tE#N@i=hYAR z%(B~b4{b&tmh;6kb0szwy4Y?#l`v5EhmA20zFcu#)>UMwM0ON$r7qGe#ERt;5kD); zg`tjC)p57R+9NuWG08A5Gst#{(Md&x7MRfi-$p2mb5BkB<~L5{zbf7b{b&CLW|!uF zDl`Jqw~c5O4N}t>Ome~kRaMhvm5U|UrF%IE7jY4rYtey;>3xVXSz;Z}6s~hMt|0J9 zycI16`eR>4p`jz;6|XY=^1gv}c2L}fAA9J$zYCe-je zr$R;bi3`*swtRdNyVP=(+z7nwisqy;MdMA|Y+i|uFf`J1U%_2xzVVryn?l?R1BkOv zbnJBK^b^PPWzObs6!zuMC+&9$GfINbbcxGFCVqUY^i)1=PY&_;wqP@i2P2Lbu+9;l z)34JS<*%TB)Vp})CQZ4en&<}j+4QRjZ4FbbH{H+!M!84r zM$=&p=JF*rqgAoVm~ZJOQ}J7ml*)CR&xVF-dJU~shs)U^+tacVS?*;Ke;?9X!)5$5 eE#u4*mdP1cCK%iEIaLU0>gJqYXEc4Cp8O3--sK_y diff --git a/blockstore/badger/versions/.badger_v4.go.swp b/blockstore/badger/versions/.badger_v4.go.swp deleted file mode 100644 index 0d5e1d36435f93965fa8003eb75de79d06e0a10d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O^6&t6vr!yAEV|21w~L(yOOnMveO-Pkwn>pvjfSn*%-3PsEBA!PxWlOJ>5fh z_3Z2}8}Et)j3(+;iHKg!5>QWGje0XNV05DwK@krgqUgW+Yqp1;i8(4Y@S{KK)vH(k zSM|D=?Z%#ZZI13!_YnMc5OVV8cg($?_sF@M329r#g3fH8SvK=T`;gyUY*#+0cevY8 zZP(E49NXg=hgy?$H50B4*K*1o&wYs*T(eys_*w%l7Jcq|Ol#=P7W1?&^L)#7&=g+3 z(4A6SZq69;mQ+A0kfT7r&GO!@_)g zhrk5*qkuVsD_{eB1WtiBz)A2TI0_DfF|ZwM0e{_!xWFZF5nKSLK>&QdsCcrN6C%)y_0N;bl;1loxcpsbruYxtO3Rb`qU?;dA z+y{Qg+V~b+0N;QQ!3iMNOcThTR6r^)LV-0>@E8xg!J|Vh&f02i*nQDhk3M`-io^^Y zgDRp2Emgv=pz2dKToq|*Lj%j%qLhJh}aV(2Ryxvw5{TZcVnT0J^VJK@PFUpzHeB4eSq<+PI2 z`0Amcz2Ai|M;u4G3N4k;j-s3-E|Tm&F{C(la4My_{&-h>raZ{eI%zkX|&znuPI@BEb;H)WP?lDKLmTl4zW zPSeuY7P{o=9iQ@r Date: Fri, 19 Jul 2024 14:11:54 -1000 Subject: [PATCH 03/59] check in wip --- blockstore/badger/blockstore.go | 84 ++++++++----------- blockstore/badger/versions/badger.go | 25 ++++-- .../badger/versions/badger_interface.go | 13 ++- blockstore/badger/versions/badger_v2.go | 40 ++++++++- blockstore/badger/versions/badger_v4.go | 40 ++++++++- documentation/en/default-lotus-config.toml | 4 + go.mod | 5 +- go.sum | 5 ++ node/config/def.go | 1 + node/config/doc_gen.go | 6 ++ node/config/types.go | 1 + 11 files changed, 163 insertions(+), 61 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index bbdfd36f599..5e004c1fe35 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/options" "github.com/dgraph-io/badger/pb" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -26,6 +25,11 @@ import ( "github.com/filecoin-project/lotus/blockstore" ) +// aliases to mask badger dependencies. +const ( + defaultGCThreshold = 0.125 +) + var ( // KeyPool is the buffer pool we use to compute storage keys. KeyPool *pool.BufferPool = pool.GlobalPool @@ -39,33 +43,6 @@ var ( log = logger.Logger("badgerbs") ) -// aliases to mask badger dependencies. -const ( - // FileIO is equivalent to badger/options.FileIO. - FileIO = options.FileIO - // MemoryMap is equivalent to badger/options.MemoryMap. - MemoryMap = options.MemoryMap - // LoadToRAM is equivalent to badger/options.LoadToRAM. - LoadToRAM = options.LoadToRAM - defaultGCThreshold = 0.125 -) - -// Options embeds the badger options themselves, and augments them with -// blockstore-specific options. -type Options struct { - badger.Options - - // Prefix is an optional prefix to prepend to keys. Default: "". - Prefix string -} - -func DefaultOptions(path string) Options { - return Options{ - Options: badger.DefaultOptions(path), - Prefix: "", - } -} - // badgerLogger is a local wrapper for go-log to make the interface // compatible with badger.Logger (namely, aliasing Warnf to Warningf) type badgerLogger struct { @@ -118,7 +95,7 @@ type Blockstore struct { //change this db badger.BadgerDB dbNext badger.BadgerDB // when moving - opts Options + opts badger.Options prefixing bool prefix []byte @@ -133,13 +110,13 @@ var _ blockstore.BlockstoreSize = (*Blockstore)(nil) var _ io.Closer = (*Blockstore)(nil) // Open creates a new badger-backed blockstore, with the supplied options. -func Open(opts Options) (*Blockstore, error) { +func Open(opts badger.Options) (*Blockstore, error) { opts.Logger = &badgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } - db, err := badger.Open(opts.Options) + db, err := badger.OpenBadgerDB(opts) if err != nil { return nil, fmt.Errorf("failed to open badger blockstore: %w", err) } @@ -319,7 +296,7 @@ func (b *Blockstore) movingGC() error { opts.Dir = newPath opts.ValueDir = newPath - dbNew, err := badger.Open(opts.Options) + dbNew, err := badger.OpenBadgerDB(opts) if err != nil { return fmt.Errorf("failed to open badger blockstore in %s: %w", newPath, err) } @@ -608,11 +585,14 @@ func (b *Blockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) erro defer KeyPool.Put(k) } - return b.db.View(func(txn *badger.Txn) error { + return b.db.View(func(txn badger.Txn) error { + + errKeyNotFound := b.db.GetErrKeyNotFound() + switch item, err := txn.Get(k); err { case nil: return item.Value(fn) - case badger.ErrKeyNotFound: + case errKeyNotFound: return ipld.ErrNotFound{Cid: cid} default: return fmt.Errorf("failed to view block from badger blockstore: %w", err) @@ -647,13 +627,14 @@ func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { defer KeyPool.Put(k) } - err := b.db.View(func(txn *badger.Txn) error { + err := b.db.View(func(txn badger.Txn) error { _, err := txn.Get(k) return err }) + errKeyNotFound := b.db.GetErrKeyNotFound() switch err { - case badger.ErrKeyNotFound: + case errKeyNotFound: return false, nil case nil: return true, nil @@ -682,12 +663,13 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) } var val []byte - err := b.db.View(func(txn *badger.Txn) error { + err := b.db.View(func(txn badger.Txn) error { + errKeyNotFound := b.db.GetErrKeyNotFound() switch item, err := txn.Get(k); err { case nil: val, err = item.ValueCopy(nil) return err - case badger.ErrKeyNotFound: + case errKeyNotFound: return ipld.ErrNotFound{Cid: cid} default: return fmt.Errorf("failed to get block from badger blockstore: %w", err) @@ -715,11 +697,12 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { } var size int - err := b.db.View(func(txn *badger.Txn) error { + err := b.db.View(func(txn badger.Txn) error { + errKeyNotFound := b.db.GetErrKeyNotFound() switch item, err := txn.Get(k); err { case nil: size = int(item.ValueSize()) - case badger.ErrKeyNotFound: + case errKeyNotFound: return ipld.ErrNotFound{Cid: cid} default: return fmt.Errorf("failed to get block size from badger blockstore: %w", err) @@ -747,13 +730,15 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error { defer KeyPool.Put(k) } - put := func(db *badger.BadgerDB) error { + put := func(db badger.BadgerDB) error { + errKeyNotFound := db.GetErrKeyNotFound() + // Check if we have it before writing it. - switch err := db.View(func(txn *badger.Txn) error { + switch err := db.View(func(txn badger.Txn) error { _, err := txn.Get(k) return err }); err { - case badger.ErrKeyNotFound: + case errKeyNotFound: case nil: // Already exists, skip the put. return nil @@ -762,7 +747,7 @@ func (b *Blockstore) Put(ctx context.Context, block blocks.Block) error { } // Then write it. - err := db.Update(func(txn *badger.Txn) error { + err := db.Update(func(txn badger.Txn) error { return txn.Set(k, block.RawData()) }) if err != nil { @@ -817,10 +802,13 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { keys = append(keys, k) } - err := b.db.View(func(txn *badger.Txn) error { + err := b.db.View(func(txn badger.Txn) error { + + errKeyNotFound := b.db.GetErrKeyNotFound() + for i, k := range keys { switch _, err := txn.Get(k); err { - case badger.ErrKeyNotFound: + case errKeyNotFound: case nil: keys[i] = nil default: @@ -834,7 +822,7 @@ func (b *Blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { return err } - put := func(db *badger.BadgerDB) error { + put := func(db badger.BadgerDB) error { batch := db.NewWriteBatch() defer batch.Cancel() @@ -885,7 +873,7 @@ func (b *Blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { defer KeyPool.Put(k) } - return b.db.Update(func(txn *badger.Txn) error { + return b.db.Update(func(txn badger.Txn) error { return txn.Delete(k) }) } diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 19127be1f51..11caf36ac07 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -7,22 +7,33 @@ import ( badgerV4 "github.com/dgraph-io/badger/v4" ) -func openBadgerDB(path string, inMemory bool, version string) (BadgerDB, error) { +// Options embeds the badger options themselves, and augments them with +// blockstore-specific options. +type Options struct { + // BadgerVersion sets the release version of badger to use + BadgerVersion int + + // Prefix is an optional prefix to prepend to keys. Default: "". + Prefix string +} + +func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error + prefix := opts.Prefix + version := opts.BadgerVersion + switch version { - case "v4": - opts := badgerV4.DefaultOptions(path) - opts.InMemory = inMemory + case 4: + opts := badgerV4.DefaultOptions(prefix) var dbV4 *badgerV4.DB dbV4, err = badgerV4.Open(opts) if err == nil { db = BadgerDB(&BadgerV4{dbV4}) } - case "v2": - opts := badgerV2.DefaultOptions(path) - opts.InMemory = inMemory + case 2: + opts := badgerV2.DefaultOptions(prefix) var dbV2 *badgerV2.DB dbV2, err = badgerV2.Open(opts) if err == nil { diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index ae289fb04c2..46f34077330 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -15,7 +15,7 @@ type BadgerDB interface { NewStream() BadgerStream Update(func(txn Txn) error) error View(func(txn Txn) error) error - NewTxn(update bool) Txn + NewTransaction(update bool) Txn RunValueLogGC(discardRatio float64) error Sync() error MaxBatchCount() int64 @@ -23,6 +23,8 @@ type BadgerDB interface { Subscribe(ctx context.Context, cb func(kv *KVList) error, prefixes ...[]byte) error BlockCacheMetrics() *ristretto.Metrics IndexCacheMetrics() *ristretto.Metrics + GetErrKeyNotFound() error + NewWriteBatch() WriteBatch } // BadgerStream defines the common interface for streaming data in Badger. @@ -47,6 +49,8 @@ type Item interface { Value(fn func([]byte) error) error Key() []byte Version() uint64 + ValueCopy(dst []byte) ([]byte, error) + ValueSize() int64 } // KVList is an alias for the KVList type from the Badger package. @@ -56,3 +60,10 @@ type Buffer struct { kvList KVList buf z.Buffer } + +type WriteBatch interface { + Set(key, val []byte) error + Delete(key []byte) error + Flush() error + Cancel() +} diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index dbf1f0e6777..e05b221d6f0 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -37,8 +37,8 @@ func (b *BadgerV2) View(fn func(txn Txn) error) error { }) } -func (b *BadgerV2) NewTxn(update bool) Txn { - return &BadgerV2Txn{b.DB.NewTxn(update)} +func (b *BadgerV2) NewTransaction(update bool) Txn { + return &BadgerV2Txn{b.DB.NewTransaction(update)} } func (b *BadgerV2) RunValueLogGC(discardRatio float64) error { @@ -69,6 +69,34 @@ func (b *BadgerV2) IndexCacheMetrics() *ristretto.Metrics { return b.DB.IndexCacheMetrics() } +func (b *BadgerV2) GetErrKeyNotFound() error { + return badger.ErrKeyNotFound +} + +func (b *BadgerV2) NewWriteBatch() WriteBatch { + return &BadgerV2WriteBatch{b.DB.NewWriteBatch()} +} + +type BadgerV2WriteBatch struct { + *badger.WriteBatch +} + +func (wb *BadgerV2WriteBatch) Set(key, val []byte) error { + return wb.WriteBatch.Set(key, val) +} + +func (wb *BadgerV2WriteBatch) Delete(key []byte) error { + return wb.WriteBatch.Delete(key) +} + +func (wb *BadgerV2WriteBatch) Flush() error { + return wb.WriteBatch.Flush() +} + +func (wb *BadgerV2WriteBatch) Cancel() { + wb.WriteBatch.Cancel() +} + type BadgerV2Stream struct { *badger.Stream } @@ -131,3 +159,11 @@ func (item *BadgerV2Item) Key() []byte { func (item *BadgerV2Item) Version() uint64 { return item.Item.Version() } + +func (item *BadgerV2Item) ValueCopy(dst []byte) ([]byte, error) { + return item.Item.ValueCopy(dst) +} + +func (item *BadgerV2Item) ValueSize() int64 { + return item.Item.ValueSize() +} diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 0b7e42ac5b7..c9cba892540 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -36,8 +36,8 @@ func (b *BadgerV4) View(fn func(txn Txn) error) error { }) } -func (b *BadgerV4) NewTxn(update bool) Txn { - return &BadgerV4Txn{b.DB.NewTxn(update)} +func (b *BadgerV4) NewTransaction(update bool) Txn { + return &BadgerV4Txn{b.DB.NewTransaction(update)} } func (b *BadgerV4) RunValueLogGC(discardRatio float64) error { @@ -69,6 +69,34 @@ func (b *BadgerV4) IndexCacheMetrics() *ristretto.Metrics { return b.DB.IndexCacheMetrics() } +func (b *BadgerV4) GetErrKeyNotFound() error { + return badger.ErrKeyNotFound +} + +func (b *BadgerV4) NewWriteBatch() WriteBatch { + return &BadgerV4WriteBatch{b.DB.NewWriteBatch()} +} + +type BadgerV4WriteBatch struct { + *badger.WriteBatch +} + +func (wb *BadgerV4WriteBatch) Set(key, val []byte) error { + return wb.WriteBatch.Set(key, val) +} + +func (wb *BadgerV4WriteBatch) Delete(key []byte) error { + return wb.WriteBatch.Delete(key) +} + +func (wb *BadgerV4WriteBatch) Flush() error { + return wb.WriteBatch.Flush() +} + +func (wb *BadgerV4WriteBatch) Cancel() { + wb.WriteBatch.Cancel() +} + type BadgerV4Stream struct { *badger.Stream } @@ -136,3 +164,11 @@ func (item *BadgerV4Item) Key() []byte { func (item *BadgerV4Item) Version() uint64 { return item.Item.Version() } + +func (item *BadgerV4Item) ValueCopy(dst []byte) ([]byte, error) { + return item.Item.ValueCopy(dst) +} + +func (item *BadgerV4Item) ValueSize() int64 { + return item.Item.ValueSize() +} diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 2971a4e9199..c1ab9da5bed 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -153,6 +153,10 @@ # env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE EnableSplitstore = true + # type: int + # env var: LOTUS_CHAINSTORE_BADGERVERSION + #BadgerVersion = 2 + [Chainstore.Splitstore] # ColdStoreType specifies the type of the coldstore. # It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state.. diff --git a/go.mod b/go.mod index 1bb7acf78c6..1a45f2aa97e 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,10 @@ require ( github.com/containerd/cgroups v1.1.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e + github.com/dgraph-io/badger v1.6.2 github.com/dgraph-io/badger/v2 v2.2007.4 + github.com/dgraph-io/badger/v4 v4.2.0 + github.com/dgraph-io/ristretto v0.1.1 github.com/docker/go-units v0.5.0 github.com/drand/drand v1.5.11 github.com/drand/kyber v1.3.1 @@ -182,7 +185,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/drand/kyber-bls12381 v0.3.1 // indirect github.com/elastic/go-windows v1.0.0 // indirect @@ -209,6 +211,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect diff --git a/go.sum b/go.sum index 1da4af986c6..8623f99c652 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,9 @@ github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrV github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= @@ -454,6 +457,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= diff --git a/node/config/def.go b/node/config/def.go index d8b8e0babb3..5af777992e7 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -69,6 +69,7 @@ func DefaultFullNode() *FullNode { Chainstore: Chainstore{ EnableSplitstore: true, + BadgerVersion: 2, Splitstore: Splitstore{ ColdStoreType: "discard", HotStoreType: "badger", diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index a3360627741..39ae079fc09 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -82,6 +82,12 @@ your node if metadata log is disabled`, Name: "Splitstore", Type: "Splitstore", + Comment: ``, + }, + { + Name: "BadgerVersion", + Type: "int", + Comment: ``, }, }, diff --git a/node/config/types.go b/node/config/types.go index fdda3b84f7a..03e4539353c 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -484,6 +484,7 @@ type Pubsub struct { type Chainstore struct { EnableSplitstore bool Splitstore Splitstore + BadgerVersion int } type Splitstore struct { From b077093ba673d4400960593f069f4914bdf87af5 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 24 Jul 2024 17:16:41 -1000 Subject: [PATCH 04/59] wip --- blockstore/badger/blockstore.go | 30 ++++--------------- blockstore/badger/versions/badger.go | 19 ++++++++++++ .../badger/versions/badger_interface.go | 4 +++ blockstore/badger/versions/badger_v2.go | 12 ++++++++ blockstore/badger/versions/badger_v4.go | 17 +++++++++++ 5 files changed, 58 insertions(+), 24 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index 5e004c1fe35..e1f2cfc6ee6 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -17,7 +17,6 @@ import ( logger "github.com/ipfs/go-log/v2" pool "github.com/libp2p/go-buffer-pool" "github.com/multiformats/go-base32" - "go.uber.org/zap" "golang.org/x/xerrors" badger "github.com/filecoin-project/lotus/blockstore/badger/versions" @@ -43,19 +42,6 @@ var ( log = logger.Logger("badgerbs") ) -// badgerLogger is a local wrapper for go-log to make the interface -// compatible with badger.Logger (namely, aliasing Warnf to Warningf) -type badgerLogger struct { - *zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options. - - skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger. -} - -// Warningf is required by the badger logger APIs. -func (b *badgerLogger) Warningf(format string, args ...interface{}) { - b.skip2.Warnf(format, args...) -} - // bsState is the current blockstore state type bsState int @@ -111,10 +97,6 @@ var _ io.Closer = (*Blockstore)(nil) // Open creates a new badger-backed blockstore, with the supplied options. func Open(opts badger.Options) (*Blockstore, error) { - opts.Logger = &badgerLogger{ - SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), - skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), - } db, err := badger.OpenBadgerDB(opts) if err != nil { @@ -370,7 +352,7 @@ func symlink(path, linkTo string) error { // doCopy copies a badger blockstore to another, with an optional filter; if the filter // is not nil, then only cids that satisfy the filter will be copied. -func (b *Blockstore) doCopy(from, to *badger.BadgerDB) error { +func (b *Blockstore) doCopy(from, to badger.BadgerDB) error { workers := runtime.NumCPU() / 2 if workers < 2 { workers = 2 @@ -380,8 +362,8 @@ func (b *Blockstore) doCopy(from, to *badger.BadgerDB) error { } stream := from.NewStream() - stream.NumGo = workers - stream.LogPrefix = "doCopy" + stream.SetNumGo(workers) + stream.SetLogPrefix("doCopy") stream.Send = func(list *pb.KVList) error { batch := to.NewWriteBatch() defer batch.Cancel() @@ -454,7 +436,7 @@ func (b *Blockstore) onlineGC(ctx context.Context, threshold float64, checkFreq } } - if err == badger.ErrNoRewrite { + if err == b.db.GetErrNoRewrite() { // not really an error in this case, it signals the end of GC return nil } @@ -527,7 +509,7 @@ func (b *Blockstore) GCOnce(ctx context.Context, opts ...blockstore.BlockstoreGC // Note no compaction needed before single GC as we will hit at most one vlog anyway err := b.db.RunValueLogGC(threshold) - if err == badger.ErrNoRewrite { + if err == b.db.GetErrNoRewrite() { // not really an error in this case, it signals the end of GC return nil } @@ -1084,6 +1066,6 @@ func (b *Blockstore) StorageKey(dst []byte, cid cid.Cid) []byte { // this method is added for lotus-shed needs // WARNING: THIS IS COMPLETELY UNSAFE; DONT USE THIS IN PRODUCTION CODE -func (b *Blockstore) DB() *badger.BadgerDB { +func (b *Blockstore) DB() badger.BadgerDB { return b.db } diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 11caf36ac07..564decd66e1 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -5,18 +5,37 @@ import ( badgerV2 "github.com/dgraph-io/badger/v2" badgerV4 "github.com/dgraph-io/badger/v4" + "go.uber.org/zap" ) // Options embeds the badger options themselves, and augments them with // blockstore-specific options. type Options struct { + + badgerV2.Options + // BadgerVersion sets the release version of badger to use BadgerVersion int // Prefix is an optional prefix to prepend to keys. Default: "". Prefix string + + BadgerLogger badgerLogger +} + + +// badgerLogger is a local wrapper for go-log to make the interface +// compatible with badger.Logger (namely, aliasing Warnf to Warningf) +type badgerLogger struct { + *zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options. + + skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger. } +// Warningf is required by the badger logger APIs. +func (b *badgerLogger) Warningf(format string, args ...interface{}) { + b.skip2.Warnf(format, args...) +} func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 46f34077330..515b7817d5f 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -24,7 +24,11 @@ type BadgerDB interface { BlockCacheMetrics() *ristretto.Metrics IndexCacheMetrics() *ristretto.Metrics GetErrKeyNotFound() error + GetErrNoRewrite() error NewWriteBatch() WriteBatch + Flatten(workers int) error + Size() (lsm int64, vlog int64) + } // BadgerStream defines the common interface for streaming data in Badger. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index e05b221d6f0..64264042ae4 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -73,10 +73,22 @@ func (b *BadgerV2) GetErrKeyNotFound() error { return badger.ErrKeyNotFound } +func (b *BadgerV2) GetErrNoRewrite() error { + return badger.ErrNoRewrite +} + func (b *BadgerV2) NewWriteBatch() WriteBatch { return &BadgerV2WriteBatch{b.DB.NewWriteBatch()} } +func (b *BadgerV2) Flatten(workers int) error{ + return b.DB.Flatten(workers) +} + +func (b *BadgerV2) Size() (lsm int64, vlog int64){ +return b.DB.Size() +} + type BadgerV2WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index c9cba892540..e2cca3dd527 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -73,14 +73,31 @@ func (b *BadgerV4) GetErrKeyNotFound() error { return badger.ErrKeyNotFound } + +func (b *BadgerV4) GetErrNoRewrite() error { + return badger.ErrNoRewrite +} + func (b *BadgerV4) NewWriteBatch() WriteBatch { return &BadgerV4WriteBatch{b.DB.NewWriteBatch()} } +func (b *BadgerV4) Flatten(workers int) error{ + return b.DB.Flatten(workers) +} + +func (b *BadgerV4) Size() (lsm int64, vlog int64){ +return b.DB.Size() +} + + + type BadgerV4WriteBatch struct { *badger.WriteBatch } + + func (wb *BadgerV4WriteBatch) Set(key, val []byte) error { return wb.WriteBatch.Set(key, val) } From 0e14b09505221f7055a6d6ad46349964bfde4b47 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 26 Jul 2024 17:47:20 -1000 Subject: [PATCH 05/59] wip --- blockstore/badger/blockstore.go | 33 +-------- blockstore/badger/versions/badger.go | 11 ++- .../badger/versions/badger_interface.go | 21 +++++- blockstore/badger/versions/badger_v2.go | 56 +++++++++++++--- blockstore/badger/versions/badger_v4.go | 67 ++++++++++++++----- 5 files changed, 127 insertions(+), 61 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index e1f2cfc6ee6..55e61002acb 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -10,7 +10,6 @@ import ( "sync" "time" - "github.com/dgraph-io/badger/pb" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -19,6 +18,7 @@ import ( "github.com/multiformats/go-base32" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/blockstore/badger/versions" badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/filecoin-project/lotus/blockstore" @@ -352,35 +352,8 @@ func symlink(path, linkTo string) error { // doCopy copies a badger blockstore to another, with an optional filter; if the filter // is not nil, then only cids that satisfy the filter will be copied. -func (b *Blockstore) doCopy(from, to badger.BadgerDB) error { - workers := runtime.NumCPU() / 2 - if workers < 2 { - workers = 2 - } - if workers > 8 { - workers = 8 - } - - stream := from.NewStream() - stream.SetNumGo(workers) - stream.SetLogPrefix("doCopy") - stream.Send = func(list *pb.KVList) error { - batch := to.NewWriteBatch() - defer batch.Cancel() - - for _, kv := range list.Kv { - if kv.Key == nil || kv.Value == nil { - continue - } - if err := batch.Set(kv.Key, kv.Value); err != nil { - return err - } - } - - return batch.Flush() - } - - return stream.Orchestrate(context.Background()) +func (b *Blockstore) doCopy(from versions.BadgerDB, to versions.BadgerDB) error { + return from.Copy(to) } func (b *Blockstore) deleteDB(path string) { diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 564decd66e1..30a364e4d04 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -23,7 +23,6 @@ type Options struct { BadgerLogger badgerLogger } - // badgerLogger is a local wrapper for go-log to make the interface // compatible with badger.Logger (namely, aliasing Warnf to Warningf) type badgerLogger struct { @@ -67,3 +66,13 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { } return db, nil } + +func clamp(x, min, max int) int { + if x < min { + return min + } + if x > max { + return max + } + return x +} diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 515b7817d5f..d6b0c5c6f32 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -27,15 +27,15 @@ type BadgerDB interface { GetErrNoRewrite() error NewWriteBatch() WriteBatch Flatten(workers int) error - Size() (lsm int64, vlog int64) - + Size() (lsm int64, vlog int64) + Copy(to BadgerDB) error } // BadgerStream defines the common interface for streaming data in Badger. type BadgerStream interface { SetNumGo(numGo int) SetLogPrefix(prefix string) - Send(buf *Buffer) error + Orchestrate(ctx context.Context) error } @@ -46,6 +46,21 @@ type Txn interface { Delete(key []byte) error Commit() error Discard() + NewIterator(opts IteratorOptions) Iterator +} + +type IteratorOptions struct { + PrefetchSize int + Prefix []byte +} + +type Iterator interface { + Next() + Rewind() + Seek(key []byte) + Close() + Item() Item + Valid() bool } // Item defines the common interface for items in a transaction. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index 64264042ae4..aba28d97e9b 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -2,9 +2,10 @@ package versions import ( "context" - "fmt" + "runtime" "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" ) @@ -81,12 +82,35 @@ func (b *BadgerV2) NewWriteBatch() WriteBatch { return &BadgerV2WriteBatch{b.DB.NewWriteBatch()} } -func (b *BadgerV2) Flatten(workers int) error{ +func (b *BadgerV2) Flatten(workers int) error { return b.DB.Flatten(workers) } -func (b *BadgerV2) Size() (lsm int64, vlog int64){ -return b.DB.Size() +func (b *BadgerV2) Size() (lsm int64, vlog int64) { + return b.DB.Size() +} + +func (b *BadgerV2) Copy(to BadgerDB) error { + stream := b.DB.NewStream() + stream.LogPrefix = "doCopy" + stream.NumGo = clamp(runtime.NumCPU()/2, 2, 8) + stream.Send = func(list *pb.KVList) error { + batch := to.NewWriteBatch() + defer batch.Cancel() + + for _, kv := range list.Kv { + if kv.Key == nil || kv.Value == nil { + continue + } + if err := batch.Set(kv.Key, kv.Value); err != nil { + return err + } + } + + return batch.Flush() + } + + return stream.Orchestrate(context.Background()) } type BadgerV2WriteBatch struct { @@ -121,12 +145,6 @@ func (s *BadgerV2Stream) SetLogPrefix(prefix string) { s.LogPrefix = prefix } -func (s *BadgerV2Stream) Send(buf *Buffer) error { - /* ??? */ - fmt.Println("MIKE") - return nil -} - func (s *BadgerV2Stream) Orchestrate(ctx context.Context) error { return s.Stream.Orchestrate(ctx) } @@ -156,6 +174,24 @@ func (txn *BadgerV2Txn) Discard() { txn.Txn.Discard() } +func (txn *BadgerV2Txn) NewIterator(opts IteratorOptions) Iterator { + badgerOpts := badger.DefaultIteratorOptions + badgerOpts.PrefetchSize = opts.PrefetchSize + badgerOpts.Prefix = opts.Prefix + return &BadgerV2Iterator{txn.Txn.NewIterator(badgerOpts)} +} + +type BadgerV2Iterator struct { + *badger.Iterator +} + +func (it *BadgerV2Iterator) Next() { it.Iterator.Next() } +func (it *BadgerV2Iterator) Rewind() { it.Iterator.Rewind() } +func (it *BadgerV2Iterator) Seek(key []byte) { it.Iterator.Seek(key) } +func (it *BadgerV2Iterator) Close() { it.Iterator.Close() } +func (it *BadgerV2Iterator) Item() Item { return &BadgerV2Item{it.Iterator.Item()} } +func (it *BadgerV2Iterator) Valid() bool { return it.Iterator.Valid() } + type BadgerV2Item struct { *badger.Item } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index e2cca3dd527..c624c26878e 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -2,9 +2,12 @@ package versions import ( "context" + "fmt" + "runtime" "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/ristretto" + "github.com/dgraph-io/ristretto/z" ) // BadgerV4 wraps the Badger v4 database to implement the BadgerDB interface. @@ -73,7 +76,6 @@ func (b *BadgerV4) GetErrKeyNotFound() error { return badger.ErrKeyNotFound } - func (b *BadgerV4) GetErrNoRewrite() error { return badger.ErrNoRewrite } @@ -82,22 +84,46 @@ func (b *BadgerV4) NewWriteBatch() WriteBatch { return &BadgerV4WriteBatch{b.DB.NewWriteBatch()} } -func (b *BadgerV4) Flatten(workers int) error{ +func (b *BadgerV4) Flatten(workers int) error { return b.DB.Flatten(workers) } -func (b *BadgerV4) Size() (lsm int64, vlog int64){ -return b.DB.Size() +func (b *BadgerV4) Size() (lsm int64, vlog int64) { + return b.DB.Size() } +func (b *BadgerV4) Copy(to BadgerDB) error { + stream := b.DB.NewStream() + stream.LogPrefix = "doCopy" + stream.NumGo = clamp(runtime.NumCPU()/2, 2, 8) + stream.Send = func(buf *z.Buffer) error { + list, err := badger.BufferToKVList(buf) + if err != nil { + return fmt.Errorf("buffer to KV list conversion: %w", err) + } + + batch := to.NewWriteBatch() + defer batch.Cancel() + + for _, kv := range list.Kv { + if kv.Key == nil || kv.Value == nil { + continue + } + if err := batch.Set(kv.Key, kv.Value); err != nil { + return err + } + } + + return batch.Flush() + } + return stream.Orchestrate(context.Background()) +} type BadgerV4WriteBatch struct { *badger.WriteBatch } - - func (wb *BadgerV4WriteBatch) Set(key, val []byte) error { return wb.WriteBatch.Set(key, val) } @@ -126,17 +152,6 @@ func (s *BadgerV4Stream) SetLogPrefix(prefix string) { s.LogPrefix = prefix } -func (s *BadgerV4Stream) Send(buf *Buffer) error { - return nil - /* - list, err := badger.BufferToKVList(&buf.buf) - if err != nil { - return err - } - return s.Stream.Send(list) - */ -} - func (s *BadgerV4Stream) Orchestrate(ctx context.Context) error { return s.Stream.Orchestrate(ctx) } @@ -166,6 +181,24 @@ func (txn *BadgerV4Txn) Discard() { txn.Txn.Discard() } +func (txn *BadgerV4Txn) NewIterator(opts IteratorOptions) Iterator { + badgerOpts := badger.DefaultIteratorOptions + badgerOpts.PrefetchSize = opts.PrefetchSize + badgerOpts.Prefix = opts.Prefix + return &BadgerV4Iterator{txn.Txn.NewIterator(badgerOpts)} +} + +type BadgerV4Iterator struct { + *badger.Iterator +} + +func (it *BadgerV4Iterator) Next() { it.Iterator.Next() } +func (it *BadgerV4Iterator) Rewind() { it.Iterator.Rewind() } +func (it *BadgerV4Iterator) Seek(key []byte) { it.Iterator.Seek(key) } +func (it *BadgerV4Iterator) Close() { it.Iterator.Close() } +func (it *BadgerV4Iterator) Item() Item { return &BadgerV4Item{it.Iterator.Item()} } +func (it *BadgerV4Iterator) Valid() bool { return it.Iterator.Valid() } + type BadgerV4Item struct { *badger.Item } From a4b7538c089c03cd41f225a54e938bb298fdb9b3 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 26 Jul 2024 18:57:48 -1000 Subject: [PATCH 06/59] options --- blockstore/badger/versions/badger.go | 10 ++-- .../badger/versions/badger_interface.go | 1 + blockstore/badger/versions/badger_v2.go | 27 +++++++++ blockstore/badger/versions/badger_v4.go | 20 +++++++ blockstore/badger/versions/options.go | 49 ++++++++++++++++ node/repo/blockstore_opts.go | 57 +------------------ 6 files changed, 106 insertions(+), 58 deletions(-) create mode 100644 blockstore/badger/versions/options.go diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 30a364e4d04..d9b04ca85a3 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -11,14 +11,16 @@ import ( // Options embeds the badger options themselves, and augments them with // blockstore-specific options. type Options struct { - - badgerV2.Options + V2Options *badgerV2.Options + V4Options *badgerV4.Options // BadgerVersion sets the release version of badger to use BadgerVersion int - // Prefix is an optional prefix to prepend to keys. Default: "". - Prefix string + Prefix string + Dir string + ValueDir string + SyncWrites bool BadgerLogger badgerLogger } diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index d6b0c5c6f32..e49e022f6fe 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -29,6 +29,7 @@ type BadgerDB interface { Flatten(workers int) error Size() (lsm int64, vlog int64) Copy(to BadgerDB) error + DefaultOptions(prefix string, readonly bool) Options } // BadgerStream defines the common interface for streaming data in Badger. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index aba28d97e9b..281f1a1ad1f 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -2,9 +2,12 @@ package versions import ( "context" + "os" "runtime" + "strconv" "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" ) @@ -113,6 +116,30 @@ func (b *BadgerV2) Copy(to BadgerDB) error { return stream.Orchestrate(context.Background()) } +func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { + var opts Options + opts.Prefix = "/blocks/" + + bopts := badger.DefaultOptions(path) + bopts.DetectConflicts = false + bopts.CompactL0OnClose = true + bopts.Truncate = true + bopts.ValueLogLoadingMode = options.MemoryMap + bopts.TableLoadingMode = options.MemoryMap + bopts.ValueThreshold = 128 + bopts.MaxTableSize = 64 << 20 + bopts.ReadOnly = readonly + + // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { + if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { + bopts.NumCompactors = numWorkers + } + } + opts.V2Options = &bopts + return opts +} + type BadgerV2WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index c624c26878e..1325931f725 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -3,9 +3,12 @@ package versions import ( "context" "fmt" + "os" "runtime" + "strconv" "github.com/dgraph-io/badger/v4" + badgerV4 "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" ) @@ -120,6 +123,23 @@ func (b *BadgerV4) Copy(to BadgerDB) error { return stream.Orchestrate(context.Background()) } +func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { + var opts Options + bopts := badgerV4.DefaultOptions(path) + bopts.ReadOnly = readonly + + // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { + if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { + bopts.NumCompactors = numWorkers + } + } + opts.V4Options = &bopts + opts.Prefix = "/blocks/" + return opts + +} + type BadgerV4WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go new file mode 100644 index 00000000000..04e7a3468fe --- /dev/null +++ b/blockstore/badger/versions/options.go @@ -0,0 +1,49 @@ +package versions + +import ( + "os" + "strconv" + + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" + badgerV4 "github.com/dgraph-io/badger/v4" +) + +func DefaultOptions(path string, readonly bool) Options { + var opts Options + opts.Prefix = "/blocks/" + + //v2 + bopts := badger.DefaultOptions(path) + bopts.DetectConflicts = false + bopts.CompactL0OnClose = true + bopts.Truncate = true + bopts.ValueLogLoadingMode = options.MemoryMap + bopts.TableLoadingMode = options.MemoryMap + bopts.ValueThreshold = 128 + bopts.MaxTableSize = 64 << 20 + bopts.ReadOnly = readonly + + // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { + if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { + bopts.NumCompactors = numWorkers + } + } + opts.V2Options = &bopts + + //v4 + + boptsv4 := badgerV4.DefaultOptions(path) + boptsv4.ReadOnly = readonly + + // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { + if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { + boptsv4.NumCompactors = numWorkers + } + } + opts.V4Options = &boptsv4 + + return opts +} diff --git a/node/repo/blockstore_opts.go b/node/repo/blockstore_opts.go index 81f8b9ff416..33839e16ef7 100644 --- a/node/repo/blockstore_opts.go +++ b/node/repo/blockstore_opts.go @@ -1,63 +1,12 @@ package repo import ( - "os" - "strconv" - - badgerbs "github.com/filecoin-project/lotus/blockstore/badger" + versions "github.com/filecoin-project/lotus/blockstore/badger/versions" ) // BadgerBlockstoreOptions returns the badger options to apply for the provided // domain. -func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (badgerbs.Options, error) { - opts := badgerbs.DefaultOptions(path) - - // Due to legacy usage of blockstore.Blockstore, over a datastore, all - // blocks are prefixed with this namespace. In the future, this can go away, - // in order to shorten keys, but it'll require a migration. - opts.Prefix = "/blocks/" - - // Blockstore values are immutable; therefore we do not expect any - // conflicts to emerge. - opts.DetectConflicts = false - - // This is to optimize the database on close so it can be opened - // read-only and efficiently queried. - opts.CompactL0OnClose = true - - // The alternative is "crash on start and tell the user to fix it". This - // will truncate corrupt and unsynced data, which we don't guarantee to - // persist anyways. - opts.Truncate = true - - // We mmap the index and the value logs; this is important to enable - // zero-copy value access. - opts.ValueLogLoadingMode = badgerbs.MemoryMap - opts.TableLoadingMode = badgerbs.MemoryMap - - // Embed only values < 128 bytes in the LSM tree; larger values are stored - // in value logs. - opts.ValueThreshold = 128 - - // Default table size is already 64MiB. This is here to make it explicit. - opts.MaxTableSize = 64 << 20 - - // NOTE: The chain blockstore doesn't require any GC (blocks are never - // deleted). This will change if we move to a tiered blockstore. - - opts.ReadOnly = readonly - - // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM - // Allows the number of compaction workers used by BadgerDB to be adjusted - // Unset - leaves the default number of compaction workers (4) - // "0" - disables compaction - // Positive integer - enables that number of compaction workers - if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { - if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { - opts.NumCompactors = numWorkers - } - } - +func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (versions.Options, error) { + opts := versions.DefaultOptions(path, readonly) return opts, nil - } From 6c4f012bc26efba6d5bb52cd1b0c013b96ea4c84 Mon Sep 17 00:00:00 2001 From: Mikers Date: Fri, 26 Jul 2024 18:59:05 -1000 Subject: [PATCH 07/59] clean --- blockstore/badger/blockstore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index 55e61002acb..c446fe1ecdb 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -78,7 +78,6 @@ type Blockstore struct { moveState bsMoveState rlock int - //change this db badger.BadgerDB dbNext badger.BadgerDB // when moving opts badger.Options From 4adc08746e34044b046863fb8f4d7f0f821b95cf Mon Sep 17 00:00:00 2001 From: Mikers Date: Sat, 27 Jul 2024 08:18:50 -1000 Subject: [PATCH 08/59] docsgen --- blockstore/badger/blockstore.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index 4da21731f8d..6cb3227584a 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -18,10 +18,9 @@ import ( "github.com/multiformats/go-base32" "golang.org/x/xerrors" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/blockstore/badger/versions" badger "github.com/filecoin-project/lotus/blockstore/badger/versions" - - "github.com/filecoin-project/lotus/blockstore" ) // aliases to mask badger dependencies. From a3c1c2d171f21267f507efc7901b57320b56801c Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 08:56:30 -1000 Subject: [PATCH 09/59] progress on options/logger --- blockstore/badger/versions/badger.go | 12 +++++----- .../badger/versions/badger_interface.go | 3 +++ blockstore/badger/versions/badger_v2.go | 9 ++++++++ blockstore/badger/versions/badger_v4.go | 9 ++++++++ blockstore/splitstore/markset_badger.go | 4 ++-- cmd/lotus-shed/datastore.go | 11 +++++----- cmd/lotus-shed/export.go | 16 +++++++------- cmd/lotus-shed/splitstore.go | 22 +++++-------------- 8 files changed, 49 insertions(+), 37 deletions(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index d9b04ca85a3..524082a4bf2 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -22,20 +22,20 @@ type Options struct { ValueDir string SyncWrites bool - BadgerLogger badgerLogger + Logger BadgerLogger } -// badgerLogger is a local wrapper for go-log to make the interface +// BadgerLogger is a local wrapper for go-log to make the interface // compatible with badger.Logger (namely, aliasing Warnf to Warningf) -type badgerLogger struct { +type BadgerLogger struct { *zap.SugaredLogger // skips 1 caller to get useful line info, skipping over badger.Options. - skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger. + Skip2 *zap.SugaredLogger // skips 2 callers, just like above + this logger. } // Warningf is required by the badger logger APIs. -func (b *badgerLogger) Warningf(format string, args ...interface{}) { - b.skip2.Warnf(format, args...) +func (b *BadgerLogger) Warningf(format string, args ...interface{}) { + b.Skip2.Warnf(format, args...) } func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index e49e022f6fe..52a9d28fb82 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -2,6 +2,7 @@ package versions import ( "context" + "io" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" @@ -30,6 +31,8 @@ type BadgerDB interface { Size() (lsm int64, vlog int64) Copy(to BadgerDB) error DefaultOptions(prefix string, readonly bool) Options + Backup(w io.Writer, since uint64) (uint64, error) + Load(r io.Reader, maxPendingWrites int) error } // BadgerStream defines the common interface for streaming data in Badger. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index 281f1a1ad1f..b3655cc68f3 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -2,6 +2,7 @@ package versions import ( "context" + "io" "os" "runtime" "strconv" @@ -140,6 +141,14 @@ func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { return opts } +func (b *BadgerV2) Backup(w io.Writer, since uint64) (uint64, error) { + return b.DB.Backup(w, since) +} + +func (b *BadgerV2) Load(r io.Reader, maxPendingWrites int) error { + return b.DB.Load(r, maxPendingWrites) +} + type BadgerV2WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 1325931f725..4322ae60c16 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -3,6 +3,7 @@ package versions import ( "context" "fmt" + "io" "os" "runtime" "strconv" @@ -140,6 +141,14 @@ func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { } +func (b *BadgerV4) Backup(w io.Writer, since uint64) (uint64, error) { + return b.DB.Backup(w, since) +} + +func (b *BadgerV4) Load(r io.Reader, maxPendingWrites int) error { + return b.DB.Load(r, maxPendingWrites) +} + type BadgerV4WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 2dac673cd76..dbb12a28091 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -416,7 +416,7 @@ func openBadgerDB(path string, recover bool) (*badger.DB, error) { } opts.Logger = &badgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), - skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), + Skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } return badger.Open(opts) @@ -443,7 +443,7 @@ func closeBadgerDB(db *badger.DB, path string, persist bool) error { // badger logging through go-log type badgerLogger struct { *zap.SugaredLogger - skip2 *zap.SugaredLogger + Skip2 *zap.SugaredLogger } func (b *badgerLogger) Warningf(format string, args ...interface{}) {} diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 8e31ccc3c44..85ee6e36caf 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -9,8 +9,8 @@ import ( "os" "strings" - "github.com/dgraph-io/badger/v2" "github.com/docker/go-units" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log/v2" @@ -321,8 +321,8 @@ var datastoreRewriteCmd = &cli.Command{ } var ( - from *badger.DB - to *badger.DB + from badger.BadgerDB + to badger.BadgerDB ) // open the destination (to) store. @@ -331,7 +331,8 @@ var datastoreRewriteCmd = &cli.Command{ return xerrors.Errorf("failed to get badger options: %w", err) } opts.SyncWrites = false - if to, err = badger.Open(opts.Options); err != nil { + if to, err = badger.OpenBadgerDB(opts); err != nil { + return xerrors.Errorf("opening 'to' badger store: %w", err) } @@ -340,7 +341,7 @@ var datastoreRewriteCmd = &cli.Command{ if err != nil { return xerrors.Errorf("failed to get badger options: %w", err) } - if from, err = badger.Open(opts.Options); err != nil { + if from, err = badger.OpenBadgerDB(opts); err != nil { return xerrors.Errorf("opening 'from' datastore: %w", err) } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index e6d0c4e056f..a02de0bccb7 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -12,9 +12,9 @@ import ( "strings" "sync" - "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/pb" "github.com/dustin/go-humanize" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/boxo/blockservice" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/ipld/merkledag" @@ -240,14 +240,14 @@ var exportRawCmd = &cli.Command{ return err } - opts.Logger = &badgerLog{ + opts.Logger = badger.BadgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), - skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), + Skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } log.Infow("open db") - db, err := badger.Open(opts.Options) + db, err := badger.OpenBadgerDB(opts) if err != nil { return fmt.Errorf("failed to open badger blockstore: %w", err) } @@ -258,8 +258,8 @@ var exportRawCmd = &cli.Command{ var wlk sync.Mutex str := db.NewStream() - str.NumGo = 16 - str.LogPrefix = "bstream" + str.SetNumGo(16) + str.SetLogPrefix("bstream") str.Send = func(list *pb.KVList) (err error) { defer func() { if err != nil { @@ -482,9 +482,9 @@ var _ blockstore.Blockstore = &rawCarb{} type badgerLog struct { *zap.SugaredLogger - skip2 *zap.SugaredLogger + Skip2 *zap.SugaredLogger } func (b *badgerLog) Warningf(format string, args ...interface{}) { - b.skip2.Warnf(format, args...) + b.Skip2.Warnf(format, args...) } diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index e8c45a0c5e8..e0548710a8a 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -9,7 +9,7 @@ import ( "path/filepath" "runtime" - "github.com/dgraph-io/badger/v2" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/urfave/cli/v2" @@ -195,9 +195,9 @@ func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error { coldPath := filepath.Join(dataPath, "chain") hotPath := filepath.Join(dataPath, "splitstore", "hot.badger") - blog := &badgerLogger{ + blog := badger.BadgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), - skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), + Skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } coldOpts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, coldPath, false) @@ -213,13 +213,13 @@ func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error { } hotOpts.Logger = blog - cold, err := badger.Open(coldOpts.Options) + cold, err := badger.OpenBadgerDB(coldOpts) if err != nil { return xerrors.Errorf("error opening coldstore: %w", err) } defer cold.Close() //nolint - hot, err := badger.Open(hotOpts.Options) + hot, err := badger.OpenBadgerDB(hotOpts) if err != nil { return xerrors.Errorf("error opening hotstore: %w", err) } @@ -278,7 +278,7 @@ func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error { err = cold.RunValueLogGC(0.0625) } - if err != badger.ErrNoRewrite { + if err != cold.GetErrNoRewrite() { return xerrors.Errorf("error garbage collecting coldstore: %w", err) } } @@ -353,16 +353,6 @@ func deleteSplitstoreKeys(lr repo.LockedRepo) error { return nil } -// badger logging through go-log -type badgerLogger struct { - *zap.SugaredLogger - skip2 *zap.SugaredLogger -} - -func (b *badgerLogger) Warningf(format string, args ...interface{}) {} -func (b *badgerLogger) Infof(format string, args ...interface{}) {} -func (b *badgerLogger) Debugf(format string, args ...interface{}) {} - var splitstoreCheckCmd = &cli.Command{ Name: "check", Description: "runs a healthcheck on a splitstore installation", From da5f823fa718e5701c92272eb9fbde7fe6a3afb0 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 10:34:03 -1000 Subject: [PATCH 10/59] last usage of Send wrapped using ForEach --- .../badger/versions/badger_interface.go | 15 +++-- blockstore/badger/versions/badger_v2.go | 25 +++++-- blockstore/badger/versions/badger_v4.go | 29 +++++++-- cmd/lotus-shed/export.go | 65 ++++++++----------- 4 files changed, 82 insertions(+), 52 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 52a9d28fb82..99d64fb55fc 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -4,7 +4,6 @@ import ( "context" "io" - "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" ) @@ -21,7 +20,6 @@ type BadgerDB interface { Sync() error MaxBatchCount() int64 MaxBatchSize() int64 - Subscribe(ctx context.Context, cb func(kv *KVList) error, prefixes ...[]byte) error BlockCacheMetrics() *ristretto.Metrics IndexCacheMetrics() *ristretto.Metrics GetErrKeyNotFound() error @@ -41,6 +39,7 @@ type BadgerStream interface { SetLogPrefix(prefix string) Orchestrate(ctx context.Context) error + ForEach(ctx context.Context, fn func(key string, value string) error) error } // Txn defines the common interface for transactions in Badger. @@ -76,9 +75,6 @@ type Item interface { ValueSize() int64 } -// KVList is an alias for the KVList type from the Badger package. -type KVList = pb.KVList - type Buffer struct { kvList KVList buf z.Buffer @@ -90,3 +86,12 @@ type WriteBatch interface { Flush() error Cancel() } + +type KVList interface { + GetKV() []*KV +} + +type KV struct { + Key []byte + Value []byte +} diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index b3655cc68f3..996c3615ef8 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -11,6 +11,7 @@ import ( "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" + "golang.org/x/xerrors" ) // BadgerV2 wraps the Badger v2 database to implement the BadgerDB interface. @@ -62,10 +63,6 @@ func (b *BadgerV2) MaxBatchSize() int64 { return b.DB.MaxBatchSize() } -func (b *BadgerV2) Subscribe(ctx context.Context, cb func(kv *KVList) error, prefixes ...[]byte) error { - return b.DB.Subscribe(ctx, cb, prefixes...) -} - func (b *BadgerV2) BlockCacheMetrics() *ristretto.Metrics { return b.DB.BlockCacheMetrics() } @@ -185,6 +182,26 @@ func (s *BadgerV2Stream) Orchestrate(ctx context.Context) error { return s.Stream.Orchestrate(ctx) } +func (s *BadgerV2Stream) ForEach(ctx context.Context, fn func(key string, value string) error) error { + s.Send = func(list *pb.KVList) error { + for _, kv := range list.Kv { + if kv.Key == nil || kv.Value == nil { + continue + } + err := fn(string(kv.Key), string(kv.Value)) + if err != nil { + return xerrors.Errorf("foreach function: %w", err) + } + + } + return nil + } + if err := s.Orchestrate(ctx); err != nil { + return xerrors.Errorf("orchestrate stream: %w", err) + } + return nil +} + type BadgerV2Txn struct { *badger.Txn } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 4322ae60c16..c0e3b363d00 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -12,6 +12,7 @@ import ( badgerV4 "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" + "golang.org/x/xerrors" ) // BadgerV4 wraps the Badger v4 database to implement the BadgerDB interface. @@ -63,11 +64,6 @@ func (b *BadgerV4) MaxBatchSize() int64 { return b.DB.MaxBatchSize() } -func (b *BadgerV4) Subscribe(ctx context.Context, cb func(kv *KVList) error, prefixes ...[]byte) error { - //todo - return nil -} - func (b *BadgerV4) BlockCacheMetrics() *ristretto.Metrics { return b.DB.BlockCacheMetrics() } @@ -180,6 +176,29 @@ func (s *BadgerV4Stream) SetNumGo(numGo int) { func (s *BadgerV4Stream) SetLogPrefix(prefix string) { s.LogPrefix = prefix } +func (s *BadgerV4Stream) ForEach(ctx context.Context, fn func(key string, value string) error) error { + s.Send = func(buf *z.Buffer) error { + list, err := badger.BufferToKVList(buf) + if err != nil { + return fmt.Errorf("buffer to KV list conversion: %w", err) + } + for _, kv := range list.Kv { + if kv.Key == nil || kv.Value == nil { + continue + } + err := fn(string(kv.Key), string(kv.Value)) + if err != nil { + return xerrors.Errorf("foreach function: %w", err) + } + + } + return nil + } + if err := s.Orchestrate(ctx); err != nil { + return xerrors.Errorf("orchestrate stream: %w", err) + } + return nil +} func (s *BadgerV4Stream) Orchestrate(ctx context.Context) error { return s.Stream.Orchestrate(ctx) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index a02de0bccb7..201899296b7 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -12,7 +12,6 @@ import ( "strings" "sync" - "github.com/dgraph-io/badger/v2/pb" "github.com/dustin/go-humanize" badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/boxo/blockservice" @@ -260,48 +259,38 @@ var exportRawCmd = &cli.Command{ str := db.NewStream() str.SetNumGo(16) str.SetLogPrefix("bstream") - str.Send = func(list *pb.KVList) (err error) { - defer func() { - if err != nil { - log.Errorw("send error", "err", err) - } - }() - - for _, kv := range list.Kv { - if kv.Key == nil || kv.Value == nil { - continue - } - if !strings.HasPrefix(string(kv.Key), "/blocks/") { - log.Infow("no blocks prefix", "key", string(kv.Key)) - continue - } - - h, err := base32.RawStdEncoding.DecodeString(string(kv.Key[len("/blocks/"):])) - if err != nil { - return xerrors.Errorf("decode b32 ds key %x: %w", kv.Key, err) - } - - c := cid.NewCidV1(cid.Raw, h) - - b, err := block.NewBlockWithCid(kv.Value, c) - if err != nil { - return xerrors.Errorf("readblk: %w", err) - } - - wlk.Lock() - err = consume(c, b) - wlk.Unlock() - if err != nil { - return xerrors.Errorf("consume stream block: %w", err) - } + err = str.ForEach(ctx, func(key string, value string) error { + if !strings.HasPrefix(key, "/blocks/") { + log.Infow("no blocks prefix", "key", key) + return nil } - return nil - } + h, err := base32.RawStdEncoding.DecodeString(string(key[len("/blocks/"):])) + if err != nil { + return xerrors.Errorf("decode b32 ds key %x: %w", key, err) + } + + c := cid.NewCidV1(cid.Raw, h) + + b, err := block.NewBlockWithCid([]byte(value), c) + if err != nil { + return xerrors.Errorf("readblk: %w", err) + } + + wlk.Lock() + err = consume(c, b) + wlk.Unlock() + if err != nil { + return xerrors.Errorf("consume stream block: %w", err) + } - if err := str.Orchestrate(ctx); err != nil { + return nil + }) + if err != nil { + log.Errorw("send error", "err", err) return xerrors.Errorf("orchestrate stream: %w", err) } + } } From e4ffc68766e6b7ca054bbfa6828952dab4a1b935 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 10:37:22 -1000 Subject: [PATCH 11/59] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03836c83080..5b426a2c6a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ ## New features - feat: Add trace filter API supporting RPC method `trace_filter` ([filecoin-project/lotus#12123](https://github.com/filecoin-project/lotus/pull/12123)). Configuring `EthTraceFilterMaxResults` sets a limit on how many results are returned in any individual `trace_filter` RPC API call. +- feat: Add support for Badger version 4. The default remains Badger version 2 to ensure backward compatibility. ([filecoin-project/lotus#12316](https://github.com/filecoin-project/lotus/pull/12316)). Configuring `LOTUS_CHAINSTORE_BADGERVERSION` can configure lotus to use badger version 2 and 4. # v1.28.0 / 2024-07-23 This is the MANDATORY Lotus v1.28.0 release, which will deliver the Filecoin network version 23, codenamed Waffle 🧇. From b3ecb25899dea9f0c9796144ce5d2fe8f68f7517 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 10:39:17 -1000 Subject: [PATCH 12/59] go mod tidy --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 910104a1900..e9a06408b9a 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/containerd/cgroups v1.1.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e - github.com/dgraph-io/badger v1.6.2 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/ristretto v0.1.1 diff --git a/go.sum b/go.sum index f8f7b59be89..dcfb6f52039 100644 --- a/go.sum +++ b/go.sum @@ -200,7 +200,6 @@ github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdw github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= From 6aa56cc9c3d3b00d7803205567a193423fa2b5e8 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 11:28:40 -1000 Subject: [PATCH 13/59] default to 2 --- blockstore/badger/versions/badger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 524082a4bf2..6d08ce7ebf2 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -52,6 +52,7 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { if err == nil { db = BadgerDB(&BadgerV4{dbV4}) } + case 0: case 2: opts := badgerV2.DefaultOptions(prefix) var dbV2 *badgerV2.DB From 94f3ce4d55a3755c1734112939b2aefe1e7c4204 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 12:47:33 -1000 Subject: [PATCH 14/59] set default badger version to 2 --- blockstore/badger/versions/badger.go | 1 - blockstore/badger/versions/options.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 6d08ce7ebf2..524082a4bf2 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -52,7 +52,6 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { if err == nil { db = BadgerDB(&BadgerV4{dbV4}) } - case 0: case 2: opts := badgerV2.DefaultOptions(prefix) var dbV2 *badgerV2.DB diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 04e7a3468fe..22d64c71fdb 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -12,6 +12,7 @@ import ( func DefaultOptions(path string, readonly bool) Options { var opts Options opts.Prefix = "/blocks/" + opts.BadgerVersion = 2 //v2 bopts := badger.DefaultOptions(path) From 5aefc8e12d4de6ce66cd04d0e1f8fa55effd1c7f Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 13:10:20 -1000 Subject: [PATCH 15/59] prefix bugfix --- blockstore/badger/versions/badger.go | 11 ++++------- blockstore/badger/versions/badger_v2.go | 3 +-- blockstore/badger/versions/badger_v4.go | 3 +-- blockstore/badger/versions/options.go | 4 ++-- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 524082a4bf2..2f3704a58a0 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -11,8 +11,8 @@ import ( // Options embeds the badger options themselves, and augments them with // blockstore-specific options. type Options struct { - V2Options *badgerV2.Options - V4Options *badgerV4.Options + V2Options badgerV2.Options + V4Options badgerV4.Options // BadgerVersion sets the release version of badger to use BadgerVersion int @@ -41,21 +41,18 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error - prefix := opts.Prefix version := opts.BadgerVersion switch version { case 4: - opts := badgerV4.DefaultOptions(prefix) var dbV4 *badgerV4.DB - dbV4, err = badgerV4.Open(opts) + dbV4, err = badgerV4.Open(opts.V4Options) if err == nil { db = BadgerDB(&BadgerV4{dbV4}) } case 2: - opts := badgerV2.DefaultOptions(prefix) var dbV2 *badgerV2.DB - dbV2, err = badgerV2.Open(opts) + dbV2, err = badgerV2.Open(opts.V2Options) if err == nil { db = BadgerDB(&BadgerV2{dbV2}) } diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index 996c3615ef8..6d7b3269da3 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -116,7 +116,6 @@ func (b *BadgerV2) Copy(to BadgerDB) error { func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { var opts Options - opts.Prefix = "/blocks/" bopts := badger.DefaultOptions(path) bopts.DetectConflicts = false @@ -134,7 +133,7 @@ func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { bopts.NumCompactors = numWorkers } } - opts.V2Options = &bopts + opts.V2Options = bopts return opts } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index c0e3b363d00..ab05742ce80 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -131,8 +131,7 @@ func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { bopts.NumCompactors = numWorkers } } - opts.V4Options = &bopts - opts.Prefix = "/blocks/" + opts.V4Options = bopts return opts } diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 22d64c71fdb..d1b1f93cb3b 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -31,7 +31,7 @@ func DefaultOptions(path string, readonly bool) Options { bopts.NumCompactors = numWorkers } } - opts.V2Options = &bopts + opts.V2Options = bopts //v4 @@ -44,7 +44,7 @@ func DefaultOptions(path string, readonly bool) Options { boptsv4.NumCompactors = numWorkers } } - opts.V4Options = &boptsv4 + opts.V4Options = boptsv4 return opts } From d680f6fe81cf4b8230cd17850f19d5ec5aaf12a4 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 16:52:42 -1000 Subject: [PATCH 16/59] clean up lotus-bench for pr --- .../badger/versions/badger_interface.go | 5 ++-- blockstore/badger/versions/badger_v2.go | 19 ++++++++------- blockstore/badger/versions/badger_v4.go | 17 ++++++------- cmd/lotus-bench/import.go | 24 +++++++++---------- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 99d64fb55fc..35fbcb0b468 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -6,6 +6,7 @@ import ( "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" + "github.com/ipfs/go-cid" ) // BadgerDB defines the common interface for both v2 and v4 versions of Badger. @@ -20,7 +21,6 @@ type BadgerDB interface { Sync() error MaxBatchCount() int64 MaxBatchSize() int64 - BlockCacheMetrics() *ristretto.Metrics IndexCacheMetrics() *ristretto.Metrics GetErrKeyNotFound() error GetErrNoRewrite() error @@ -29,8 +29,9 @@ type BadgerDB interface { Size() (lsm int64, vlog int64) Copy(to BadgerDB) error DefaultOptions(prefix string, readonly bool) Options - Backup(w io.Writer, since uint64) (uint64, error) Load(r io.Reader, maxPendingWrites int) error + AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) + DeleteBlock(context.Context, cid.Cid) error } // BadgerStream defines the common interface for streaming data in Badger. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index 6d7b3269da3..d687d62786e 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -2,6 +2,7 @@ package versions import ( "context" + "fmt" "io" "os" "runtime" @@ -11,6 +12,7 @@ import ( "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) @@ -58,15 +60,10 @@ func (b *BadgerV2) Sync() error { func (b *BadgerV2) MaxBatchCount() int64 { return b.DB.MaxBatchCount() } - func (b *BadgerV2) MaxBatchSize() int64 { return b.DB.MaxBatchSize() } -func (b *BadgerV2) BlockCacheMetrics() *ristretto.Metrics { - return b.DB.BlockCacheMetrics() -} - func (b *BadgerV2) IndexCacheMetrics() *ristretto.Metrics { return b.DB.IndexCacheMetrics() } @@ -137,14 +134,18 @@ func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { return opts } -func (b *BadgerV2) Backup(w io.Writer, since uint64) (uint64, error) { - return b.DB.Backup(w, since) -} - func (b *BadgerV2) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } +func (b *BadgerV2) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, fmt.Errorf("AllKeysChan is not implemented") +} + +func (b *BadgerV2) DeleteBlock(context.Context, cid.Cid) error { + return fmt.Errorf("DeleteBlock is not implemented") +} + type BadgerV2WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index ab05742ce80..ff11a5d64d8 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -12,6 +12,7 @@ import ( badgerV4 "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" + "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) @@ -64,10 +65,6 @@ func (b *BadgerV4) MaxBatchSize() int64 { return b.DB.MaxBatchSize() } -func (b *BadgerV4) BlockCacheMetrics() *ristretto.Metrics { - return b.DB.BlockCacheMetrics() -} - func (b *BadgerV4) IndexCacheMetrics() *ristretto.Metrics { return b.DB.IndexCacheMetrics() } @@ -136,14 +133,18 @@ func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { } -func (b *BadgerV4) Backup(w io.Writer, since uint64) (uint64, error) { - return b.DB.Backup(w, since) -} - func (b *BadgerV4) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } +func (b *BadgerV4) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, fmt.Errorf("AllKeysChan is not implemented") +} + +func (b *BadgerV4) DeleteBlock(context.Context, cid.Cid) error { + return fmt.Errorf("DeleteBlock is not implemented") +} + type BadgerV4WriteBatch struct { *badger.WriteBatch } diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 16adbad60be..42dec3e1a8e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -16,10 +16,13 @@ import ( "time" ocprom "contrib.go.opencensus.io/exporter/prometheus" - bdg "github.com/dgraph-io/badger/v2" + badgerbs "github.com/filecoin-project/lotus/blockstore/badger" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - badger "github.com/ipfs/go-ds-badger2" + + badgerIpfs "github.com/ipfs/go-ds-badger2" + measure "github.com/ipfs/go-ds-measure" metricsprometheus "github.com/ipfs/go-metrics-prometheus" "github.com/ipld/go-car" @@ -32,7 +35,6 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/blockstore" - badgerbs "github.com/filecoin-project/lotus/blockstore/badger" "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/index" @@ -174,7 +176,7 @@ var importBenchCmd = &cli.Command{ switch { case cctx.Bool("use-native-badger"): log.Info("using native badger") - var opts badgerbs.Options + var opts badger.Options if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil { return err } @@ -183,14 +185,12 @@ var importBenchCmd = &cli.Command{ default: // legacy badger via datastore. log.Info("using legacy badger") - bdgOpt := badger.DefaultOptions - bdgOpt.GcInterval = 0 - bdgOpt.Options = bdg.DefaultOptions("") - bdgOpt.Options.SyncWrites = false - bdgOpt.Options.Truncate = true - bdgOpt.Options.DetectConflicts = false + var opts badger.Options + if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil { + return err + } - ds, err = badger.NewDatastore(tdir, &bdgOpt) + bs, err = badgerbs.Open(opts) } if err != nil { @@ -209,7 +209,7 @@ var importBenchCmd = &cli.Command{ var verifier storiface.Verifier = ffiwrapper.ProofVerifier if cctx.IsSet("syscall-cache") { - scds, err := badger.NewDatastore(cctx.String("syscall-cache"), &badger.DefaultOptions) + scds, err := badgerIpfs.NewDatastore(cctx.String("syscall-cache"), &badgerIpfs.DefaultOptions) if err != nil { return xerrors.Errorf("opening syscall-cache datastore: %w", err) } From 0daa826d13e32108eed75416608a278fbffbbc25 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 17:46:50 -1000 Subject: [PATCH 17/59] make docsgen --- cmd/lotus-bench/import.go | 6 ++---- cmd/lotus-shed/datastore.go | 2 +- cmd/lotus-shed/export.go | 2 +- cmd/lotus-shed/splitstore.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 42dec3e1a8e..14540ca3b3e 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -16,13 +16,9 @@ import ( "time" ocprom "contrib.go.opencensus.io/exporter/prometheus" - badgerbs "github.com/filecoin-project/lotus/blockstore/badger" - badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - badgerIpfs "github.com/ipfs/go-ds-badger2" - measure "github.com/ipfs/go-ds-measure" metricsprometheus "github.com/ipfs/go-metrics-prometheus" "github.com/ipld/go-car" @@ -35,6 +31,8 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/blockstore" + badgerbs "github.com/filecoin-project/lotus/blockstore/badger" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/index" diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 85ee6e36caf..08c946ffc2d 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/docker/go-units" - badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" logging "github.com/ipfs/go-log/v2" @@ -20,6 +19,7 @@ import ( "go.uber.org/multierr" "golang.org/x/xerrors" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/backupds" "github.com/filecoin-project/lotus/node/repo" diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 201899296b7..cc75a184588 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -13,7 +13,6 @@ import ( "sync" "github.com/dustin/go-humanize" - badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/boxo/blockservice" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/ipld/merkledag" @@ -30,6 +29,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/blockstore" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/filecoin-project/lotus/chain/store" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/cmd/lotus-shed/shedgen" diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index e0548710a8a..e0477c4c5ae 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -9,7 +9,6 @@ import ( "path/filepath" "runtime" - badger "github.com/filecoin-project/lotus/blockstore/badger/versions" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/query" "github.com/urfave/cli/v2" @@ -18,6 +17,7 @@ import ( "golang.org/x/sync/errgroup" "golang.org/x/xerrors" + badger "github.com/filecoin-project/lotus/blockstore/badger/versions" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" From dafc88058ff5d1634862ba4c8550dc513c9dec00 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 29 Jul 2024 20:56:01 -1000 Subject: [PATCH 18/59] lint fixes --- blockstore/badger/blockstore_test.go | 23 ++++++++++++----------- blockstore/badger/versions/badger_v2.go | 8 ++++---- blockstore/badger/versions/badger_v4.go | 8 ++++---- blockstore/badger/versions/options.go | 4 ++-- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index d253f37d95d..08450bbde09 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -16,18 +16,19 @@ import ( "golang.org/x/sync/errgroup" "github.com/filecoin-project/lotus/blockstore" + versions "github.com/filecoin-project/lotus/blockstore/badger/versions" ) func TestBadgerBlockstore(t *testing.T) { //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 (&Suite{ - NewBlockstore: newBlockstore(DefaultOptions), - OpenBlockstore: openBlockstore(DefaultOptions), + NewBlockstore: newBlockstore(versions.DefaultOptions), + OpenBlockstore: openBlockstore(versions.DefaultOptions), }).RunTests(t, "non_prefixed") - prefixed := func(path string) Options { - opts := DefaultOptions(path) + prefixed := func(path string) versions.Options { + opts := versions.DefaultOptions(path) opts.Prefix = "/prefixed/" return opts } @@ -41,7 +42,7 @@ func TestBadgerBlockstore(t *testing.T) { func TestStorageKey(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_STORAGE_KEY_001 - bs, _ := newBlockstore(DefaultOptions)(t) + bs, _ := newBlockstore(versions.DefaultOptions)(t) bbs := bs.(*Blockstore) defer bbs.Close() //nolint:errcheck @@ -72,7 +73,7 @@ func TestStorageKey(t *testing.T) { require.Equal(t, k3, k2) } -func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { +func newBlockstore(optsSupplier func(path string) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { tb.Helper() @@ -87,14 +88,14 @@ func newBlockstore(optsSupplier func(path string) Options) func(tb testing.TB) ( } } -func openBlockstore(optsSupplier func(path string) Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { +func openBlockstore(optsSupplier func(path string) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { tb.Helper() return Open(optsSupplier(path)) } } -func testMove(t *testing.T, optsF func(string) Options) { +func testMove(t *testing.T, optsF func(string) versions.Options) { ctx := context.Background() basePath := t.TempDir() @@ -257,15 +258,15 @@ func TestMoveNoPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, DefaultOptions) + testMove(t, versions.DefaultOptions) } func TestMoveWithPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, func(path string) Options { - opts := DefaultOptions(path) + testMove(t, func(path string) versions.Options { + opts := versions.DefaultOptions(path) opts.Prefix = "/prefixed/" return opts }) diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index d687d62786e..cf1aa80e7b6 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -8,7 +8,7 @@ import ( "runtime" "strconv" - "github.com/dgraph-io/badger/v2" + badger "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" @@ -171,11 +171,11 @@ type BadgerV2Stream struct { } func (s *BadgerV2Stream) SetNumGo(numGo int) { - s.NumGo = numGo + s.Stream.NumGo = numGo } func (s *BadgerV2Stream) SetLogPrefix(prefix string) { - s.LogPrefix = prefix + s.Stream.LogPrefix = prefix } func (s *BadgerV2Stream) Orchestrate(ctx context.Context) error { @@ -183,7 +183,7 @@ func (s *BadgerV2Stream) Orchestrate(ctx context.Context) error { } func (s *BadgerV2Stream) ForEach(ctx context.Context, fn func(key string, value string) error) error { - s.Send = func(list *pb.KVList) error { + s.Stream.Send = func(list *pb.KVList) error { for _, kv := range list.Kv { if kv.Key == nil || kv.Value == nil { continue diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index ff11a5d64d8..46cdf9c55d6 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -8,7 +8,7 @@ import ( "runtime" "strconv" - "github.com/dgraph-io/badger/v4" + badger "github.com/dgraph-io/badger/v4" badgerV4 "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" @@ -170,14 +170,14 @@ type BadgerV4Stream struct { } func (s *BadgerV4Stream) SetNumGo(numGo int) { - s.NumGo = numGo + s.Stream.NumGo = numGo } func (s *BadgerV4Stream) SetLogPrefix(prefix string) { - s.LogPrefix = prefix + s.Stream.LogPrefix = prefix } func (s *BadgerV4Stream) ForEach(ctx context.Context, fn func(key string, value string) error) error { - s.Send = func(buf *z.Buffer) error { + s.Stream.Send = func(buf *z.Buffer) error { list, err := badger.BufferToKVList(buf) if err != nil { return fmt.Errorf("buffer to KV list conversion: %w", err) diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index d1b1f93cb3b..4f5c77ad0de 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -4,7 +4,7 @@ import ( "os" "strconv" - "github.com/dgraph-io/badger/v2" + badgerV2 "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/options" badgerV4 "github.com/dgraph-io/badger/v4" ) @@ -15,7 +15,7 @@ func DefaultOptions(path string, readonly bool) Options { opts.BadgerVersion = 2 //v2 - bopts := badger.DefaultOptions(path) + bopts := badgerV2.DefaultOptions(path) bopts.DetectConflicts = false bopts.CompactL0OnClose = true bopts.Truncate = true From 395be919ac62918b668b3c9c28dc62e6e75db0eb Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 07:03:20 -1000 Subject: [PATCH 19/59] fix func signature --- blockstore/badger/blockstore_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index 08450bbde09..e4fd6804a3e 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -27,8 +27,8 @@ func TestBadgerBlockstore(t *testing.T) { OpenBlockstore: openBlockstore(versions.DefaultOptions), }).RunTests(t, "non_prefixed") - prefixed := func(path string) versions.Options { - opts := versions.DefaultOptions(path) + prefixed := func(path string, _ bool) versions.Options { + opts := versions.DefaultOptions(path, false) opts.Prefix = "/prefixed/" return opts } @@ -73,13 +73,13 @@ func TestStorageKey(t *testing.T) { require.Equal(t, k3, k2) } -func newBlockstore(optsSupplier func(path string) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { +func newBlockstore(optsSupplier func(path string, readonly bool) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { tb.Helper() path = tb.TempDir() - db, err := Open(optsSupplier(path)) + db, err := Open(optsSupplier(path, false)) if err != nil { tb.Fatal(err) } @@ -88,20 +88,20 @@ func newBlockstore(optsSupplier func(path string) versions.Options) func(tb test } } -func openBlockstore(optsSupplier func(path string) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { +func openBlockstore(optsSupplier func(path string, readonly bool) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { tb.Helper() - return Open(optsSupplier(path)) + return Open(optsSupplier(path, false)) } } -func testMove(t *testing.T, optsF func(string) versions.Options) { +func testMove(t *testing.T, optsF func(string, bool) versions.Options) { ctx := context.Background() basePath := t.TempDir() dbPath := filepath.Join(basePath, "db") - db, err := Open(optsF(dbPath)) + db, err := Open(optsF(dbPath, false)) if err != nil { t.Fatal(err) } @@ -244,7 +244,7 @@ func testMove(t *testing.T, optsF func(string) versions.Options) { t.Fatal(err) } - db, err = Open(optsF(dbPath)) + db, err = Open(optsF(dbPath, false)) if err != nil { t.Fatal(err) } @@ -265,8 +265,8 @@ func TestMoveWithPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, func(path string) versions.Options { - opts := versions.DefaultOptions(path) + testMove(t, func(path string, readonly bool) versions.Options { + opts := versions.DefaultOptions(path, false) opts.Prefix = "/prefixed/" return opts }) From fd626f9d10fea08e491c9969ef6171bfafa483a5 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 07:24:38 -1000 Subject: [PATCH 20/59] bugfix --- blockstore/badger/versions/badger_interface.go | 1 + blockstore/badger/versions/badger_v2.go | 4 ++++ blockstore/badger/versions/badger_v4.go | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 35fbcb0b468..0b948b894b0 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -32,6 +32,7 @@ type BadgerDB interface { Load(r io.Reader, maxPendingWrites int) error AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) DeleteBlock(context.Context, cid.Cid) error + Backup(w io.Writer, since uint64) (uint64, error) } // BadgerStream defines the common interface for streaming data in Badger. diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index cf1aa80e7b6..cbd1ba4b5d4 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -146,6 +146,10 @@ func (b *BadgerV2) DeleteBlock(context.Context, cid.Cid) error { return fmt.Errorf("DeleteBlock is not implemented") } +func (b *BadgerV2) Backup(w io.Writer, since uint64) (uint64, error) { + return b.DB.Backup(w, since) +} + type BadgerV2WriteBatch struct { *badger.WriteBatch } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 46cdf9c55d6..5f12783c103 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -145,6 +145,10 @@ func (b *BadgerV4) DeleteBlock(context.Context, cid.Cid) error { return fmt.Errorf("DeleteBlock is not implemented") } +func (b *BadgerV4) Backup(w io.Writer, since uint64) (uint64, error) { + return b.DB.Backup(w, since) +} + type BadgerV4WriteBatch struct { *badger.WriteBatch } From 900d31f97fa7bd1ccd1cfe88e1664bd698e1601b Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 07:34:09 -1000 Subject: [PATCH 21/59] unused lint fix --- blockstore/badger/versions/badger_interface.go | 6 ------ cmd/lotus-shed/export.go | 9 --------- 2 files changed, 15 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 0b948b894b0..21e5171eff1 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -5,7 +5,6 @@ import ( "io" "github.com/dgraph-io/ristretto" - "github.com/dgraph-io/ristretto/z" "github.com/ipfs/go-cid" ) @@ -77,11 +76,6 @@ type Item interface { ValueSize() int64 } -type Buffer struct { - kvList KVList - buf z.Buffer -} - type WriteBatch interface { Set(key, val []byte) error Delete(key []byte) error diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index cc75a184588..424beaa3494 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -468,12 +468,3 @@ func (rc *rawCarb) writeCar(ctx context.Context, path string, root cid.Cid) erro } var _ blockstore.Blockstore = &rawCarb{} - -type badgerLog struct { - *zap.SugaredLogger - Skip2 *zap.SugaredLogger -} - -func (b *badgerLog) Warningf(format string, args ...interface{}) { - b.Skip2.Warnf(format, args...) -} From d0143fbfc8f5ec5a8b52f13b7df14761d488cfb9 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 09:07:09 -1000 Subject: [PATCH 22/59] lintfix --- cmd/lotus-shed/export.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 424beaa3494..391e62e5f86 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -265,7 +265,7 @@ var exportRawCmd = &cli.Command{ return nil } - h, err := base32.RawStdEncoding.DecodeString(string(key[len("/blocks/"):])) + h, err := base32.RawStdEncoding.DecodeString(key[len("/blocks/"):]) if err != nil { return xerrors.Errorf("decode b32 ds key %x: %w", key, err) } From 1b1bbd008f49075b51247edf0d470e0087c7f8dd Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 09:19:59 -1000 Subject: [PATCH 23/59] debugging test --- blockstore/badger/blockstore_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index e4fd6804a3e..d01580123ab 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -54,6 +54,8 @@ func TestStorageKey(t *testing.T) { // nil slice; let StorageKey allocate for us. k1 := bbs.StorageKey(nil, cid1) + fmt.Println(cid1) + fmt.Println(k1) require.Len(t, k1, 55) require.True(t, cap(k1) == len(k1)) From 55918dbcdf76930fec6b62e3903546df22763d4d Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 17:50:13 -1000 Subject: [PATCH 24/59] wip - fixes tests --- blockstore/badger/blockstore.go | 4 +-- blockstore/badger/versions/badger.go | 17 ------------ blockstore/badger/versions/options.go | 40 ++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index dc990c9fff5..f2d1fcba8ec 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -274,8 +274,8 @@ func (b *Blockstore) movingGC(ctx context.Context) error { log.Infof("moving blockstore from %s to %s", b.opts.Dir, newPath) opts := b.opts - opts.Dir = newPath - opts.ValueDir = newPath + opts.SetDir(newPath) + opts.SetValueDir(newPath) dbNew, err := badger.OpenBadgerDB(opts) if err != nil { diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 2f3704a58a0..44b77925ad8 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -8,23 +8,6 @@ import ( "go.uber.org/zap" ) -// Options embeds the badger options themselves, and augments them with -// blockstore-specific options. -type Options struct { - V2Options badgerV2.Options - V4Options badgerV4.Options - - // BadgerVersion sets the release version of badger to use - BadgerVersion int - - Prefix string - Dir string - ValueDir string - SyncWrites bool - - Logger BadgerLogger -} - // BadgerLogger is a local wrapper for go-log to make the interface // compatible with badger.Logger (namely, aliasing Warnf to Warningf) type BadgerLogger struct { diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 4f5c77ad0de..4e2dc541f69 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -9,11 +9,49 @@ import ( badgerV4 "github.com/dgraph-io/badger/v4" ) +// Options embeds the badger options themselves, and augments them with +// blockstore-specific options. +type Options struct { + V2Options badgerV2.Options + V4Options badgerV4.Options + + // BadgerVersion sets the release version of badger to use + BadgerVersion int + + Prefix string + Dir string + ValueDir string + SyncWrites bool + + Logger BadgerLogger +} + +func (o *Options) SetDir(dir string) { + o.Dir = dir + o.V2Options.Dir = dir + o.V4Options.Dir = dir +} + +func (o *Options) SetValueDir(valueDir string) { + o.ValueDir = valueDir + o.V2Options.ValueDir = valueDir + o.V4Options.ValueDir = valueDir +} + func DefaultOptions(path string, readonly bool) Options { var opts Options - opts.Prefix = "/blocks/" + + /* + TODO determine where this code came from and if it needs to be added back somewhere + //opts.Prefix = "/blocks/" + */ + + //TODO remove hard code version # and connect config opts.BadgerVersion = 2 + opts.SetDir(path) + opts.SetValueDir(path) + //v2 bopts := badgerV2.DefaultOptions(path) bopts.DetectConflicts = false From a1d39a7a96fbcacff403ae3ac2c083241fa0e594 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 18:23:43 -1000 Subject: [PATCH 25/59] clean up creation of blockstore --- blockstore/badger/versions/options.go | 63 ++++++++++++++------------- node/repo/blockstore_opts.go | 2 +- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 4e2dc541f69..9c378412bbc 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -5,7 +5,8 @@ import ( "strconv" badgerV2 "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/options" + optionsV2 "github.com/dgraph-io/badger/v2/options" + badgerV4 "github.com/dgraph-io/badger/v4" ) @@ -38,50 +39,50 @@ func (o *Options) SetValueDir(valueDir string) { o.V4Options.ValueDir = valueDir } +func BlockStoreOptions(path string, readonly bool) Options { + opts := DefaultOptions(path, readonly) + opts.Prefix = "/blocks/" + + opts.V2Options.DetectConflicts = false + opts.V2Options.CompactL0OnClose = true + opts.V2Options.Truncate = true + opts.V2Options.ValueLogLoadingMode = optionsV2.MemoryMap + opts.V2Options.TableLoadingMode = optionsV2.MemoryMap + opts.V2Options.ValueThreshold = 128 + opts.V2Options.MaxTableSize = 64 << 20 + opts.V2Options.ReadOnly = readonly + + opts.V4Options.DetectConflicts = false + opts.V4Options.CompactL0OnClose = true + opts.V4Options.ValueThreshold = 148 + opts.V4Options.ReadOnly = readonly + + // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { + if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { + opts.V2Options.NumCompactors = numWorkers + opts.V4Options.NumCompactors = numWorkers + } + } + + return opts +} + func DefaultOptions(path string, readonly bool) Options { var opts Options - /* - TODO determine where this code came from and if it needs to be added back somewhere - //opts.Prefix = "/blocks/" - */ - //TODO remove hard code version # and connect config - opts.BadgerVersion = 2 + opts.BadgerVersion = 4 opts.SetDir(path) opts.SetValueDir(path) //v2 bopts := badgerV2.DefaultOptions(path) - bopts.DetectConflicts = false - bopts.CompactL0OnClose = true - bopts.Truncate = true - bopts.ValueLogLoadingMode = options.MemoryMap - bopts.TableLoadingMode = options.MemoryMap - bopts.ValueThreshold = 128 - bopts.MaxTableSize = 64 << 20 - bopts.ReadOnly = readonly - - // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM - if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { - if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { - bopts.NumCompactors = numWorkers - } - } opts.V2Options = bopts //v4 - boptsv4 := badgerV4.DefaultOptions(path) - boptsv4.ReadOnly = readonly - - // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM - if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { - if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { - boptsv4.NumCompactors = numWorkers - } - } opts.V4Options = boptsv4 return opts diff --git a/node/repo/blockstore_opts.go b/node/repo/blockstore_opts.go index 33839e16ef7..0ad62d6e768 100644 --- a/node/repo/blockstore_opts.go +++ b/node/repo/blockstore_opts.go @@ -7,6 +7,6 @@ import ( // BadgerBlockstoreOptions returns the badger options to apply for the provided // domain. func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (versions.Options, error) { - opts := versions.DefaultOptions(path, readonly) + opts := versions.BlockStoreOptions(path, readonly) return opts, nil } From dd950249db4344c6f41db1e3abf9e6293dc84439 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 19:20:14 -1000 Subject: [PATCH 26/59] connect badgerVersion config with creation of blockstores --- blockstore/badger/versions/options.go | 9 ++++----- node/modules/blockstore.go | 11 ++++++++++- node/repo/blockstore_opts.go | 4 ++-- node/repo/fsrepo.go | 14 +++++++++++++- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 9c378412bbc..ee0007e9642 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -39,8 +39,8 @@ func (o *Options) SetValueDir(valueDir string) { o.V4Options.ValueDir = valueDir } -func BlockStoreOptions(path string, readonly bool) Options { - opts := DefaultOptions(path, readonly) +func BlockStoreOptions(path string, readonly bool, badgerVersion int) Options { + opts := DefaultOptions(path, readonly, badgerVersion) opts.Prefix = "/blocks/" opts.V2Options.DetectConflicts = false @@ -68,11 +68,10 @@ func BlockStoreOptions(path string, readonly bool) Options { return opts } -func DefaultOptions(path string, readonly bool) Options { +func DefaultOptions(path string, readonly bool, badgerVersion int) Options { var opts Options - //TODO remove hard code version # and connect config - opts.BadgerVersion = 4 + opts.BadgerVersion = badgerVersion opts.SetDir(path) opts.SetValueDir(path) diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 9c54d51e60f..01e2998893f 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -56,7 +56,16 @@ func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlocksto return nil, err } - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly()) + c, err := r.Config() + if err != nil { + return nil, err + } + cfg, ok := c.(*config.FullNode) + if !ok { + return nil, xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := cfg.Chainstore.BadgerVersion + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly(), badgerVersion) if err != nil { return nil, err } diff --git a/node/repo/blockstore_opts.go b/node/repo/blockstore_opts.go index 0ad62d6e768..7be35b2bc48 100644 --- a/node/repo/blockstore_opts.go +++ b/node/repo/blockstore_opts.go @@ -6,7 +6,7 @@ import ( // BadgerBlockstoreOptions returns the badger options to apply for the provided // domain. -func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (versions.Options, error) { - opts := versions.BlockStoreOptions(path, readonly) +func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool, badgerVersion int) (versions.Options, error) { + opts := versions.BlockStoreOptions(path, readonly, badgerVersion) return opts, nil } diff --git a/node/repo/fsrepo.go b/node/repo/fsrepo.go index 26cbbd6b135..0e0a1b807a8 100644 --- a/node/repo/fsrepo.go +++ b/node/repo/fsrepo.go @@ -437,7 +437,19 @@ func (fsr *fsLockedRepo) Blockstore(ctx context.Context, domain BlockstoreDomain return } - opts, err := BadgerBlockstoreOptions(domain, path, readonly) + c, err := fsr.Config() + if err != nil { + fsr.bsErr = err + return + } + cfg, ok := c.(*config.FullNode) + if !ok { + fsr.bsErr = xerrors.Errorf("invalid config for repo, got: %T", c) + return + } + badgerVersion := cfg.Chainstore.BadgerVersion + + opts, err := BadgerBlockstoreOptions(domain, path, readonly, badgerVersion) if err != nil { fsr.bsErr = err return From 279f2bad8f451fbcf6e76c8ed396107abc1a2138 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 30 Jul 2024 22:52:06 -1000 Subject: [PATCH 27/59] bugfixes --- cmd/lotus-shed/migrations.go | 12 +++++++++++- cmd/lotus-shed/splitstore.go | 13 +++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index 6350c217c97..b3aaa4a501a 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -69,6 +69,7 @@ import ( "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/must" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) @@ -142,7 +143,16 @@ var migrationsCmd = &cli.Command{ return err } - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + c, err := r.Config() + if err != nil { + return nil, err + } + cfg, ok := c.(*config.FullNode) + if !ok { + return nil, xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := cfg.Chainstore.BadgerVersion + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly(), badgerVersion) if err != nil { return err } diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index e0477c4c5ae..b606a8412c9 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -200,14 +200,23 @@ func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error { Skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } - coldOpts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, coldPath, false) + c, err := r.Config() + if err != nil { + return nil, err + } + cfg, ok := c.(*config.FullNode) + if !ok { + return nil, xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := cfg.Chainstore.BadgerVersion + coldOpts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, coldPath, false, badgerVersion) if err != nil { return xerrors.Errorf("error getting coldstore badger options: %w", err) } coldOpts.SyncWrites = false coldOpts.Logger = blog - hotOpts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, hotPath, true) + hotOpts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, hotPath, true, badgerVersion) if err != nil { return xerrors.Errorf("error getting hotstore badger options: %w", err) } From 9f82d83dae4e04f464927407f8337964ecdd409b Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 07:17:53 -1000 Subject: [PATCH 28/59] bugfix for make buildall --- cmd/lotus-shed/datastore.go | 26 ++++++++++++++++++++++++-- cmd/lotus-shed/export.go | 14 +++++++++++++- cmd/lotus-shed/fevmanalytics.go | 25 +++++++++++++++++++++++-- cmd/lotus-shed/invariants.go | 12 +++++++++++- cmd/lotus-shed/migrations.go | 10 +++++----- cmd/lotus-shed/splitstore.go | 6 +++--- 6 files changed, 79 insertions(+), 14 deletions(-) diff --git a/cmd/lotus-shed/datastore.go b/cmd/lotus-shed/datastore.go index 08c946ffc2d..b3a1e6e65c8 100644 --- a/cmd/lotus-shed/datastore.go +++ b/cmd/lotus-shed/datastore.go @@ -22,6 +22,7 @@ import ( badger "github.com/filecoin-project/lotus/blockstore/badger/versions" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) @@ -325,8 +326,29 @@ var datastoreRewriteCmd = &cli.Command{ to badger.BadgerDB ) + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + c, err := lkrepo.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + // open the destination (to) store. - opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false) + opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, toPath, false, badgerVersion) if err != nil { return xerrors.Errorf("failed to get badger options: %w", err) } @@ -337,7 +359,7 @@ var datastoreRewriteCmd = &cli.Command{ } // open the source (from) store. - opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true) + opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, fromPath, true, badgerVersion) if err != nil { return xerrors.Errorf("failed to get badger options: %w", err) } diff --git a/cmd/lotus-shed/export.go b/cmd/lotus-shed/export.go index 391e62e5f86..9374282f541 100644 --- a/cmd/lotus-shed/export.go +++ b/cmd/lotus-shed/export.go @@ -33,6 +33,7 @@ import ( "github.com/filecoin-project/lotus/chain/store" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/cmd/lotus-shed/shedgen" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) @@ -233,8 +234,19 @@ var exportRawCmd = &cli.Command{ } { + + c, err := lr.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + path := filepath.Join(lr.Path(), "datastore", "chain") - opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, path, false) + opts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, path, false, badgerVersion) if err != nil { return err } diff --git a/cmd/lotus-shed/fevmanalytics.go b/cmd/lotus-shed/fevmanalytics.go index 19416b77e12..061e0c0ed87 100644 --- a/cmd/lotus-shed/fevmanalytics.go +++ b/cmd/lotus-shed/fevmanalytics.go @@ -21,6 +21,7 @@ import ( evm2 "github.com/filecoin-project/lotus/chain/actors/builtin/evm" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) @@ -80,7 +81,17 @@ var FevmBalanceCmd = &cli.Command{ return err } - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + c, err := lkrepo.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly(), badgerVersion) if err != nil { return err } @@ -175,7 +186,17 @@ var FevmActorsCmd = &cli.Command{ return err } - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + c, err := lkrepo.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly(), badgerVersion) if err != nil { return err } diff --git a/cmd/lotus-shed/invariants.go b/cmd/lotus-shed/invariants.go index ede3ae35b4e..d637f3d9caf 100644 --- a/cmd/lotus-shed/invariants.go +++ b/cmd/lotus-shed/invariants.go @@ -35,6 +35,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" ) @@ -93,7 +94,16 @@ var invariantsCmd = &cli.Command{ return err } - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly()) + c, err := lkrepo.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly(), badgerVersion) if err != nil { return err } diff --git a/cmd/lotus-shed/migrations.go b/cmd/lotus-shed/migrations.go index b3aaa4a501a..f2a75714f0a 100644 --- a/cmd/lotus-shed/migrations.go +++ b/cmd/lotus-shed/migrations.go @@ -143,15 +143,15 @@ var migrationsCmd = &cli.Command{ return err } - c, err := r.Config() + c, err := lkrepo.Config() if err != nil { - return nil, err + return err } - cfg, ok := c.(*config.FullNode) + lotusCfg, ok := c.(*config.FullNode) if !ok { - return nil, xerrors.Errorf("invalid config for repo, got: %T", c) + return xerrors.Errorf("invalid config for repo, got: %T", c) } - badgerVersion := cfg.Chainstore.BadgerVersion + badgerVersion := lotusCfg.Chainstore.BadgerVersion opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly(), badgerVersion) if err != nil { return err diff --git a/cmd/lotus-shed/splitstore.go b/cmd/lotus-shed/splitstore.go index b606a8412c9..e686ff3602f 100644 --- a/cmd/lotus-shed/splitstore.go +++ b/cmd/lotus-shed/splitstore.go @@ -200,13 +200,13 @@ func copyHotstoreToColdstore(lr repo.LockedRepo, gcColdstore bool) error { Skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), } - c, err := r.Config() + c, err := lr.Config() if err != nil { - return nil, err + return err } cfg, ok := c.(*config.FullNode) if !ok { - return nil, xerrors.Errorf("invalid config for repo, got: %T", c) + return xerrors.Errorf("invalid config for repo, got: %T", c) } badgerVersion := cfg.Chainstore.BadgerVersion coldOpts, err := repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, coldPath, false, badgerVersion) From c01521f96502791046b6b29d8c5d642f6eb57568 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 07:22:05 -1000 Subject: [PATCH 29/59] autogen --- blockstore/badger/versions/options.go | 1 - 1 file changed, 1 deletion(-) diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index ee0007e9642..e068013e657 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -6,7 +6,6 @@ import ( badgerV2 "github.com/dgraph-io/badger/v2" optionsV2 "github.com/dgraph-io/badger/v2/options" - badgerV4 "github.com/dgraph-io/badger/v4" ) From 92c3f2def74010deb5933173b4b60679933b7385 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 07:43:20 -1000 Subject: [PATCH 30/59] bugfix import bench --- cmd/lotus-bench/import.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go index 14540ca3b3e..4e0333984b4 100644 --- a/cmd/lotus-bench/import.go +++ b/cmd/lotus-bench/import.go @@ -44,6 +44,7 @@ import ( _ "github.com/filecoin-project/lotus/lib/sigs/bls" _ "github.com/filecoin-project/lotus/lib/sigs/delegated" _ "github.com/filecoin-project/lotus/lib/sigs/secp" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" "github.com/filecoin-project/lotus/storage/sealer/ffiwrapper" "github.com/filecoin-project/lotus/storage/sealer/storiface" @@ -171,11 +172,33 @@ var importBenchCmd = &cli.Command{ err error ) + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + c, err := lkrepo.Config() + if err != nil { + return err + } + lotusCfg, ok := c.(*config.FullNode) + if !ok { + return xerrors.Errorf("invalid config for repo, got: %T", c) + } + badgerVersion := lotusCfg.Chainstore.BadgerVersion + switch { case cctx.Bool("use-native-badger"): log.Info("using native badger") + var opts badger.Options - if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil { + if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false, badgerVersion); err != nil { return err } opts.SyncWrites = false @@ -184,7 +207,7 @@ var importBenchCmd = &cli.Command{ default: // legacy badger via datastore. log.Info("using legacy badger") var opts badger.Options - if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false); err != nil { + if opts, err = repo.BadgerBlockstoreOptions(repo.UniversalBlockstore, tdir, false, badgerVersion); err != nil { return err } From ab99ef0906e4003b0182183622c1ee03a9ad0cbd Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 08:13:31 -1000 Subject: [PATCH 31/59] get tests to work with v2 --- blockstore/badger/blockstore_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index d01580123ab..1d9ba1d4953 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -19,6 +19,8 @@ import ( versions "github.com/filecoin-project/lotus/blockstore/badger/versions" ) +const BADGER_DEFAULT_VERSION = 2 + func TestBadgerBlockstore(t *testing.T) { //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 @@ -27,8 +29,8 @@ func TestBadgerBlockstore(t *testing.T) { OpenBlockstore: openBlockstore(versions.DefaultOptions), }).RunTests(t, "non_prefixed") - prefixed := func(path string, _ bool) versions.Options { - opts := versions.DefaultOptions(path, false) + prefixed := func(path string, _ bool, _ int) versions.Options { + opts := versions.DefaultOptions(path, false, BADGER_DEFAULT_VERSION) opts.Prefix = "/prefixed/" return opts } @@ -75,13 +77,13 @@ func TestStorageKey(t *testing.T) { require.Equal(t, k3, k2) } -func newBlockstore(optsSupplier func(path string, readonly bool) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { +func newBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { tb.Helper() path = tb.TempDir() - db, err := Open(optsSupplier(path, false)) + db, err := Open(optsSupplier(path, false, BADGER_DEFAULT_VERSION)) if err != nil { tb.Fatal(err) } @@ -90,20 +92,20 @@ func newBlockstore(optsSupplier func(path string, readonly bool) versions.Option } } -func openBlockstore(optsSupplier func(path string, readonly bool) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { +func openBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { tb.Helper() - return Open(optsSupplier(path, false)) + return Open(optsSupplier(path, false, BADGER_DEFAULT_VERSION)) } } -func testMove(t *testing.T, optsF func(string, bool) versions.Options) { +func testMove(t *testing.T, optsF func(string, bool, int) versions.Options) { ctx := context.Background() basePath := t.TempDir() dbPath := filepath.Join(basePath, "db") - db, err := Open(optsF(dbPath, false)) + db, err := Open(optsF(dbPath, false, BADGER_DEFAULT_VERSION)) if err != nil { t.Fatal(err) } @@ -246,7 +248,7 @@ func testMove(t *testing.T, optsF func(string, bool) versions.Options) { t.Fatal(err) } - db, err = Open(optsF(dbPath, false)) + db, err = Open(optsF(dbPath, false, BADGER_DEFAULT_VERSION)) if err != nil { t.Fatal(err) } @@ -267,8 +269,8 @@ func TestMoveWithPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, func(path string, readonly bool) versions.Options { - opts := versions.DefaultOptions(path, false) + testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { + opts := versions.DefaultOptions(path, false, badgerVersion) opts.Prefix = "/prefixed/" return opts }) From feac779764f74697145752b2ab9126c5c4360f30 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 08:20:17 -1000 Subject: [PATCH 32/59] add both versions to badger test --- blockstore/badger/blockstore_test.go | 142 ++++++++++++++++----------- 1 file changed, 84 insertions(+), 58 deletions(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index 1d9ba1d4953..441007368da 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -19,71 +19,86 @@ import ( versions "github.com/filecoin-project/lotus/blockstore/badger/versions" ) -const BADGER_DEFAULT_VERSION = 2 +const ( + BADGER_VERSION_2 = 2 + BADGER_VERSION_4 = 4 +) + +var SUPPORTED_BADGER_VERSIONS = []int{BADGER_VERSION_2, BADGER_VERSION_4} func TestBadgerBlockstore(t *testing.T) { //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 - (&Suite{ - NewBlockstore: newBlockstore(versions.DefaultOptions), - OpenBlockstore: openBlockstore(versions.DefaultOptions), - }).RunTests(t, "non_prefixed") - - prefixed := func(path string, _ bool, _ int) versions.Options { - opts := versions.DefaultOptions(path, false, BADGER_DEFAULT_VERSION) - opts.Prefix = "/prefixed/" - return opts - } + for _, version := range SUPPORTED_BADGER_VERSIONS { + t.Run(fmt.Sprintf("non_prefixed_v%d", version), func(t *testing.T) { + (&Suite{ + NewBlockstore: newBlockstore(versions.DefaultOptions, version), + OpenBlockstore: openBlockstore(versions.DefaultOptions, version), + }).RunTests(t, "non_prefixed") + }) + + t.Run(fmt.Sprintf("prefixed_v%d", version), func(t *testing.T) { + prefixed := func(path string, _ bool, _ int) versions.Options { + opts := versions.DefaultOptions(path, false, version) + opts.Prefix = "/prefixed/" + return opts + } - (&Suite{ - NewBlockstore: newBlockstore(prefixed), - OpenBlockstore: openBlockstore(prefixed), - }).RunTests(t, "prefixed") + (&Suite{ + NewBlockstore: newBlockstore(prefixed, version), + OpenBlockstore: openBlockstore(prefixed, version), + }).RunTests(t, "prefixed") + }) + } } func TestStorageKey(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_STORAGE_KEY_001 - bs, _ := newBlockstore(versions.DefaultOptions)(t) - bbs := bs.(*Blockstore) - defer bbs.Close() //nolint:errcheck - - cid1 := blocks.NewBlock([]byte("some data")).Cid() - cid2 := blocks.NewBlock([]byte("more data")).Cid() - cid3 := blocks.NewBlock([]byte("a little more data")).Cid() - require.NotEqual(t, cid1, cid2) // sanity check - require.NotEqual(t, cid2, cid3) // sanity check - - // nil slice; let StorageKey allocate for us. - k1 := bbs.StorageKey(nil, cid1) - fmt.Println(cid1) - fmt.Println(k1) - require.Len(t, k1, 55) - require.True(t, cap(k1) == len(k1)) - - // k1's backing array is reused. - k2 := bbs.StorageKey(k1, cid2) - require.Len(t, k2, 55) - require.True(t, cap(k2) == len(k1)) - - // bring k2 to len=0, and verify that its backing array gets reused - // (i.e. k1 and k2 are overwritten) - k3 := bbs.StorageKey(k2[:0], cid3) - require.Len(t, k3, 55) - require.True(t, cap(k3) == len(k3)) - - // backing array of k1 and k2 has been modified, i.e. memory is shared. - require.Equal(t, k3, k1) - require.Equal(t, k3, k2) + for _, version := range SUPPORTED_BADGER_VERSIONS { + t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { + bs, _ := newBlockstore(versions.DefaultOptions, version)(t) + bbs := bs.(*Blockstore) + defer bbs.Close() //nolint:errcheck + + cid1 := blocks.NewBlock([]byte("some data")).Cid() + cid2 := blocks.NewBlock([]byte("more data")).Cid() + cid3 := blocks.NewBlock([]byte("a little more data")).Cid() + require.NotEqual(t, cid1, cid2) // sanity check + require.NotEqual(t, cid2, cid3) // sanity check + + // nil slice; let StorageKey allocate for us. + k1 := bbs.StorageKey(nil, cid1) + fmt.Println(cid1) + fmt.Println(k1) + require.Len(t, k1, 55) + require.True(t, cap(k1) == len(k1)) + + // k1's backing array is reused. + k2 := bbs.StorageKey(k1, cid2) + require.Len(t, k2, 55) + require.True(t, cap(k2) == len(k1)) + + // bring k2 to len=0, and verify that its backing array gets reused + // (i.e. k1 and k2 are overwritten) + k3 := bbs.StorageKey(k2[:0], cid3) + require.Len(t, k3, 55) + require.True(t, cap(k3) == len(k3)) + + // backing array of k1 and k2 has been modified, i.e. memory is shared. + require.Equal(t, k3, k1) + require.Equal(t, k3, k2) + }) + } } -func newBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { +func newBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options, version int) func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { return func(tb testing.TB) (bs blockstore.BasicBlockstore, path string) { tb.Helper() path = tb.TempDir() - db, err := Open(optsSupplier(path, false, BADGER_DEFAULT_VERSION)) + db, err := Open(optsSupplier(path, false, version)) if err != nil { tb.Fatal(err) } @@ -92,10 +107,10 @@ func newBlockstore(optsSupplier func(path string, readonly bool, badgerVersion i } } -func openBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { +func openBlockstore(optsSupplier func(path string, readonly bool, badgerVersion int) versions.Options, version int) func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { return func(tb testing.TB, path string) (bs blockstore.BasicBlockstore, err error) { tb.Helper() - return Open(optsSupplier(path, false, BADGER_DEFAULT_VERSION)) + return Open(optsSupplier(path, false, version)) } } @@ -105,7 +120,7 @@ func testMove(t *testing.T, optsF func(string, bool, int) versions.Options) { dbPath := filepath.Join(basePath, "db") - db, err := Open(optsF(dbPath, false, BADGER_DEFAULT_VERSION)) + db, err := Open(optsF(dbPath, false, BADGER_VERSION_2)) if err != nil { t.Fatal(err) } @@ -248,7 +263,7 @@ func testMove(t *testing.T, optsF func(string, bool, int) versions.Options) { t.Fatal(err) } - db, err = Open(optsF(dbPath, false, BADGER_DEFAULT_VERSION)) + db, err = Open(optsF(dbPath, false, BADGER_VERSION_2)) if err != nil { t.Fatal(err) } @@ -262,16 +277,27 @@ func TestMoveNoPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, versions.DefaultOptions) + for _, version := range SUPPORTED_BADGER_VERSIONS { + t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { + testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { + return versions.DefaultOptions(path, false, version) + }) + }) + } } func TestMoveWithPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_OPEN_001, @SPLITSTORE_BADGER_CLOSE_001 //stm: @SPLITSTORE_BADGER_PUT_001, @SPLITSTORE_BADGER_POOLED_STORAGE_KEY_001 //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 - testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { - opts := versions.DefaultOptions(path, false, badgerVersion) - opts.Prefix = "/prefixed/" - return opts - }) + for _, version := range SUPPORTED_BADGER_VERSIONS { + t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { + testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { + opts := versions.DefaultOptions(path, false, version) + opts.Prefix = "/prefixed/" + return opts + }) + }) + } } + From 3fd27074f8aac837883b2931b0a6950caab1fa73 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 08:21:57 -1000 Subject: [PATCH 33/59] fmt --- blockstore/badger/blockstore_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index 441007368da..779ceec8c6e 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -300,4 +300,3 @@ func TestMoveWithPrefix(t *testing.T) { }) } } - From 48f50a6b3812d09f90d498a3b03ce602a4b71a8c Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 09:43:34 -1000 Subject: [PATCH 34/59] keep ctx in doCopy --- blockstore/badger/blockstore.go | 9 ++- .../badger/versions/badger_interface.go | 2 +- blockstore/badger/versions/badger_v2.go | 64 +++++++++++++---- blockstore/badger/versions/badger_v4.go | 68 +++++++++++++------ 4 files changed, 106 insertions(+), 37 deletions(-) diff --git a/blockstore/badger/blockstore.go b/blockstore/badger/blockstore.go index f2d1fcba8ec..b3611dd2b47 100644 --- a/blockstore/badger/blockstore.go +++ b/blockstore/badger/blockstore.go @@ -287,7 +287,7 @@ func (b *Blockstore) movingGC(ctx context.Context) error { b.unlockMove(moveStateMoving) log.Info("copying blockstore") - err = b.doCopy(b.db, b.dbNext) + err = b.doCopy(ctx, b.db, b.dbNext) if err != nil { return fmt.Errorf("error moving badger blockstore to %s: %w", newPath, err) } @@ -349,10 +349,9 @@ func symlink(path, linkTo string) error { return os.Symlink(path, linkTo) } -// doCopy copies a badger blockstore to another, with an optional filter; if the filter -// is not nil, then only cids that satisfy the filter will be copied. -func (b *Blockstore) doCopy(from versions.BadgerDB, to versions.BadgerDB) error { - return from.Copy(to) +// doCopy copies a badger blockstore to another +func (b *Blockstore) doCopy(ctx context.Context, from versions.BadgerDB, to versions.BadgerDB) error { + return from.Copy(ctx, to) } func (b *Blockstore) deleteDB(path string) { diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 21e5171eff1..12d480323f1 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -26,7 +26,7 @@ type BadgerDB interface { NewWriteBatch() WriteBatch Flatten(workers int) error Size() (lsm int64, vlog int64) - Copy(to BadgerDB) error + Copy(ctx context.Context, to BadgerDB) error DefaultOptions(prefix string, readonly bool) Options Load(r io.Reader, maxPendingWrites int) error AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index cbd1ba4b5d4..e167db56d13 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -87,28 +87,68 @@ func (b *BadgerV2) Flatten(workers int) error { func (b *BadgerV2) Size() (lsm int64, vlog int64) { return b.DB.Size() } + +func (b *BadgerV2) Copy(ctx context.Context, to BadgerDB) (defErr error) { + + batch := to.NewWriteBatch() + defer func() { + if defErr == nil { + defErr = batch.Flush() + } + if defErr != nil { + batch.Cancel() + } + }() -func (b *BadgerV2) Copy(to BadgerDB) error { stream := b.DB.NewStream() - stream.LogPrefix = "doCopy" - stream.NumGo = clamp(runtime.NumCPU()/2, 2, 8) - stream.Send = func(list *pb.KVList) error { - batch := to.NewWriteBatch() - defer batch.Cancel() - for _, kv := range list.Kv { - if kv.Key == nil || kv.Value == nil { - continue - } + return iterateBadger(ctx, stream, func(kvs []*pb.KV) error { + // check whether context is closed on every kv group + if err := ctx.Err(); err != nil { + return err + } + for _, kv := range kvs { if err := batch.Set(kv.Key, kv.Value); err != nil { return err } } + return nil + }) +} + +var IterateLSMWorkers int // defaults to between( 2, 8, runtime.NumCPU/2 ) - return batch.Flush() +func iterateBadger(ctx context.Context, stream * badger.Stream, iter func([]*pb.KV) error) error { + workers := IterateLSMWorkers + if workers == 0 { + workers = between(2, 8, runtime.NumCPU()/2) + } + + stream.NumGo = workers + stream.LogPrefix = "iterateBadgerKVs" + stream.Send = func(kvl *pb.KVList) error { + kvs := make([]*pb.KV, 0, len(kvl.Kv)) + for _, kv := range kvl.Kv { + if kv.Key != nil && kv.Value != nil { + kvs = append(kvs, kv) + } + } + if len(kvs) == 0 { + return nil + } + return iter(kvs) } + return stream.Orchestrate(ctx) +} - return stream.Orchestrate(context.Background()) +func between(min, max, val int) int { + if val > max { + val = max + } + if val < min { + val = min + } + return val } func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 5f12783c103..13d7170c69e 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -9,7 +9,7 @@ import ( "strconv" badger "github.com/dgraph-io/badger/v4" - badgerV4 "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v4/pb" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" "github.com/ipfs/go-cid" @@ -87,39 +87,69 @@ func (b *BadgerV4) Flatten(workers int) error { func (b *BadgerV4) Size() (lsm int64, vlog int64) { return b.DB.Size() -} +} -func (b *BadgerV4) Copy(to BadgerDB) error { - stream := b.DB.NewStream() - stream.LogPrefix = "doCopy" - stream.NumGo = clamp(runtime.NumCPU()/2, 2, 8) - stream.Send = func(buf *z.Buffer) error { - list, err := badger.BufferToKVList(buf) - if err != nil { - return fmt.Errorf("buffer to KV list conversion: %w", err) + +func (b *BadgerV4) Copy(ctx context.Context, to BadgerDB) (defErr error) { + + batch := to.NewWriteBatch() + defer func() { + if defErr == nil { + defErr = batch.Flush() + } + if defErr != nil { + batch.Cancel() } + }() - batch := to.NewWriteBatch() - defer batch.Cancel() + stream := b.DB.NewStream() - for _, kv := range list.Kv { - if kv.Key == nil || kv.Value == nil { - continue - } + return iterateBadgerV4(ctx, stream, func(kvs []*pb.KV) error { + // check whether context is closed on every kv group + if err := ctx.Err(); err != nil { + return err + } + for _, kv := range kvs { if err := batch.Set(kv.Key, kv.Value); err != nil { return err } } + return nil + }) +} - return batch.Flush() +func iterateBadgerV4(ctx context.Context, stream * badger.Stream, iter func([]*pb.KV) error) error { + workers := IterateLSMWorkers + if workers == 0 { + workers = between(2, 8, runtime.NumCPU()/2) } - return stream.Orchestrate(context.Background()) + stream.NumGo = workers + stream.LogPrefix = "iterateBadgerKVs" + stream.Send = func(buf *z.Buffer) error { + kvl, err := badger.BufferToKVList(buf) + if err != nil { + return fmt.Errorf("buffer to KV list conversion: %w", err) + } + + kvs := make([]*pb.KV, 0, len(kvl.Kv)) + for _, kv := range kvl.Kv { + if kv.Key != nil && kv.Value != nil { + kvs = append(kvs, kv) + } + } + if len(kvs) == 0 { + return nil + } + return iter(kvs) + } + return stream.Orchestrate(ctx) } + func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { var opts Options - bopts := badgerV4.DefaultOptions(path) + bopts := badger.DefaultOptions(path) bopts.ReadOnly = readonly // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM From 2cac6a6fd6bfd2490cb49f48860ca882d3210cce Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 09:46:31 -1000 Subject: [PATCH 35/59] gofmt --- blockstore/badger/versions/badger_v4.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 13d7170c69e..83e90320a46 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -87,8 +87,7 @@ func (b *BadgerV4) Flatten(workers int) error { func (b *BadgerV4) Size() (lsm int64, vlog int64) { return b.DB.Size() -} - +} func (b *BadgerV4) Copy(ctx context.Context, to BadgerDB) (defErr error) { @@ -118,7 +117,7 @@ func (b *BadgerV4) Copy(ctx context.Context, to BadgerDB) (defErr error) { }) } -func iterateBadgerV4(ctx context.Context, stream * badger.Stream, iter func([]*pb.KV) error) error { +func iterateBadgerV4(ctx context.Context, stream *badger.Stream, iter func([]*pb.KV) error) error { workers := IterateLSMWorkers if workers == 0 { workers = between(2, 8, runtime.NumCPU()/2) @@ -146,7 +145,6 @@ func iterateBadgerV4(ctx context.Context, stream * badger.Stream, iter func([]*p return stream.Orchestrate(ctx) } - func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { var opts Options bopts := badger.DefaultOptions(path) From df59d5dd403862a59387483a066674fee2e1e240 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 09:48:34 -1000 Subject: [PATCH 36/59] gofmt --- blockstore/badger/versions/badger_v2.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index e167db56d13..b6684260ce7 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -87,7 +87,7 @@ func (b *BadgerV2) Flatten(workers int) error { func (b *BadgerV2) Size() (lsm int64, vlog int64) { return b.DB.Size() } - + func (b *BadgerV2) Copy(ctx context.Context, to BadgerDB) (defErr error) { batch := to.NewWriteBatch() @@ -118,7 +118,7 @@ func (b *BadgerV2) Copy(ctx context.Context, to BadgerDB) (defErr error) { var IterateLSMWorkers int // defaults to between( 2, 8, runtime.NumCPU/2 ) -func iterateBadger(ctx context.Context, stream * badger.Stream, iter func([]*pb.KV) error) error { +func iterateBadger(ctx context.Context, stream *badger.Stream, iter func([]*pb.KV) error) error { workers := IterateLSMWorkers if workers == 0 { workers = between(2, 8, runtime.NumCPU()/2) From 2a6504476b047b1a3d62598cd50f2a1e5bb87956 Mon Sep 17 00:00:00 2001 From: Mikers Date: Wed, 31 Jul 2024 09:53:42 -1000 Subject: [PATCH 37/59] clamp unused --- blockstore/badger/versions/badger.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 44b77925ad8..8392e2fcc75 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -48,13 +48,3 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { } return db, nil } - -func clamp(x, min, max int) int { - if x < min { - return min - } - if x > max { - return max - } - return x -} From 50cabd0da4e1d1c192471b270a67f5c34b039590 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 12 Aug 2024 13:07:04 -0700 Subject: [PATCH 38/59] use cfg.Chainstore with BadgerHotBlockstore --- node/builder_chain.go | 2 +- node/modules/blockstore.go | 56 +++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/node/builder_chain.go b/node/builder_chain.go index 109eba1b0e2..4632489375b 100644 --- a/node/builder_chain.go +++ b/node/builder_chain.go @@ -207,7 +207,7 @@ func ConfigFullNode(c interface{}) Option { If(cfg.Chainstore.Splitstore.ColdStoreType == "discard", Override(new(dtypes.ColdBlockstore), modules.DiscardColdBlockstore)), If(cfg.Chainstore.Splitstore.HotStoreType == "badger", - Override(new(dtypes.HotBlockstore), modules.BadgerHotBlockstore)), + Override(new(dtypes.HotBlockstore), modules.BadgerHotBlockstore(&cfg.Chainstore))), Override(new(dtypes.SplitBlockstore), modules.SplitBlockstore(&cfg.Chainstore)), Override(new(dtypes.BasicChainBlockstore), modules.ChainSplitBlockstore), Override(new(dtypes.BasicStateBlockstore), modules.StateSplitBlockstore), diff --git a/node/modules/blockstore.go b/node/modules/blockstore.go index 01e2998893f..ee4ad1df35d 100644 --- a/node/modules/blockstore.go +++ b/node/modules/blockstore.go @@ -45,42 +45,36 @@ func DiscardColdBlockstore(lc fx.Lifecycle, bs dtypes.UniversalBlockstore) (dtyp return blockstore.NewDiscardStore(bs), nil } -func BadgerHotBlockstore(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) { - path, err := r.SplitstorePath() - if err != nil { - return nil, err - } +func BadgerHotBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) { + return func(lc fx.Lifecycle, r repo.LockedRepo) (dtypes.HotBlockstore, error) { + path, err := r.SplitstorePath() + if err != nil { + return nil, err + } - path = filepath.Join(path, "hot.badger") - if err := os.MkdirAll(path, 0755); err != nil { - return nil, err - } + path = filepath.Join(path, "hot.badger") + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } - c, err := r.Config() - if err != nil { - return nil, err - } - cfg, ok := c.(*config.FullNode) - if !ok { - return nil, xerrors.Errorf("invalid config for repo, got: %T", c) - } - badgerVersion := cfg.Chainstore.BadgerVersion - opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly(), badgerVersion) - if err != nil { - return nil, err - } + badgerVersion := cfg.BadgerVersion + opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, r.Readonly(), badgerVersion) + if err != nil { + return nil, err + } - bs, err := badgerbs.Open(opts) - if err != nil { - return nil, err - } + bs, err := badgerbs.Open(opts) + if err != nil { + return nil, err + } - lc.Append(fx.Hook{ - OnStop: func(_ context.Context) error { - return bs.Close() - }}) + lc.Append(fx.Hook{ + OnStop: func(_ context.Context) error { + return bs.Close() + }}) - return bs, nil + return bs, nil + } } func SplitBlockstore(cfg *config.Chainstore) func(lc fx.Lifecycle, r repo.LockedRepo, ds dtypes.MetadataDS, cold dtypes.ColdBlockstore, hot dtypes.HotBlockstore) (dtypes.SplitBlockstore, error) { From cab56a83916d67d9c8607f9617f30324f2d0856b Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 12 Aug 2024 13:35:04 -0700 Subject: [PATCH 39/59] bugfix test all badger versions --- blockstore/badger/blockstore_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/blockstore/badger/blockstore_test.go b/blockstore/badger/blockstore_test.go index 779ceec8c6e..725a20f1dd4 100644 --- a/blockstore/badger/blockstore_test.go +++ b/blockstore/badger/blockstore_test.go @@ -114,13 +114,13 @@ func openBlockstore(optsSupplier func(path string, readonly bool, badgerVersion } } -func testMove(t *testing.T, optsF func(string, bool, int) versions.Options) { +func testMove(t *testing.T, badgerVersion int, optsF func(string, bool, int) versions.Options) { ctx := context.Background() basePath := t.TempDir() dbPath := filepath.Join(basePath, "db") - db, err := Open(optsF(dbPath, false, BADGER_VERSION_2)) + db, err := Open(optsF(dbPath, false, badgerVersion)) if err != nil { t.Fatal(err) } @@ -263,7 +263,7 @@ func testMove(t *testing.T, optsF func(string, bool, int) versions.Options) { t.Fatal(err) } - db, err = Open(optsF(dbPath, false, BADGER_VERSION_2)) + db, err = Open(optsF(dbPath, false, badgerVersion)) if err != nil { t.Fatal(err) } @@ -279,7 +279,7 @@ func TestMoveNoPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 for _, version := range SUPPORTED_BADGER_VERSIONS { t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { - testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { + testMove(t, version, func(path string, readonly bool, badgerVersion int) versions.Options { return versions.DefaultOptions(path, false, version) }) }) @@ -292,7 +292,7 @@ func TestMoveWithPrefix(t *testing.T) { //stm: @SPLITSTORE_BADGER_DELETE_001, @SPLITSTORE_BADGER_COLLECT_GARBAGE_001 for _, version := range SUPPORTED_BADGER_VERSIONS { t.Run(fmt.Sprintf("v%d", version), func(t *testing.T) { - testMove(t, func(path string, readonly bool, badgerVersion int) versions.Options { + testMove(t, version, func(path string, readonly bool, badgerVersion int) versions.Options { opts := versions.DefaultOptions(path, false, version) opts.Prefix = "/prefixed/" return opts From c4611608f4b7c935665472041fe76f575103ce1c Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 12 Aug 2024 13:56:46 -0700 Subject: [PATCH 40/59] remove dead code for DefaultOptions and restore comments and keep v2 and v4 options together. --- .../badger/versions/badger_interface.go | 1 - blockstore/badger/versions/badger_v2.go | 26 ------------- blockstore/badger/versions/badger_v4.go | 18 --------- blockstore/badger/versions/options.go | 38 +++++++++++++++++-- 4 files changed, 34 insertions(+), 49 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 12d480323f1..9cfb0e30d90 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -27,7 +27,6 @@ type BadgerDB interface { Flatten(workers int) error Size() (lsm int64, vlog int64) Copy(ctx context.Context, to BadgerDB) error - DefaultOptions(prefix string, readonly bool) Options Load(r io.Reader, maxPendingWrites int) error AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) DeleteBlock(context.Context, cid.Cid) error diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index b6684260ce7..b2cf0f22143 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -4,12 +4,9 @@ import ( "context" "fmt" "io" - "os" "runtime" - "strconv" badger "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" "github.com/ipfs/go-cid" @@ -151,29 +148,6 @@ func between(min, max, val int) int { return val } -func (b *BadgerV2) DefaultOptions(path string, readonly bool) Options { - var opts Options - - bopts := badger.DefaultOptions(path) - bopts.DetectConflicts = false - bopts.CompactL0OnClose = true - bopts.Truncate = true - bopts.ValueLogLoadingMode = options.MemoryMap - bopts.TableLoadingMode = options.MemoryMap - bopts.ValueThreshold = 128 - bopts.MaxTableSize = 64 << 20 - bopts.ReadOnly = readonly - - // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM - if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { - if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { - bopts.NumCompactors = numWorkers - } - } - opts.V2Options = bopts - return opts -} - func (b *BadgerV2) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 83e90320a46..8b6f3d8f692 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -4,9 +4,7 @@ import ( "context" "fmt" "io" - "os" "runtime" - "strconv" badger "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/badger/v4/pb" @@ -145,22 +143,6 @@ func iterateBadgerV4(ctx context.Context, stream *badger.Stream, iter func([]*pb return stream.Orchestrate(ctx) } -func (b *BadgerV4) DefaultOptions(path string, readonly bool) Options { - var opts Options - bopts := badger.DefaultOptions(path) - bopts.ReadOnly = readonly - - // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM - if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { - if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { - bopts.NumCompactors = numWorkers - } - } - opts.V4Options = bopts - return opts - -} - func (b *BadgerV4) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index e068013e657..577535899c3 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -40,23 +40,53 @@ func (o *Options) SetValueDir(valueDir string) { func BlockStoreOptions(path string, readonly bool, badgerVersion int) Options { opts := DefaultOptions(path, readonly, badgerVersion) + + // Due to legacy usage of blockstore.Blockstore, over a datastore, all + // blocks are prefixed with this namespace. In the future, this can go away, + // in order to shorten keys, but it'll require a migration. opts.Prefix = "/blocks/" + // Blockstore values are immutable; therefore we do not expect any + // conflicts to emerge. opts.V2Options.DetectConflicts = false + opts.V4Options.DetectConflicts = false + + // This is to optimize the database on close so it can be opened + // read-only and efficiently queried. opts.V2Options.CompactL0OnClose = true + opts.V4Options.CompactL0OnClose = true + + // The alternative is "crash on start and tell the user to fix it". This + // will truncate corrupt and unsynced data, which we don't guarantee to + // persist anyways. + // Badger V4 has no such option opts.V2Options.Truncate = true + + // We mmap the index and the value logs; this is important to enable + // zero-copy value access. opts.V2Options.ValueLogLoadingMode = optionsV2.MemoryMap opts.V2Options.TableLoadingMode = optionsV2.MemoryMap + + // Embed only values < 128 bytes in the LSM tree; larger values are stored + // in value logs. opts.V2Options.ValueThreshold = 128 + opts.V4Options.ValueThreshold = 128 + + // Default table size is already 64MiB. This is here to make it explicit. + // Badger V4 removed the option opts.V2Options.MaxTableSize = 64 << 20 - opts.V2Options.ReadOnly = readonly - opts.V4Options.DetectConflicts = false - opts.V4Options.CompactL0OnClose = true - opts.V4Options.ValueThreshold = 148 + // NOTE: The chain blockstore doesn't require any GC (blocks are never + // deleted). This will change if we move to a tiered blockstore. + + opts.V2Options.ReadOnly = readonly opts.V4Options.ReadOnly = readonly // Envvar LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM + // Allows the number of compaction workers used by BadgerDB to be adjusted + // Unset - leaves the default number of compaction workers (4) + // "0" - disables compaction + // Positive integer - enables that number of compaction workers if badgerNumCompactors, badgerNumCompactorsSet := os.LookupEnv("LOTUS_CHAIN_BADGERSTORE_COMPACTIONWORKERNUM"); badgerNumCompactorsSet { if numWorkers, err := strconv.Atoi(badgerNumCompactors); err == nil && numWorkers >= 0 { opts.V2Options.NumCompactors = numWorkers From 52604e948e28b80829f4fe131eec2a249aa40d26 Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 12 Aug 2024 14:08:11 -0700 Subject: [PATCH 41/59] remove unused AllKeysChan --- blockstore/badger/versions/badger_interface.go | 1 - blockstore/badger/versions/badger_v2.go | 4 ---- blockstore/badger/versions/badger_v4.go | 4 ---- 3 files changed, 9 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 9cfb0e30d90..28c980483a7 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -28,7 +28,6 @@ type BadgerDB interface { Size() (lsm int64, vlog int64) Copy(ctx context.Context, to BadgerDB) error Load(r io.Reader, maxPendingWrites int) error - AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) DeleteBlock(context.Context, cid.Cid) error Backup(w io.Writer, since uint64) (uint64, error) } diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index b2cf0f22143..0b270780bad 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -152,10 +152,6 @@ func (b *BadgerV2) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } -func (b *BadgerV2) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - return nil, fmt.Errorf("AllKeysChan is not implemented") -} - func (b *BadgerV2) DeleteBlock(context.Context, cid.Cid) error { return fmt.Errorf("DeleteBlock is not implemented") } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 8b6f3d8f692..adfc3f1e9c9 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -147,10 +147,6 @@ func (b *BadgerV4) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } -func (b *BadgerV4) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - return nil, fmt.Errorf("AllKeysChan is not implemented") -} - func (b *BadgerV4) DeleteBlock(context.Context, cid.Cid) error { return fmt.Errorf("DeleteBlock is not implemented") } From 4de5f718c123105dd98596b74c7dff302ff761af Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 12 Aug 2024 14:09:22 -0700 Subject: [PATCH 42/59] remove unused DeleteBlock method --- blockstore/badger/versions/badger_interface.go | 2 -- blockstore/badger/versions/badger_v2.go | 6 ------ blockstore/badger/versions/badger_v4.go | 5 ----- 3 files changed, 13 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 28c980483a7..6a1b5b19f0b 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -5,7 +5,6 @@ import ( "io" "github.com/dgraph-io/ristretto" - "github.com/ipfs/go-cid" ) // BadgerDB defines the common interface for both v2 and v4 versions of Badger. @@ -28,7 +27,6 @@ type BadgerDB interface { Size() (lsm int64, vlog int64) Copy(ctx context.Context, to BadgerDB) error Load(r io.Reader, maxPendingWrites int) error - DeleteBlock(context.Context, cid.Cid) error Backup(w io.Writer, since uint64) (uint64, error) } diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index 0b270780bad..e9d3806c2ed 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -2,14 +2,12 @@ package versions import ( "context" - "fmt" "io" "runtime" badger "github.com/dgraph-io/badger/v2" "github.com/dgraph-io/badger/v2/pb" "github.com/dgraph-io/ristretto" - "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) @@ -152,10 +150,6 @@ func (b *BadgerV2) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } -func (b *BadgerV2) DeleteBlock(context.Context, cid.Cid) error { - return fmt.Errorf("DeleteBlock is not implemented") -} - func (b *BadgerV2) Backup(w io.Writer, since uint64) (uint64, error) { return b.DB.Backup(w, since) } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index adfc3f1e9c9..09b0dcf3f3a 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -10,7 +10,6 @@ import ( "github.com/dgraph-io/badger/v4/pb" "github.com/dgraph-io/ristretto" "github.com/dgraph-io/ristretto/z" - "github.com/ipfs/go-cid" "golang.org/x/xerrors" ) @@ -147,10 +146,6 @@ func (b *BadgerV4) Load(r io.Reader, maxPendingWrites int) error { return b.DB.Load(r, maxPendingWrites) } -func (b *BadgerV4) DeleteBlock(context.Context, cid.Cid) error { - return fmt.Errorf("DeleteBlock is not implemented") -} - func (b *BadgerV4) Backup(w io.Writer, since uint64) (uint64, error) { return b.DB.Backup(w, since) } From d2de85e481011c3ef0ef81d9e30cdd3f6a4f5c3d Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 07:06:36 -0700 Subject: [PATCH 43/59] Update node/config/types.go Co-authored-by: Rod Vagg --- node/config/types.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/node/config/types.go b/node/config/types.go index 1e57b579932..70464d5f498 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -482,9 +482,17 @@ type Pubsub struct { } type Chainstore struct { + // EXPERIMENTAL FEATURE. USE WITH CAUTION + // BadgerVersion switches the version of the Badger database engine. The default is 2 which has + // been well-tested and widely deployed. Switching this to version 4 will enable the new version + // of Badger, but the blockstore will be incompatible with Lotus running with version 2. Switching + // versions can only be done when the blockstore is empty and will be repopulated from a snapshot + // or chain sync. It cannot be upgraded or downgraded with existing data and there is currently + // no automatic migration tooling. + // This is an experimental feature and should not be used in production. + BadgerVersion int EnableSplitstore bool Splitstore Splitstore - BadgerVersion int } type Splitstore struct { From ee737c967c282cfec208ae50e4fd490635244267 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 07:08:51 -0700 Subject: [PATCH 44/59] Update blockstore/badger/versions/badger.go Co-authored-by: Rod Vagg --- blockstore/badger/versions/badger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 8392e2fcc75..434211338e4 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -40,7 +40,7 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { db = BadgerDB(&BadgerV2{dbV2}) } default: - return nil, errors.New("unsupported badger version") + err = fmt.Errorf("unsupported badger version: %v", opts.BadgerVersion) } if err != nil { From 6d5d67918400c7629cc3f42f6b7bb5e4ad8f77fc Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 08:59:26 -0700 Subject: [PATCH 45/59] item.Version is unused --- blockstore/badger/versions/badger_interface.go | 1 - blockstore/badger/versions/badger_v2.go | 4 ---- blockstore/badger/versions/badger_v4.go | 4 ---- 3 files changed, 9 deletions(-) diff --git a/blockstore/badger/versions/badger_interface.go b/blockstore/badger/versions/badger_interface.go index 6a1b5b19f0b..ecad48c94ab 100644 --- a/blockstore/badger/versions/badger_interface.go +++ b/blockstore/badger/versions/badger_interface.go @@ -67,7 +67,6 @@ type Iterator interface { type Item interface { Value(fn func([]byte) error) error Key() []byte - Version() uint64 ValueCopy(dst []byte) ([]byte, error) ValueSize() int64 } diff --git a/blockstore/badger/versions/badger_v2.go b/blockstore/badger/versions/badger_v2.go index e9d3806c2ed..26e5f528c0f 100644 --- a/blockstore/badger/versions/badger_v2.go +++ b/blockstore/badger/versions/badger_v2.go @@ -265,10 +265,6 @@ func (item *BadgerV2Item) Key() []byte { return item.Item.Key() } -func (item *BadgerV2Item) Version() uint64 { - return item.Item.Version() -} - func (item *BadgerV2Item) ValueCopy(dst []byte) ([]byte, error) { return item.Item.ValueCopy(dst) } diff --git a/blockstore/badger/versions/badger_v4.go b/blockstore/badger/versions/badger_v4.go index 09b0dcf3f3a..1b1af46d1d9 100644 --- a/blockstore/badger/versions/badger_v4.go +++ b/blockstore/badger/versions/badger_v4.go @@ -264,10 +264,6 @@ func (item *BadgerV4Item) Key() []byte { return item.Item.Key() } -func (item *BadgerV4Item) Version() uint64 { - return item.Item.Version() -} - func (item *BadgerV4Item) ValueCopy(dst []byte) ([]byte, error) { return item.Item.ValueCopy(dst) } From c973d488d994bd96f48ae7b4e92e5fd5990d2cec Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 09:15:02 -0700 Subject: [PATCH 46/59] Update blockstore/badger/versions/badger.go Co-authored-by: Rod Vagg --- blockstore/badger/versions/badger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 434211338e4..75b65b6cb22 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -20,6 +20,7 @@ type BadgerLogger struct { func (b *BadgerLogger) Warningf(format string, args ...interface{}) { b.Skip2.Warnf(format, args...) } + func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error From 0dc3a8580c5ef4910141031133a867197ac2c4ff Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 09:16:17 -0700 Subject: [PATCH 47/59] Update blockstore/badger/versions/badger.go Co-authored-by: Rod Vagg --- blockstore/badger/versions/badger.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 75b65b6cb22..1c18bb8b042 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -25,9 +25,7 @@ func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error - version := opts.BadgerVersion - - switch version { + switch opts.BadgerVersion { case 4: var dbV4 *badgerV4.DB dbV4, err = badgerV4.Open(opts.V4Options) From 624779f27a33d5a89a552d2201a66811e5d3c195 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 09:18:34 -0700 Subject: [PATCH 48/59] lint and add import --- blockstore/badger/versions/badger.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/blockstore/badger/versions/badger.go b/blockstore/badger/versions/badger.go index 434211338e4..75dfd777d86 100644 --- a/blockstore/badger/versions/badger.go +++ b/blockstore/badger/versions/badger.go @@ -1,7 +1,7 @@ package versions import ( - "errors" + "fmt" badgerV2 "github.com/dgraph-io/badger/v2" badgerV4 "github.com/dgraph-io/badger/v4" @@ -20,6 +20,7 @@ type BadgerLogger struct { func (b *BadgerLogger) Warningf(format string, args ...interface{}) { b.Skip2.Warnf(format, args...) } + func OpenBadgerDB(opts Options) (BadgerDB, error) { var db BadgerDB var err error From 7b22c1c1f808547c980f5ab3057eb1d53912d3d4 Mon Sep 17 00:00:00 2001 From: Mikers Date: Tue, 13 Aug 2024 09:29:41 -0700 Subject: [PATCH 49/59] makegen --- documentation/en/default-lotus-config.toml | 17 +++++++++++++---- node/config/doc_gen.go | 19 +++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/documentation/en/default-lotus-config.toml b/documentation/en/default-lotus-config.toml index 647840680a6..ad418eadc60 100644 --- a/documentation/en/default-lotus-config.toml +++ b/documentation/en/default-lotus-config.toml @@ -149,14 +149,23 @@ [Chainstore] - # type: bool - # env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE - EnableSplitstore = true - + # EXPERIMENTAL FEATURE. USE WITH CAUTION + # BadgerVersion switches the version of the Badger database engine. The default is 2 which has + # been well-tested and widely deployed. Switching this to version 4 will enable the new version + # of Badger, but the blockstore will be incompatible with Lotus running with version 2. Switching + # versions can only be done when the blockstore is empty and will be repopulated from a snapshot + # or chain sync. It cannot be upgraded or downgraded with existing data and there is currently + # no automatic migration tooling. + # This is an experimental feature and should not be used in production. + # # type: int # env var: LOTUS_CHAINSTORE_BADGERVERSION #BadgerVersion = 2 + # type: bool + # env var: LOTUS_CHAINSTORE_ENABLESPLITSTORE + EnableSplitstore = true + [Chainstore.Splitstore] # ColdStoreType specifies the type of the coldstore. # It can be "discard" (default) for discarding cold blocks, "messages" to store only messages or "universal" to store all chain state.. diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index ac55febf3d6..ed1d84850f8 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -72,6 +72,19 @@ your node if metadata log is disabled`, }, }, "Chainstore": { + { + Name: "BadgerVersion", + Type: "int", + + Comment: `EXPERIMENTAL FEATURE. USE WITH CAUTION +BadgerVersion switches the version of the Badger database engine. The default is 2 which has +been well-tested and widely deployed. Switching this to version 4 will enable the new version +of Badger, but the blockstore will be incompatible with Lotus running with version 2. Switching +versions can only be done when the blockstore is empty and will be repopulated from a snapshot +or chain sync. It cannot be upgraded or downgraded with existing data and there is currently +no automatic migration tooling. +This is an experimental feature and should not be used in production.`, + }, { Name: "EnableSplitstore", Type: "bool", @@ -82,12 +95,6 @@ your node if metadata log is disabled`, Name: "Splitstore", Type: "Splitstore", - Comment: ``, - }, - { - Name: "BadgerVersion", - Type: "int", - Comment: ``, }, }, From 59fe7e3509f15968978b2d91cb4c243f23cf351e Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Thu, 15 Aug 2024 17:09:21 +0200 Subject: [PATCH 50/59] feat: f3: update go-f3 to 0.2.0 (#12390) * Update go-f3 to 0.2.0 Includes: - fix for excessive bandwidth usage - significant performance improvements - minor consensus fixes Signed-off-by: Jakub Sztandera * add changelog Signed-off-by: Jakub Sztandera * chore(f3): update to final released version --------- Signed-off-by: Jakub Sztandera Co-authored-by: Steven Allen --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14d892a7f1e..74c0b2a25d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ # UNRELEASED +- https://github.com/filecoin-project/lotus/pull/12390: Update go-f3 to 0.2.0 - https://github.com/filecoin-project/lotus/pull/12203: Fix slice modification bug in ETH Tx Events Bloom Filter - https://github.com/filecoin-project/lotus/pull/12221: Fix a nil reference panic in the ETH Trace API - https://github.com/filecoin-project/lotus/pull/12112: Moved consts from build/ to build/buildconstants/ for ligher curio deps. diff --git a/go.mod b/go.mod index 047f1500084..4e81565aa27 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/filecoin-project/go-cbor-util v0.0.1 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 github.com/filecoin-project/go-crypto v0.0.1 - github.com/filecoin-project/go-f3 v0.1.0 + github.com/filecoin-project/go-f3 v0.2.0 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 github.com/filecoin-project/go-jsonrpc v0.6.0 diff --git a/go.sum b/go.sum index d2711be15e6..b6381847445 100644 --- a/go.sum +++ b/go.sum @@ -268,8 +268,8 @@ github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o= github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-f3 v0.1.0 h1:W/AcaF3FBg00Eiyuz/HIs5hCcPtVMK7JncWXpg4MP98= -github.com/filecoin-project/go-f3 v0.1.0/go.mod h1:43fBLX0iX0+Nnw4Z91wSrdfDYAd6YEDexy7GcLnIJtk= +github.com/filecoin-project/go-f3 v0.2.0 h1:Gis44+hOrDjSUEw3IDmU7CudNILi5e+bb1pgZgp680k= +github.com/filecoin-project/go-f3 v0.2.0/go.mod h1:43fBLX0iX0+Nnw4Z91wSrdfDYAd6YEDexy7GcLnIJtk= github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commp-hashhash v0.2.0 h1:HYIUugzjq78YvV3vC6rL95+SfC/aSTVSnZSZiDV5pCk= From d415d9f384af8d8996011598f8d1daa5bfcecc40 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Thu, 15 Aug 2024 14:40:44 -0700 Subject: [PATCH 51/59] docs: update references to releases branch (#12396) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 90436576728..5c088bf706e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more ## Building & Documentation -> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the most recent [`Latest release`](https://github.com/filecoin-project/lotus/releases). +> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the [latest release](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives). For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme. @@ -83,9 +83,9 @@ Once all the dependencies are installed, you can build and install the Lotus sui cd lotus/ ``` -Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the latest release[ here](https://github.com/filecoin-project/lotus/releases). +Note: The default branch `master` is the dev branch where the latest new features, bug fixes and improvement are in. However, if you want to run lotus on Filecoin mainnet and want to run a production-ready lotus, get the [latest release](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives). -2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/releases). +2. To join mainnet, checkout the [latest release](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives). If you are changing networks from a previous Lotus installation or there has been a network reset, read the [Switch networks guide](https://lotus.filecoin.io/lotus/manage/switch-networks/) before proceeding. From 517c7aeebf0ec4bb7a557fad48955513b0a7d4e1 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 19 Aug 2024 17:25:35 +1000 Subject: [PATCH 52/59] fix(ci): don't PR or changelog check for draft PRs (#12405) --- .github/workflows/changelog.yml | 4 +++- .github/workflows/pr-title-check.yml | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 5a2de8703bf..c96bbe3b2c3 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -9,6 +9,7 @@ on: - reopened - labeled - unlabeled + - ready_for_review paths: - '**.go' - '**/go.mod' @@ -17,7 +18,8 @@ on: jobs: changelog: if: contains(github.event.pull_request.title, '[skip changelog]') == false && - contains(github.event.pull_request.labels.*.name, 'skip/changelog') == false + contains(github.event.pull_request.labels.*.name, 'skip/changelog') == false && + github.event.pull_request.draft == false runs-on: ubuntu-latest name: Changelog steps: diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 84c7cae525f..f9a1deffb24 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -2,13 +2,18 @@ name: PR Title Check on: pull_request_target: - types: [opened, edited, synchronize, reopened] + types: + - opened + - edited + - reopened + - ready_for_review permissions: pull-requests: write jobs: check-pr-title: + if: github.event.pull_request.draft == false name: Check PR Title runs-on: ubuntu-latest steps: From 0225c915a157424b73c123d570ad59fbcbfe1baf Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 19 Aug 2024 13:47:44 +0200 Subject: [PATCH 53/59] chore: post release steps for #12379 (v1.28.2 miner and node patch release) (#12400) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: lotus-miner: remove provecommit1 method (#12251) * remove provecommit1 * add changelog * update precommit and commit params * fix lint error * fix commit params * dep: f3: Update go-f3 to 0.0.6, enable it on mainnet (#12295) * Update go-f3 to 0.0.6 Signed-off-by: Jakub Sztandera * Enable F3 in passive configuration in mainnet config Signed-off-by: Jakub Sztandera * Add changelog Signed-off-by: Jakub Sztandera * add new butterfly assets --------- Signed-off-by: Jakub Sztandera Co-authored-by: Jennifer Wang * retract v1.28.0 * update v1.28.0 changelog and add v1.28.1 * Update CHANGELOG.md * wip - update f3 * don't convert bigint type We now use the same one in GPBFT. * update docs * fix wrong param name * update butterfy assets * update go-f3 * update changelog * update version * fix typo * Update CHANGELOG.md Co-authored-by: Steven Allen * Update CHANGELOG.md Co-authored-by: Rod Vagg * Update CHANGELOG.md Co-authored-by: Rod Vagg * apply f3 patch * chore: bump versions and make gen/docsgen-cli chore: bump versions and make gen/docsgen-cli * chore: update v1.28.2 changelog chore: update v1.282. changelog * feat: f3: update go-f3 to 0.2.0 (#12390) * Update go-f3 to 0.2.0 Includes: - fix for excessive bandwidth usage - significant performance improvements - minor consensus fixes Signed-off-by: Jakub Sztandera * add changelog Signed-off-by: Jakub Sztandera * chore(f3): update to final released version --------- Signed-off-by: Jakub Sztandera Co-authored-by: Steven Allen * fix!: sealer: handle initialisation error without panic storage/pipeline.NewPreCommitBatcher and storage/pipeline.New now have an additional error return to deal with errors arising from fetching the sealing config. * add breaking API upgrade warning to the ChangeLog * NewCommitBatcher now has an additional error return to deal with errors arising from fetching the sealing config. * fix: miner: Fix DDO pledge math (#12341) * Power is units of Space * Time so multiply by deal duration * fix: miner: Fix DDO pledge math * appease the changelog checker * Fix gen --------- Co-authored-by: zenground0 * chore: fix lint error - Updated the logging statement in `testOutOfGasError` to correctly reference `build.BlockGasLimit` instead of `buildconstants.BlockGasLimit`. * fix: update changelog to reference bandwidth issue ticket fix: update changelog to reference bandwidth issue ticket * Update CHANGELOG.md Co-authored-by: Steve Loeppky * Update CHANGELOG.md * chore: make gen and make docsgen-cli Run `make gen` and `make docsgen-cli` --------- Signed-off-by: Jakub Sztandera Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Co-authored-by: Jakub Sztandera Co-authored-by: Jennifer Wang Co-authored-by: Jiaying Wang <42981373+jennijuju@users.noreply.github.com> Co-authored-by: Steven Allen Co-authored-by: Rod Vagg Co-authored-by: aarshkshah1992 Co-authored-by: Łukasz Magiera Co-authored-by: zenground0 Co-authored-by: Steve Loeppky --- CHANGELOG.md | 19 +++++++++++++++++-- build/openrpc/full.json | 2 +- build/openrpc/gateway.json | 2 +- build/openrpc/miner.json | 2 +- build/openrpc/worker.json | 2 +- build/version.go | 2 +- documentation/en/cli-lotus.md | 2 +- 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c0b2a25d7..19cd54b8653 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,6 @@ # UNRELEASED -- https://github.com/filecoin-project/lotus/pull/12390: Update go-f3 to 0.2.0 - https://github.com/filecoin-project/lotus/pull/12203: Fix slice modification bug in ETH Tx Events Bloom Filter - https://github.com/filecoin-project/lotus/pull/12221: Fix a nil reference panic in the ETH Trace API - https://github.com/filecoin-project/lotus/pull/12112: Moved consts from build/ to build/buildconstants/ for ligher curio deps. @@ -19,7 +18,6 @@ - https://github.com/filecoin-project/lotus/pull/12319: feat: `lotus send CLI`: allow sending to ETH addresses - https://github.com/filecoin-project/lotus/pull/12332: fix: ETH RPC: receipts: use correct txtype in receipts - https://github.com/filecoin-project/lotus/pull/12335: fix: lotus-shed: store processed tipset after backfilling events -- https://github.com/filecoin-project/lotus/pull/12341: fix: miner: Fix DDO pledge math ## ☢️ Upgrade Warnings ☢️ @@ -50,6 +48,23 @@ - The default value for the `Events.FilterTTL` config option has been reduced from 24h to 1h. This means that filters will expire on a Lotus node after 1 hour of not being accessed by the client. - feat(f3): F3 has been updated with many performance improvements and additional metrics. +# 1.28.2 / 2024-08-15 + +This is a Lotus patch release v1.28.2 for Node operators and Storage Providers. + +For node operators, this patch release is HIGHLY RECOMMENDED as it fixes an issue where excessive bandwidth usage (issue #12381) was caused by a routing loop in pubsub, where small "manifest" messages were cycling repeatedly around the network due to an ineffective routing loop prevention mechanism. The new f3 release also has a couple performance improvements around CPU usage. (If you are curious about the progress of F3 testing, follow the updates [here](https://github.com/filecoin-project/lotus/discussions/12287#discussioncomment-10343447)). + +For storage providers, this patch release fixes pledge issues users have been encountering. This update addresses existing issues, including the too-small pledge in snap and the lack of DDO-awareness in PoRep Commit. + +## ☢️ Upgrade Warnings ☢️ +- The `releases` branch has been deprecated with the 202408 split of 'Lotus Node' and 'Lotus Miner'. See https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives for more info and alternatives for getting the latest release for both the 'Lotus Node' and 'Lotus Miner' based on the [Branch and Tag Strategy](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#branch-and-tag-strategy). + - To get the latest Lotus Node tag: `git tag -l 'v*' | sort -V -r | head -n 1` + - To get the latest Lotus Miner tag: `git tag -l 'miner/v*' | sort -V -r | head -n 1` +- Breaking change in Miner public APIs `storage/pipeline.NewPreCommitBatcher` and `storage/pipeline.New`. They now have an additional error return to deal with errors arising from fetching the sealing config. + +- https://github.com/filecoin-project/lotus/pull/12390: Update go-f3 to 0.2.0 +- https://github.com/filecoin-project/lotus/pull/12341: fix: miner: Fix DDO pledge math + # v1.28.1 / 2024-07-24 This is the MANDATORY Lotus v1.28.1 release, which will deliver the Filecoin network version 23, codenamed Waffle 🧇. v1.28.1 is also the minimal version that supports nv23. diff --git a/build/openrpc/full.json b/build/openrpc/full.json index 42d764b59af..ba4c78ffbeb 100644 --- a/build/openrpc/full.json +++ b/build/openrpc/full.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.2-dev" + "version": "1.28.3-dev" }, "methods": [ { diff --git a/build/openrpc/gateway.json b/build/openrpc/gateway.json index c48d4a27508..0e7a3545743 100644 --- a/build/openrpc/gateway.json +++ b/build/openrpc/gateway.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.2-dev" + "version": "1.28.3-dev" }, "methods": [ { diff --git a/build/openrpc/miner.json b/build/openrpc/miner.json index a2a5ae75af3..103ec81d389 100644 --- a/build/openrpc/miner.json +++ b/build/openrpc/miner.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.2-dev" + "version": "1.28.3-dev" }, "methods": [ { diff --git a/build/openrpc/worker.json b/build/openrpc/worker.json index ed08b043887..83364cbdcd4 100644 --- a/build/openrpc/worker.json +++ b/build/openrpc/worker.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.2-dev" + "version": "1.28.3-dev" }, "methods": [ { diff --git a/build/version.go b/build/version.go index 1da3be510aa..eeca74c3a0e 100644 --- a/build/version.go +++ b/build/version.go @@ -7,7 +7,7 @@ import ( ) // NodeBuildVersion is the local build version of the Lotus daemon -const NodeBuildVersion string = "1.28.2-dev" +const NodeBuildVersion string = "1.28.3-dev" func NodeUserVersion() BuildVersion { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 980b8be11fd..f32d701c731 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.28.2-dev + 1.28.3-dev COMMANDS: daemon Start a lotus daemon process From dbef5ded1585bcdcf84bbe09f3908210f9775ef6 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 19 Aug 2024 17:16:34 +0000 Subject: [PATCH 54/59] feat(libp2p): expose libp2p bandwidth metrics (#12402) This exposes bandwidth metrics via async callback to avoid allocating/reporting metrics on any hot-paths. I'm using open telemetry as we've already setup a bridge for F3 and opencensus is deprecated in favor of open telemetry (so we're going to slowly move over anyways). --- CHANGELOG.md | 1 + chain/lf3/f3.go | 17 ----------------- go.mod | 2 +- metrics/otel_bridge.go | 21 +++++++++++++++++++++ node/modules/lp2p/host.go | 6 ------ node/modules/lp2p/metrics.go | 30 ++++++++++++++++++++++++++++++ node/modules/lp2p/transport.go | 34 ++++++++++++++++++++++++++++++++-- 7 files changed, 85 insertions(+), 26 deletions(-) create mode 100644 metrics/otel_bridge.go create mode 100644 node/modules/lp2p/metrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 19cd54b8653..799cf0d6483 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ - Some APIs have changed which may impact users consuming Lotus Gateway code as a library. - The default value for the `Events.FilterTTL` config option has been reduced from 24h to 1h. This means that filters will expire on a Lotus node after 1 hour of not being accessed by the client. - feat(f3): F3 has been updated with many performance improvements and additional metrics. +- feat(libp2p): expose libp2p bandwidth metrics via Prometheus. # 1.28.2 / 2024-08-15 diff --git a/chain/lf3/f3.go b/chain/lf3/f3.go index 2514320ea7b..eba612e77b1 100644 --- a/chain/lf3/f3.go +++ b/chain/lf3/f3.go @@ -10,9 +10,6 @@ import ( logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/prometheus" - "go.opentelemetry.io/otel/sdk/metric" "go.uber.org/fx" "golang.org/x/xerrors" @@ -53,20 +50,6 @@ type F3Params struct { var log = logging.Logger("f3") -func init() { - // Set up otel to prometheus reporting so that F3 metrics are reported via lotus - // prometheus metrics. This bridge eventually gets picked up by opencensus - // exporter as HTTP handler. This by default registers an otel collector against - // the global prometheus registry. In the future, we should clean up metrics in - // Lotus and move it all to use otel. For now, bridge away. - if bridge, err := prometheus.New(); err != nil { - log.Errorf("could not create the otel prometheus exporter: %v", err) - } else { - provider := metric.NewMeterProvider(metric.WithReader(bridge)) - otel.SetMeterProvider(provider) - } -} - func New(mctx helpers.MetricsCtx, lc fx.Lifecycle, params F3Params) (*F3, error) { ds := namespace.Wrap(params.Datastore, datastore.NewKey("/f3")) diff --git a/go.mod b/go.mod index 4e81565aa27..9e29ce2b3a3 100644 --- a/go.mod +++ b/go.mod @@ -147,6 +147,7 @@ require ( go.opentelemetry.io/otel/bridge/opencensus v1.28.0 go.opentelemetry.io/otel/exporters/jaeger v1.14.0 go.opentelemetry.io/otel/exporters/prometheus v0.50.0 + go.opentelemetry.io/otel/metric v1.28.0 go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 go.uber.org/atomic v1.11.0 @@ -323,7 +324,6 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/mock v0.4.0 // indirect diff --git a/metrics/otel_bridge.go b/metrics/otel_bridge.go new file mode 100644 index 00000000000..76572f5afb2 --- /dev/null +++ b/metrics/otel_bridge.go @@ -0,0 +1,21 @@ +package metrics + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" +) + +func init() { + // Set up otel to prometheus reporting so that F3 metrics are reported via lotus + // prometheus metrics. This bridge eventually gets picked up by opencensus + // exporter as HTTP handler. This by default registers an otel collector against + // the global prometheus registry. In the future, we should clean up metrics in + // Lotus and move it all to use otel. For now, bridge away. + if bridge, err := prometheus.New(); err != nil { + log.Errorf("could not create the otel prometheus exporter: %v", err) + } else { + provider := metric.NewMeterProvider(metric.WithReader(bridge)) + otel.SetMeterProvider(provider) + } +} diff --git a/node/modules/lp2p/host.go b/node/modules/lp2p/host.go index baea4cf0656..fd723fbb2c4 100644 --- a/node/modules/lp2p/host.go +++ b/node/modules/lp2p/host.go @@ -62,12 +62,6 @@ func Host(mctx helpers.MetricsCtx, buildVersion build.BuildVersion, lc fx.Lifecy return nil, err } - lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { - return h.Close() - }, - }) - return h, nil } diff --git a/node/modules/lp2p/metrics.go b/node/modules/lp2p/metrics.go new file mode 100644 index 00000000000..1d0d6473b52 --- /dev/null +++ b/node/modules/lp2p/metrics.go @@ -0,0 +1,30 @@ +package lp2p + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +var otelmeter = otel.Meter("libp2p") + +var attrIdentity = attribute.Key("identity") +var attrProtocolID = attribute.Key("protocol") +var attrDirectionInbound = attribute.String("direction", "inbound") +var attrDirectionOutbound = attribute.String("direction", "outbound") + +var otelmetrics = struct { + bandwidth metric.Int64ObservableCounter +}{ + bandwidth: must(otelmeter.Int64ObservableCounter("lotus_libp2p_bandwidth", + metric.WithDescription("Libp2p stream bandwidth."), + metric.WithUnit("By"), + )), +} + +func must[T any](v T, err error) T { + if err != nil { + panic(err) + } + return v +} diff --git a/node/modules/lp2p/transport.go b/node/modules/lp2p/transport.go index 536f612b1aa..6a95937d284 100644 --- a/node/modules/lp2p/transport.go +++ b/node/modules/lp2p/transport.go @@ -1,11 +1,16 @@ package lp2p import ( + "context" + "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/metrics" + "github.com/libp2p/go-libp2p/core/peer" noise "github.com/libp2p/go-libp2p/p2p/security/noise" tls "github.com/libp2p/go-libp2p/p2p/security/tls" libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic" + "go.opentelemetry.io/otel/metric" + "go.uber.org/fx" ) var DefaultTransports = simpleOpt(libp2p.DefaultTransports) @@ -31,8 +36,33 @@ func Security(enabled, preferTLS bool) interface{} { } } -func BandwidthCounter() (opts Libp2pOpts, reporter metrics.Reporter) { +func BandwidthCounter(lc fx.Lifecycle, id peer.ID) (opts Libp2pOpts, reporter metrics.Reporter, err error) { reporter = metrics.NewBandwidthCounter() opts.Opts = append(opts.Opts, libp2p.BandwidthReporter(reporter)) - return opts, reporter + + // Register it with open telemetry. We report by-callback instead of implementing a custom + // bandwidth counter to avoid allocating every time we read/write to a stream (and to stay + // out of the hot path). + // + // Identity is required to ensure this observer observes with unique attributes. + identityAttr := attrIdentity.String(id.String()) + registration, err := otelmeter.RegisterCallback(func(ctx context.Context, obs metric.Observer) error { + for p, bw := range reporter.GetBandwidthByProtocol() { + if p == "" { + p = "" + } + protoAttr := attrProtocolID.String(string(p)) + obs.ObserveInt64(otelmetrics.bandwidth, bw.TotalOut, + metric.WithAttributes(identityAttr, protoAttr, attrDirectionOutbound)) + obs.ObserveInt64(otelmetrics.bandwidth, bw.TotalIn, + metric.WithAttributes(identityAttr, protoAttr, attrDirectionInbound)) + } + return nil + }, otelmetrics.bandwidth) + if err != nil { + return Libp2pOpts{}, nil, err + } + lc.Append(fx.StopHook(registration.Unregister)) + + return opts, reporter, nil } From a5cb674bb11d3a438e5b9d9ca3da218f2941a7ac Mon Sep 17 00:00:00 2001 From: qwdsds <50311944+qwdsds@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:30:39 +0800 Subject: [PATCH 55/59] fix: error check --- storage/pipeline/pledge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/pipeline/pledge.go b/storage/pipeline/pledge.go index 7cdda9fa4cd..890b4e7248e 100644 --- a/storage/pipeline/pledge.go +++ b/storage/pipeline/pledge.go @@ -104,7 +104,7 @@ func (m *Sealing) sectorWeight(ctx context.Context, sector SectorInfo, expiratio alloc, err := piece.GetAllocation(ctx, m.Api, ts.Key()) if err != nil || alloc == nil { - if err == nil { + if err != nil { log.Errorw("failed to get allocation", "error", err) } w = big.Add(w, big.Mul(sectorDuration, abi.NewStoragePower(int64(piece.Piece().Size)))) From 8518d23289b800eb749ac7aff3bdb6e923018d4f Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Thu, 22 Aug 2024 12:25:28 +0200 Subject: [PATCH 56/59] build: update Lotus Node version to v1.29.1-dev in master (#12409) * chore: update Node version chore: update Node version * chore: cleanup unreleased changelog section chore: cleanup unreleased changelog section --- CHANGELOG.md | 41 ----------------------------------- build/openrpc/full.json | 2 +- build/openrpc/gateway.json | 2 +- build/openrpc/miner.json | 2 +- build/openrpc/worker.json | 2 +- build/version.go | 2 +- documentation/en/cli-lotus.md | 2 +- 7 files changed, 6 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 799cf0d6483..2639e38e496 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,53 +2,12 @@ # UNRELEASED -- https://github.com/filecoin-project/lotus/pull/12203: Fix slice modification bug in ETH Tx Events Bloom Filter -- https://github.com/filecoin-project/lotus/pull/12221: Fix a nil reference panic in the ETH Trace API -- https://github.com/filecoin-project/lotus/pull/12112: Moved consts from build/ to build/buildconstants/ for ligher curio deps. -- https://github.com/filecoin-project/lotus/pull/12237: Upgrade to go-f3 `v0.0.4`. -- https://github.com/filecoin-project/lotus/pull/12251: Dropping support from ProveCommitSector1 method from lotus-miner -- https://github.com/filecoin-project/lotus/pull/12276: chore: deps: Update GST, Filecoin-FFI and Actors to final versions NV23 -- https://github.com/filecoin-project/lotus/pull/12278: chore: Set Mainnet upgrade epoch for NV23. -- https://github.com/filecoin-project/lotus/pull/12269 Fix `logIndex` ordering in `EthGetTransactionReceipt` by using the EventIndex to fetch logs -- https://github.com/filecoin-project/lotus/pull/12270: Feat expose `settle-deal` command for lotus miner to settle deals manually -- https://github.com/filecoin-project/lotus/pull/12285 Set up OpenTelemetry metrics reporting to prometheus -- https://github.com/filecoin-project/lotus/pull/12279 Upgrade to go-f3 v0.0.5 -- https://github.com/filecoin-project/lotus/pull/12295 Upgrade to go-f3 v0.0.6 -- https://github.com/filecoin-project/lotus/pull/12292: feat: p2p: allow overriding bootstrap nodes with environmemnt variable -- https://github.com/filecoin-project/lotus/pull/12319: feat: `lotus send CLI`: allow sending to ETH addresses -- https://github.com/filecoin-project/lotus/pull/12332: fix: ETH RPC: receipts: use correct txtype in receipts -- https://github.com/filecoin-project/lotus/pull/12335: fix: lotus-shed: store processed tipset after backfilling events - ## ☢️ Upgrade Warnings ☢️ -- lotus-gateway behaviour, CLI arguments and APIs have received minor changes. See the improvements section below. - ## New features -- feat: Add trace filter API supporting RPC method `trace_filter` ([filecoin-project/lotus#12123](https://github.com/filecoin-project/lotus/pull/12123)). Configuring `EthTraceFilterMaxResults` sets a limit on how many results are returned in any individual `trace_filter` RPC API call. -- feat: `FilecoinAddressToEthAddress` RPC can now return ETH addresses for all Filecoin address types ("f0"/"f1"/"f2"/"f3") based on client's re-org tolerance. This is a breaking change if you are using the API via the go-jsonrpc library or by using Lotus as a library, but is a non-breaking change when using the API via any other RPC method as it adds an optional second argument. -([filecoin-project/lotus#12324](https://github.com/filecoin-project/lotus/pull/12324)). -- feat: Added `lotus-shed indexes inspect-events` health-check command ([filecoin-project/lotus#12346](https://github.com/filecoin-project/lotus/pull/12346)). - -## Enhancements - -- fix: add datacap balance to circ supply internal accounting as unCirc #12348 -- feat: Use a block cache to speed up the `EthGetBlockByHash` API: https://github.com/filecoin-project/lotus/pull/12359 - ## Improvements -- feat!: gateway: fix rate limiting, better stateful handling ([filecoin-project/lotus#12315](https://github.com/filecoin-project/lotus/pull/12315)). - - CLI usage documentation has been improved for `lotus-gateway` - - `--per-conn-rate-limit` now works as advertised. - - `--eth-max-filters-per-conn` is new and allows you to set the maximum number of filters and subscription per connection, it defaults to 16. - - Previously, this limit was set to `16` and applied separately to filters and subscriptions. This limit is now unified and applies to both filters and subscriptions. - - Stateful Ethereum APIs (those involving filters and subscriptions) are now disabled for plain HTTP connections. A client must be using websockets to access these APIs. - - These APIs are also now automatically removed from the node by the gateway when a client disconnects. - - Some APIs have changed which may impact users consuming Lotus Gateway code as a library. - - The default value for the `Events.FilterTTL` config option has been reduced from 24h to 1h. This means that filters will expire on a Lotus node after 1 hour of not being accessed by the client. -- feat(f3): F3 has been updated with many performance improvements and additional metrics. -- feat(libp2p): expose libp2p bandwidth metrics via Prometheus. - # 1.28.2 / 2024-08-15 This is a Lotus patch release v1.28.2 for Node operators and Storage Providers. diff --git a/build/openrpc/full.json b/build/openrpc/full.json index ba4c78ffbeb..fbb37165ab4 100644 --- a/build/openrpc/full.json +++ b/build/openrpc/full.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.3-dev" + "version": "1.29.1-dev" }, "methods": [ { diff --git a/build/openrpc/gateway.json b/build/openrpc/gateway.json index 0e7a3545743..171febdee99 100644 --- a/build/openrpc/gateway.json +++ b/build/openrpc/gateway.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.3-dev" + "version": "1.29.1-dev" }, "methods": [ { diff --git a/build/openrpc/miner.json b/build/openrpc/miner.json index 103ec81d389..0ba3fde3e16 100644 --- a/build/openrpc/miner.json +++ b/build/openrpc/miner.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.3-dev" + "version": "1.29.1-dev" }, "methods": [ { diff --git a/build/openrpc/worker.json b/build/openrpc/worker.json index 83364cbdcd4..f0a3b9e7ad5 100644 --- a/build/openrpc/worker.json +++ b/build/openrpc/worker.json @@ -2,7 +2,7 @@ "openrpc": "1.2.6", "info": { "title": "Lotus RPC API", - "version": "1.28.3-dev" + "version": "1.29.1-dev" }, "methods": [ { diff --git a/build/version.go b/build/version.go index eeca74c3a0e..0a72a46e766 100644 --- a/build/version.go +++ b/build/version.go @@ -7,7 +7,7 @@ import ( ) // NodeBuildVersion is the local build version of the Lotus daemon -const NodeBuildVersion string = "1.28.3-dev" +const NodeBuildVersion string = "1.29.1-dev" func NodeUserVersion() BuildVersion { if os.Getenv("LOTUS_VERSION_IGNORE_COMMIT") == "1" { diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index f32d701c731..3996f0c3654 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -7,7 +7,7 @@ USAGE: lotus [global options] command [command options] [arguments...] VERSION: - 1.28.3-dev + 1.29.1-dev COMMANDS: daemon Start a lotus daemon process From 475139ff95407ed9d55d3a2ef87e28da66512937 Mon Sep 17 00:00:00 2001 From: Peter Rabbitson Date: Fri, 23 Aug 2024 02:50:48 +0200 Subject: [PATCH 57/59] chore: deps: update to CGO-free go-crypto (#12411) --- chain/messagepool/messagepool_test.go | 2 +- go.mod | 5 +++-- go.sum | 8 ++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/chain/messagepool/messagepool_test.go b/chain/messagepool/messagepool_test.go index cb7fc5f8239..77bd1679146 100644 --- a/chain/messagepool/messagepool_test.go +++ b/chain/messagepool/messagepool_test.go @@ -954,7 +954,7 @@ func TestMessageSignatureInvalid(t *testing.T) { } err = mp.Add(context.TODO(), sm) assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid signature length") + assert.Contains(t, err.Error(), "invalid signature for message bafy2bz") } } diff --git a/go.mod b/go.mod index 9e29ce2b3a3..f5e5b950e8c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.1 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 - github.com/filecoin-project/go-crypto v0.0.1 + github.com/filecoin-project/go-crypto v0.1.0 github.com/filecoin-project/go-f3 v0.2.0 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-hamt-ipld/v3 v3.4.0 @@ -239,7 +239,6 @@ require ( github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.3 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect github.com/jackc/pgx/v5 v5.6.0 // indirect @@ -324,6 +323,8 @@ require ( github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect + gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect + gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/mock v0.4.0 // indirect diff --git a/go.sum b/go.sum index b6381847445..16a8178f19a 100644 --- a/go.sum +++ b/go.sum @@ -266,8 +266,8 @@ github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20240802040721-2a04ffc8 github.com/filecoin-project/go-commp-utils/v2 v2.1.0 h1:KWNRalUp2bhN1SW7STsJS2AHs9mnfGKk9LnQgzDe+gI= github.com/filecoin-project/go-commp-utils/v2 v2.1.0/go.mod h1:NbxJYlhxtWaNhlVCj/gysLNu26kYII83IV5iNrAO9iI= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= -github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o= -github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-crypto v0.1.0 h1:Pob2MphoipMbe/ksxZOMcQvmBHAd3sI/WEqcbpIsGI0= +github.com/filecoin-project/go-crypto v0.1.0/go.mod h1:K9UFXvvoyAVvB+0Le7oGlKiT9mgA5FHOJdYQXEE8IhI= github.com/filecoin-project/go-f3 v0.2.0 h1:Gis44+hOrDjSUEw3IDmU7CudNILi5e+bb1pgZgp680k= github.com/filecoin-project/go-f3 v0.2.0/go.mod h1:43fBLX0iX0+Nnw4Z91wSrdfDYAd6YEDexy7GcLnIJtk= github.com/filecoin-project/go-fil-commcid v0.1.0 h1:3R4ds1A9r6cr8mvZBfMYxTS88OqLYEo6roi+GiIeOh8= @@ -1342,6 +1342,10 @@ github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfU github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI= github.com/zyedidia/generic v1.2.1 h1:Zv5KS/N2m0XZZiuLS82qheRG4X1o5gsWreGb0hR7XDc= github.com/zyedidia/generic v1.2.1/go.mod h1:ly2RBz4mnz1yeuVbQA/VFwGjK3mnHGRj1JuoG336Bis= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= +gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= +gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= From 4a4ddaaeccc56bbd1e86db404726415921695da7 Mon Sep 17 00:00:00 2001 From: Steve Loeppky Date: Wed, 28 Aug 2024 11:43:38 -0700 Subject: [PATCH 58/59] docs: updates about branches and where to target PRs (#12416) * docs: updates about branches and where to target PRs CONTRIBUTING.md: calling out the release flow doc earlier and more of why a contributor should read it. LOTUS_RELEASE_FLOW: being more precise and organized about branch strategy. RELEASE_ISSUE_TEMPLATE: added comment on what we expect to see when backporting from release branch to master. * Update LOTUS_RELEASE_FLOW.md Co-authored-by: Rod Vagg * Update LOTUS_RELEASE_FLOW.md Co-authored-by: Rod Vagg * Incorporating review feedback. Stripped colors and made clear that we're cherry picking. * Removed one more color reference --------- Co-authored-by: Rod Vagg --- CONTRIBUTING.md | 8 +++---- LOTUS_RELEASE_FLOW.md | 25 ++++++++------------ documentation/misc/RELEASE_ISSUE_TEMPLATE.md | 4 ++-- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa2df627df9..1eba2eb51fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,10 +13,10 @@ Lotus is a universally open project and welcomes contributions of all kinds: cod ## Before Contributing -1. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs). -2. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted. -3. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn. -4. Familiarize yourself with our [release flow](LOTUS_RELEASE_FLOW.md) to understand how changes are incorporated into releases. +1. Familiarize yourself with our [release flow](LOTUS_RELEASE_FLOW.md) to understand how changes are incorporated into releases. This includes our branch strategy and where to target PRs. +2. If the proposal entails a protocol change, please first submit a [Filecoin Improvement Proposal](https://github.com/filecoin-project/FIPs). +3. If the change is complex and requires prior discussion, [open an issue](github.com/filecoin-project/lotus/issues) or a [discussion](https://github.com/filecoin-project/lotus/discussions) to request feedback before you start working on a pull request. This is to avoid disappointment and sunk costs, in case the change is not actually needed or accepted. +4. Please refrain from submitting PRs to adapt existing code to subjective preferences. The changeset should contain functional or technical improvements/enhancements, bug fixes, new features, or some other clear material contribution. Simple stylistic changes are likely to be rejected in order to reduce code churn. ## Implementing Changes diff --git a/LOTUS_RELEASE_FLOW.md b/LOTUS_RELEASE_FLOW.md index 387c8be6e38..3223603a630 100644 --- a/LOTUS_RELEASE_FLOW.md +++ b/LOTUS_RELEASE_FLOW.md @@ -107,24 +107,19 @@ Unless a security issue is actively being exploited or a significant number of u ## Branch and Tag Strategy -> [!NOTE] -> - Blue text indicates node-related information. -> - Orange text indicates miner-related information. -> - System default colored text applies to both node and miner releases. - -* Releases are branched from the `master` branch, regardless of whether they include a network upgrade or not. +* Releases are usually branched from the `master` branch, regardless of whether they include a network upgrade or not. + * For certain patch releases where we can't risk including recent `master` changes (such as for security or emergency bug-fix releases): + * Node: `release/vX.Y.Z+1` will be created from `release/vX.Y.Z` + * Miner: `release/miner/vX.Y.Z+1` will be created from `release/miner/vX.Y.Z` * PRs usually target the `master` branch, even if they need to be backported to a release branch. + * The primary exception is CHANGELOG editorializing and callouts. As part of the [release process](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md), those changes happen directly in a release branch and are cherry-picked back to `master` at the end of a release. * PRs that need to be backported should be marked with a `backport` label. -* Node release branches are named `release/vX.Y.Z` -* Miner release branches are named `release/miner/vX.Y.Z` +* Node release branches are named `release/vX.Y.Z` +* Miner release branches are named `release/miner/vX.Y.Z` * By the end of the release process: - * A `release/vX.Y.Z` branch (node) will have an associated `vX.Y.Z` tag - * A `release/miner/vX.Y.Z` branch (miner) will have an associated `miner/vX.Y.Z` tag -* Both node and miner releases may have additional `vX.Y.Z-rcN` or `miner/vX.Y.Z-rcN` tags for release candidates -* The `master` branch is typically the source for creating release branches -* For emergency patch releases where we can't risk including recent `master` changes: - * Node: `release/vX.Y.Z+1` will be created from `release/vX.Y.Z` - * Miner: `release/miner/vX.Y.Z+1` will be created from `release/miner/vX.Y.Z` + * A `release/vX.Y.Z` branch (node) will have an associated `vX.Y.Z` tag + * A `release/miner/vX.Y.Z` branch (miner) will have an associated `miner/vX.Y.Z` tag +* Both node and miner releases may have additional `vX.Y.Z-rcN` or `miner/vX.Y.Z-rcN` tags for release candidates. * As of 202408, the `releases` branch is no longer used and no longer tracks the latest release. See [Why is the `releases` branch deprecated and what are alternatives?](#why-is-the-releases-branch-deprecated-and-what-are-alternatives). ## FAQ diff --git a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md index 3a4ec3259a1..b40888b57d7 100644 --- a/documentation/misc/RELEASE_ISSUE_TEMPLATE.md +++ b/documentation/misc/RELEASE_ISSUE_TEMPLATE.md @@ -129,9 +129,9 @@ ### Post-Release -- [ ] Open a pull request against `master` with a merge of the `release/vX.Y.Z` branch. - - [ ] Conflict resolution should ignore the changes to `version.go` (keep the `-dev` version from master). +- [ ] Open a pull request against `master` cherry-picking the CHANGELOG commits from the `release/vX.Y.Z` branch. - Link to PR: + - Assuming we followed [the process of merging changes into `master` first before backporting to the release branch](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#branch-and-tag-strategy), the only changes should be CHANGELOG updates. - [ ] Finish updating/merging the [RELEASE_ISSUE_TEMPLATE.md](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md) PR from `Before RC1` with any improvements determined from this latest release iteration. ## ❤️ Contributors From 8f4299ef8a7c4ac3775834b38824e55890944dec Mon Sep 17 00:00:00 2001 From: Mikers Date: Thu, 29 Aug 2024 16:32:47 -1000 Subject: [PATCH 59/59] disable snappy compression in badger v4 --- blockstore/badger/versions/options.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blockstore/badger/versions/options.go b/blockstore/badger/versions/options.go index 577535899c3..9a5182a5ea5 100644 --- a/blockstore/badger/versions/options.go +++ b/blockstore/badger/versions/options.go @@ -7,6 +7,7 @@ import ( badgerV2 "github.com/dgraph-io/badger/v2" optionsV2 "github.com/dgraph-io/badger/v2/options" badgerV4 "github.com/dgraph-io/badger/v4" + optionsV4 "github.com/dgraph-io/badger/v4/options" ) // Options embeds the badger options themselves, and augments them with @@ -46,6 +47,9 @@ func BlockStoreOptions(path string, readonly bool, badgerVersion int) Options { // in order to shorten keys, but it'll require a migration. opts.Prefix = "/blocks/" + // Disable Snappy Compression + opts.V4Options.Compression = optionsV4.None + // Blockstore values are immutable; therefore we do not expect any // conflicts to emerge. opts.V2Options.DetectConflicts = false