merge main into feature pos (#519)

* [stable] Release 3.4.4

* add node version endpoint (#505)

Co-authored-by: Lazy Nina <>

* [stable] Release 3.4.5

* hotfix to exchange test

* Add captcha verification (#509)

* Updates to captcha verification

* Updates to backend

* Updates to captcha verify

* Update captcha verification

* Cleanup logs

* Add routes to store reward amount in global state, track usage via data dog

* Update verify captcha validation ordering, add back comp profile config bool

* Update badger sync settings to optimize memory usage during hypersync (#506)

* Update hypersync to use default badger settings, switch to performance settings once hypersync completes

* Update test dockerfile to accept core branch name as parameter

* Blank commit to trigger build

* ln/fix-transaction-info-mempool (#510)

Co-authored-by: Lazy Nina <>

* ln/no-comp-when-0-create-profile-fee (#511)

Co-authored-by: Lazy Nina <>

* Empty commit to trigger build (#515)

* Add extra data to basic transfer and diamond txn construction endpoints (#516)

* trigger build (#517)

Co-authored-by: Lazy Nina <>

* trigger build

* Add RWLock around AllCountryLevelSignUpBonuses (#518)

Co-authored-by: Lazy Nina <>

---------

Co-authored-by: Lazy Nina <>
Co-authored-by: superzordon <[email protected]>

cmd/run.go

@@ -177,6 +177,7 @@ func init() {
 	runCmd.PersistentFlags().String("metamask-airdrop-eth-minimum", "100000000000000",
 		"In Wei, amount of Eth required to receive an airdrop during Metamask signup.")
 	runCmd.PersistentFlags().Uint64("metamask-airdrop-deso-nanos-amount", 0, "Amount of DESO in nanos to send to metamask users as an airdrop")
+	runCmd.PersistentFlags().String("hcaptcha-secret", "", "Secret key for hcaptcha service. Used to verify captcha token verifications.")
 	runCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) {
 		viper.BindPFlag(flag.Name, flag)
 	})

config/config.go

@@ -94,6 +94,9 @@ type Config struct {
 	MetamaskAirdropEthMinimum *uint256.Int
 	// Amount of DESO in nanos metamask users receive as an airdrop
 	MetamaskAirdropDESONanosAmount uint64
+
+	// Secret used to validate hCaptcha tokens.
+	HCaptchaSecret string
 }
 
 func LoadConfig(coreConfig *coreCmd.Config) *Config {
@@ -194,6 +197,9 @@ func LoadConfig(coreConfig *coreCmd.Config) *Config {
 	// Node source ID
 	config.NodeSource = viper.GetUint64("node-source")
 
+	// hCaptcha secret
+	config.HCaptchaSecret = viper.GetString("hcaptcha-secret")
+
 	// Public keys that need their balances monitored. Map of Label to Public key
 	labelsToPublicKeys := viper.GetString("public-key-balances-to-monitor")
 	if len(labelsToPublicKeys) > 0 {

routes/admin_jumio.go

@@ -341,6 +341,8 @@ type GetAllCountryLevelSignUpBonusResponse struct {
 }
 
 func (fes *APIServer) AdminGetAllCountryLevelSignUpBonuses(ww http.ResponseWriter, req *http.Request) {
+	fes.allCountryLevelSignUpBonusesLock.RLock()
+	defer fes.allCountryLevelSignUpBonusesLock.RUnlock()
 	res := GetAllCountryLevelSignUpBonusResponse{
 		SignUpBonusMetadata:        fes.AllCountryLevelSignUpBonuses,
 		DefaultSignUpBonusMetadata: fes.GetDefaultJumioCountrySignUpBonus(),
@@ -373,6 +375,8 @@ func (fes *APIServer) SetAllCountrySignUpBonusMetadata() {
 // SetSingleCountrySignUpBonus sets the sign up bonus configuration for a given country in the cached map.
 func (fes *APIServer) SetSingleCountrySignUpBonus(countryDetails countries.Alpha3CountryCodeDetails,
 	signUpBonus CountryLevelSignUpBonus) {
+	fes.allCountryLevelSignUpBonusesLock.Lock()
+	defer fes.allCountryLevelSignUpBonusesLock.Unlock()
 	fes.AllCountryLevelSignUpBonuses[countryDetails.Name] = CountrySignUpBonusResponse{
 		CountryLevelSignUpBonus: signUpBonus,
 		CountryCodeDetails:      countryDetails,
@@ -385,6 +389,8 @@ func (fes *APIServer) GetSingleCountrySignUpBonus(countryCode string) CountryLev
 	countryCodeDetails := countries.Alpha3CountryCodes[strings.ToUpper(countryCode)]
 	// If we can't find the signup bonus from the map, return the default. Else, return the sign up bonus we found in
 	// the map.
+	fes.allCountryLevelSignUpBonusesLock.RLock()
+	defer fes.allCountryLevelSignUpBonusesLock.RUnlock()
 	if countrySignUpBonusResponse, exists := fes.AllCountryLevelSignUpBonuses[countryCodeDetails.Name]; !exists {
 		return fes.GetDefaultJumioCountrySignUpBonus()
 	} else {

routes/base.go

@@ -358,6 +358,7 @@ type GetAppStateResponse struct {
 	JumioKickbackUSDCents           uint64
 	// CountrySignUpBonus is the sign-up bonus configuration for the country inferred from a request's IP address.
 	CountrySignUpBonus CountryLevelSignUpBonus
+	CaptchaDeSoNanos   uint64
 
 	DefaultFeeRateNanosPerKB uint64
 	TransactionFeeMap        map[string][]TransactionFee
@@ -394,6 +395,11 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) {
 		defaultFeeRateNanosPerKB = globalParams.MinimumNetworkFeeNanosPerKB
 	}
 
+	captchaDesoNanos, err := fes.getCaptchaRewardNanosFromGlobalState()
+	if err != nil {
+		captchaDesoNanos = 0
+	}
+
 	res := &GetAppStateResponse{
 		MinSatoshisBurnedForProfileCreation: fes.Config.MinSatoshisForProfile,
 		BlockHeight:                         fes.backendServer.GetBlockchain().BlockTip().Height,
@@ -417,6 +423,7 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) {
 		TransactionFeeMap:                   fes.TxnFeeMapToResponse(true),
 		BuyETHAddress:                       fes.Config.BuyDESOETHAddress,
 		Nodes:                               lib.NODES,
+		CaptchaDeSoNanos:                    captchaDesoNanos,
 
 		// Deprecated
 		USDCentsPerBitCloutExchangeRate: fes.GetExchangeDeSoPrice(),

routes/exchange.go

@@ -31,6 +31,8 @@ var (
 	IsBlacklisted = []byte{1}
 )
 
+const NodeVersion = "3.4.5"
+
 const (
 	// RoutePathAPIBase ...
 	RoutePathAPIBase = "/api/v1"
@@ -46,6 +48,8 @@ const (
 	RoutePathAPINodeInfo = "/api/v1/node-info"
 	// RoutePathAPIBlock ...
 	RoutePathAPIBlock = "/api/v1/block"
+	// RoutePathAPINodeVersion ...
+	RoutePathAPINodeVersion = "/api/v1/node-version"
 )
 
 // APIRoutes returns the routes for the public-facing API.
@@ -100,6 +104,13 @@ func (fes *APIServer) APIRoutes() []Route {
 			fes.APIBlock,
 			PublicAccess,
 		},
+		{
+			"APINodeVersion",
+			[]string{"GET"},
+			RoutePathAPINodeVersion,
+			fes.APINodeVersion,
+			PublicAccess,
+		},
 	}
 
 	return APIRoutes
@@ -764,7 +775,7 @@ func (fes *APIServer) APITransferDeSo(ww http.ResponseWriter, rr *http.Request)
 	if transferDeSoRequest.AmountNanos < 0 {
 		// Create a MAX transaction
 		txnn, totalInputt, spendAmountt, feeNanoss, err = fes.blockchain.CreateMaxSpend(
-			senderPublicKeyBytes, recipientPub.SerializeCompressed(),
+			senderPublicKeyBytes, recipientPub.SerializeCompressed(), nil,
 			uint64(minFeeRateNanosPerKB),
 			fes.backendServer.GetMempool(), additionalOutputs, fes.backendServer.GetFeeEstimator())
 		if err != nil {
@@ -1169,7 +1180,7 @@ func (fes *APIServer) APITransactionInfo(ww http.ResponseWriter, rr *http.Reques
 			}
 
 			// Skip irrelevant transactions
-			if !isRelevantTxn {
+			if !isRelevantTxn && txnMeta.TransactorPublicKeyBase58Check != transactionInfoRequest.PublicKeyBase58Check {
 				continue
 			}
 
@@ -1369,6 +1380,21 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) {
 	}
 }
 
+type APINodeVersionResponse struct {
+	Version string
+}
+
+// APINodeVersion returns the version of the node.
+func (fes *APIServer) APINodeVersion(ww http.ResponseWriter, rr *http.Request) {
+	if err := json.NewEncoder(ww).Encode(&APINodeVersionResponse{
+		Version: NodeVersion,
+	}); err != nil {
+		APIAddError(ww, fmt.Sprintf("APINodeVersion: Problem encoding response "+
+			"as JSON: %v", err))
+		return
+	}
+}
+
 // TODO: This is a somewhat redundant version of processTransaction It exists
 // because the API needed to cut out the derivation of the public key from the
 // user object, among other things.

routes/exchange_test.go

@@ -73,7 +73,106 @@ func GetTestBadgerDb(t *testing.T) (_db *badger.DB, _dir string) {
 	return db, dir
 }
 
-func newTestAPIServer(t *testing.T, globalStateRemoteNode string, txindex bool) (*APIServer, *APIServer, *lib.DeSoMiner) {
+func NewLowDifficultyBlockchain(t *testing.T) (*lib.Blockchain, *lib.DeSoParams, *badger.DB, string) {
+
+	// Set the number of txns per view regeneration to one while creating the txns
+	lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1
+
+	return NewLowDifficultyBlockchainWithParams(t, &lib.DeSoTestnetParams)
+}
+
+func NewLowDifficultyBlockchainWithParams(t *testing.T, params *lib.DeSoParams) (
+	*lib.Blockchain, *lib.DeSoParams, *badger.DB, string) {
+
+	// Set the number of txns per view regeneration to one while creating the txns
+	lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1
+
+	db, dir := GetTestBadgerDb(t)
+	timesource := chainlib.NewMedianTime()
+
+	// Set some special parameters for testing. If the blocks above are changed
+	// these values should be updated to reflect the latest testnet values.
+	paramsCopy := *params
+	paramsCopy.GenesisBlock = &lib.MsgDeSoBlock{
+		Header: &lib.MsgDeSoHeader{
+			Version:               0,
+			PrevBlockHash:         lib.MustDecodeHexBlockHash("0000000000000000000000000000000000000000000000000000000000000000"),
+			TransactionMerkleRoot: lib.MustDecodeHexBlockHash("097158f0d27e6d10565c4dc696c784652c3380e0ff8382d3599a4d18b782e965"),
+			TstampSecs:            uint64(1560735050),
+			Height:                uint64(0),
+			Nonce:                 uint64(0),
+			// No ExtraNonce is set in the genesis block
+		},
+		Txns: []*lib.MsgDeSoTxn{
+			{
+				TxInputs:  []*lib.DeSoInput{},
+				TxOutputs: []*lib.DeSoOutput{},
+				TxnMeta: &lib.BlockRewardMetadataa{
+					ExtraData: []byte("They came here, to the new world. World 2.0, version 1776."),
+				},
+				// A signature is not required for BLOCK_REWARD transactions since they
+				// don't spend anything.
+			},
+		},
+	}
+	paramsCopy.MinDifficultyTargetHex = "999999948931e5874cf66a74c0fda790dd8c7458243d400324511a4c71f54faa"
+	paramsCopy.MinChainWorkHex = "0000000000000000000000000000000000000000000000000000000000000000"
+	paramsCopy.MiningIterationsPerCycle = 500
+	// Set maturity to 2 blocks so we can test spending on short chains. The
+	// tests rely on the maturity equaling exactly two blocks (i.e. being
+	// two times the time between blocks).
+	paramsCopy.TimeBetweenBlocks = 2 * time.Second
+	paramsCopy.BlockRewardMaturity = time.Second * 4
+	paramsCopy.TimeBetweenDifficultyRetargets = 100 * time.Second
+	paramsCopy.MaxDifficultyRetargetFactor = 2
+	paramsCopy.SeedBalances = []*lib.DeSoOutput{
+		{
+			PublicKey:   lib.MustBase58CheckDecode(moneyPkString),
+			AmountNanos: uint64(2000000 * lib.NanosPerUnit),
+		},
+	}
+
+	// Temporarily modify the seed balances to make a specific public
+	// key have some DeSo
+	chain, err := lib.NewBlockchain([]string{blockSignerPk}, 0, 0,
+		&paramsCopy, timesource, db, nil, nil, nil, false)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return chain, &paramsCopy, db, dir
+}
+
+func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, isSender bool) (*lib.DeSoMempool, *lib.DeSoMiner) {
+	assert := assert.New(t)
+	require := require.New(t)
+	_ = assert
+	_ = require
+
+	mempool := lib.NewDeSoMempool(
+		chain, 0, /* rateLimitFeeRateNanosPerKB */
+		0 /* minFeeRateNanosPerKB */, "", true,
+		"" /*dataDir*/, "", true)
+	minerPubKeys := []string{}
+	if isSender {
+		minerPubKeys = append(minerPubKeys, senderPkString)
+	} else {
+		minerPubKeys = append(minerPubKeys, recipientPkString)
+	}
+
+	blockProducer, err := lib.NewDeSoBlockProducer(
+		0, 1,
+		blockSignerSeed,
+		mempool, chain,
+		params, nil)
+	require.NoError(err)
+
+	newMiner, err := lib.NewDeSoMiner(minerPubKeys, 1 /*numThreads*/, blockProducer, params)
+	require.NoError(err)
+	return mempool, newMiner
+}
+
+func newTestAPIServer(t *testing.T, globalStateRemoteNode string) (*APIServer, *APIServer, *lib.DeSoMiner) {
 	assert := assert.New(t)
 	require := require.New(t)
 	_, _ = assert, require

routes/global_state.go

@@ -240,8 +240,11 @@ var (
 	// <prefix, username> -> <IsBlacklisted>
 	_GlobalStatePrefixUsernameToBlacklistState = []byte{47}
 
-	// NEXT_TAG: 48
+	// The prefix for modifying the starter nanos reward for solving a captcha on signup.
+	// <prefix> -> <uint64>
+	_GlobalStatePrefixToCaptchaReward = []byte{48}
 
+	// NEXT_TAG: 49
 )
 
 type HotFeedApprovedPostOp struct {
@@ -389,6 +392,12 @@ type UserMetadata struct {
 	UnreadNotifications uint64
 	// The most recently scanned notification transaction index in the database. Stored in order to prevent unnecessary re-scanning.
 	LatestUnreadNotificationIndex int64
+
+	// The last block height that the user has submitted hcaptcha verification for.
+	LastHcaptchaBlockHeight uint32
+	// HcaptchaShouldCompProfileCreation = True if we should comp the create profile fee because the user went through the
+	// Captcha flow.
+	HcaptchaShouldCompProfileCreation bool
 }
 
 type TutorialStatus string
@@ -673,6 +682,12 @@ func GlobalStateKeyForBlacklistedProfileByUsername(username string) []byte {
 	return key
 }
 
+// Key for accessing the captcha reward amount.
+func GlobalStateKeyForCaptchaRewardAmountNanos() []byte {
+	key := append([]byte{}, _GlobalStatePrefixToCaptchaReward...)
+	return key
+}
+
 // Key for accessing the blacklist audit logs associated with a user.
 func GlobalStateKeyForBlacklistAuditLogs(username string) []byte {
 	key := append([]byte{}, _GlobalStatePrefixBlacklistAuditLog...)

routes/hot_feed.go

@@ -83,13 +83,23 @@ func (fes *APIServer) StartHotFeedRoutine() {
 		for {
 			select {
 			case <-time.After(30 * time.Second):
-				resetCache := false
-				if cacheResetCounter >= ResetCachesIterationLimit {
-					resetCache = true
-					cacheResetCounter = 0
-				}
-				fes.UpdateHotFeed(resetCache)
-				cacheResetCounter += 1
+				// Use an inner function to unlock the mutex with a defer statement.
+				func() {
+					// If we're syncing a snapshot, we need to lock the DB mutex before updating the hot feed.
+					// This is because at the end of a snapshot sync, we re-start the DB, which will cause
+					// the hot feed routine to panic if it's in the middle of updating the hot feed.
+					if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot {
+						fes.backendServer.DbMutex.Lock()
+						defer fes.backendServer.DbMutex.Unlock()
+					}
+					resetCache := false
+					if cacheResetCounter >= ResetCachesIterationLimit {
+						resetCache = true
+						cacheResetCounter = 0
+					}
+					fes.UpdateHotFeed(resetCache)
+					cacheResetCounter += 1
+				}()
 			case <-fes.quit:
 				break out
 			}