From 289c1d297f6e4c5bcca131453332b0cde44a4969 Mon Sep 17 00:00:00 2001 From: Lazy Nina <> Date: Tue, 13 Jun 2023 10:32:53 -0400 Subject: [PATCH 01/13] [stable] Release 3.4.4 From c24e48c83c4c8318fc19d9c9b65311696a545cd3 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Tue, 11 Jul 2023 14:58:19 -0400 Subject: [PATCH 02/13] add node version endpoint (#505) Co-authored-by: Lazy Nina <> --- routes/exchange.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/routes/exchange.go b/routes/exchange.go index 89eba7f5..66c5bf70 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -31,6 +31,8 @@ var ( IsBlacklisted = []byte{1} ) +const NodeVersion = "3.4.4" + const ( // RoutePathAPIBase ... RoutePathAPIBase = "/api/v1" @@ -46,6 +48,8 @@ const ( RoutePathAPINodeInfo = "/api/v1/node-info" // RoutePathAPIBlock ... RoutePathAPIBlock = "/api/v1/block" + // RoutePathAPINodeVersion ... + RoutePathAPINodeVersion = "/api/v1/node-version" ) // APIRoutes returns the routes for the public-facing API. @@ -100,6 +104,13 @@ func (fes *APIServer) APIRoutes() []Route { fes.APIBlock, PublicAccess, }, + { + "APINodeVersion", + []string{"GET"}, + RoutePathAPINodeVersion, + fes.APINodeVersion, + PublicAccess, + }, } return APIRoutes @@ -1373,6 +1384,21 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) { } } +type APINodeVersionResponse struct { + Version string +} + +// APINodeVersion returns the version of the node. +func (fes *APIServer) APINodeVersion(ww http.ResponseWriter, rr *http.Request) { + if err := json.NewEncoder(ww).Encode(&APINodeVersionResponse{ + Version: NodeVersion, + }); err != nil { + APIAddError(ww, fmt.Sprintf("APINodeVersion: Problem encoding response "+ + "as JSON: %v", err)) + return + } +} + // TODO: This is a somewhat redundant version of processTransaction It exists // because the API needed to cut out the derivation of the public key from the // user object, among other things. From dda345edebe2b2a831f1d2b9b54397f36e4e0a74 Mon Sep 17 00:00:00 2001 From: Lazy Nina <> Date: Thu, 27 Jul 2023 16:48:35 -0400 Subject: [PATCH 03/13] [stable] Release 3.4.5 --- routes/exchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/exchange.go b/routes/exchange.go index 66c5bf70..5d8025e8 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -31,7 +31,7 @@ var ( IsBlacklisted = []byte{1} ) -const NodeVersion = "3.4.4" +const NodeVersion = "3.4.5" const ( // RoutePathAPIBase ... From dba653dc043cf58bebb8183ba2061419e845a947 Mon Sep 17 00:00:00 2001 From: Lazy Nina <> Date: Thu, 27 Jul 2023 16:54:36 -0400 Subject: [PATCH 04/13] hotfix to exchange test --- routes/exchange_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/exchange_test.go b/routes/exchange_test.go index 663ce57c..413d8c7d 100644 --- a/routes/exchange_test.go +++ b/routes/exchange_test.go @@ -157,7 +157,7 @@ func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, i mempool := lib.NewDeSoMempool( chain, 0, /* rateLimitFeeRateNanosPerKB */ 0 /* minFeeRateNanosPerKB */, "", true, - "" /*dataDir*/, "") + "" /*dataDir*/, "", true) minerPubKeys := []string{} if isSender { minerPubKeys = append(minerPubKeys, senderPkString) From cf64bd400669c8da44df6ff07673ba723736417f Mon Sep 17 00:00:00 2001 From: superzordon <88362450+superzordon@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:15:01 -0400 Subject: [PATCH 05/13] Add captcha verification (#509) * Updates to captcha verification * Updates to backend * Updates to captcha verify * Update captcha verification * Cleanup logs * Add routes to store reward amount in global state, track usage via data dog * Update verify captcha validation ordering, add back comp profile config bool --- cmd/run.go | 1 + config/config.go | 6 ++ routes/base.go | 7 ++ routes/global_state.go | 17 +++- routes/server.go | 16 ++++ routes/transaction.go | 20 +++- routes/verify.go | 210 ++++++++++++++++++++++++++++++++++++++++- scripts/nodes/n0_test | 3 +- 8 files changed, 274 insertions(+), 6 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index aeac0335..02486a57 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -177,6 +177,7 @@ func init() { runCmd.PersistentFlags().String("metamask-airdrop-eth-minimum", "100000000000000", "In Wei, amount of Eth required to receive an airdrop during Metamask signup.") runCmd.PersistentFlags().Uint64("metamask-airdrop-deso-nanos-amount", 0, "Amount of DESO in nanos to send to metamask users as an airdrop") + runCmd.PersistentFlags().String("hcaptcha-secret", "", "Secret key for hcaptcha service. Used to verify captcha token verifications.") runCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { viper.BindPFlag(flag.Name, flag) }) diff --git a/config/config.go b/config/config.go index 55ecc326..81b9b47a 100644 --- a/config/config.go +++ b/config/config.go @@ -94,6 +94,9 @@ type Config struct { MetamaskAirdropEthMinimum *uint256.Int // Amount of DESO in nanos metamask users receive as an airdrop MetamaskAirdropDESONanosAmount uint64 + + // Secret used to validate hCaptcha tokens. + HCaptchaSecret string } func LoadConfig(coreConfig *coreCmd.Config) *Config { @@ -194,6 +197,9 @@ func LoadConfig(coreConfig *coreCmd.Config) *Config { // Node source ID config.NodeSource = viper.GetUint64("node-source") + // hCaptcha secret + config.HCaptchaSecret = viper.GetString("hcaptcha-secret") + // Public keys that need their balances monitored. Map of Label to Public key labelsToPublicKeys := viper.GetString("public-key-balances-to-monitor") if len(labelsToPublicKeys) > 0 { diff --git a/routes/base.go b/routes/base.go index 8649c30e..59d6555e 100644 --- a/routes/base.go +++ b/routes/base.go @@ -358,6 +358,7 @@ type GetAppStateResponse struct { JumioKickbackUSDCents uint64 // CountrySignUpBonus is the sign-up bonus configuration for the country inferred from a request's IP address. CountrySignUpBonus CountryLevelSignUpBonus + CaptchaDeSoNanos uint64 DefaultFeeRateNanosPerKB uint64 TransactionFeeMap map[string][]TransactionFee @@ -394,6 +395,11 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) { defaultFeeRateNanosPerKB = globalParams.MinimumNetworkFeeNanosPerKB } + captchaDesoNanos, err := fes.getCaptchaRewardNanosFromGlobalState() + if err != nil { + captchaDesoNanos = 0 + } + res := &GetAppStateResponse{ MinSatoshisBurnedForProfileCreation: fes.Config.MinSatoshisForProfile, BlockHeight: fes.backendServer.GetBlockchain().BlockTip().Height, @@ -417,6 +423,7 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) { TransactionFeeMap: fes.TxnFeeMapToResponse(true), BuyETHAddress: fes.Config.BuyDESOETHAddress, Nodes: lib.NODES, + CaptchaDeSoNanos: captchaDesoNanos, // Deprecated USDCentsPerBitCloutExchangeRate: fes.GetExchangeDeSoPrice(), diff --git a/routes/global_state.go b/routes/global_state.go index 88c88acc..c478695c 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -240,8 +240,11 @@ var ( // -> _GlobalStatePrefixUsernameToBlacklistState = []byte{47} - // NEXT_TAG: 48 + // The prefix for modifying the starter nanos reward for solving a captcha on signup. + // -> + _GlobalStatePrefixToCaptchaReward = []byte{48} + // NEXT_TAG: 49 ) type HotFeedApprovedPostOp struct { @@ -389,6 +392,12 @@ type UserMetadata struct { UnreadNotifications uint64 // The most recently scanned notification transaction index in the database. Stored in order to prevent unnecessary re-scanning. LatestUnreadNotificationIndex int64 + + // The last block height that the user has submitted hcaptcha verification for. + LastHcaptchaBlockHeight uint32 + // HcaptchaShouldCompProfileCreation = True if we should comp the create profile fee because the user went through the + // Captcha flow. + HcaptchaShouldCompProfileCreation bool } type TutorialStatus string @@ -673,6 +682,12 @@ func GlobalStateKeyForBlacklistedProfileByUsername(username string) []byte { return key } +// Key for accessing the captcha reward amount. +func GlobalStateKeyForCaptchaRewardAmountNanos() []byte { + key := append([]byte{}, _GlobalStatePrefixToCaptchaReward...) + return key +} + // Key for accessing the blacklist audit logs associated with a user. func GlobalStateKeyForBlacklistAuditLogs(username string) []byte { key := append([]byte{}, _GlobalStatePrefixBlacklistAuditLog...) diff --git a/routes/server.go b/routes/server.go index cc6208ec..83f75138 100644 --- a/routes/server.go +++ b/routes/server.go @@ -155,10 +155,12 @@ const ( RoutePathSubmitPhoneNumberVerificationCode = "/api/v0/submit-phone-number-verification-code" RoutePathResendVerifyEmail = "/api/v0/resend-verify-email" RoutePathVerifyEmail = "/api/v0/verify-email" + RoutePathVerifyCaptcha = "/api/v0/verify-captcha" RoutePathJumioBegin = "/api/v0/jumio-begin" RoutePathJumioCallback = "/api/v0/jumio-callback" RoutePathJumioFlowFinished = "/api/v0/jumio-flow-finished" RoutePathGetJumioStatusForPublicKey = "/api/v0/get-jumio-status-for-public-key" + RoutePathAdminSetCaptchaRewardNanos = "/api/v0/admin/set-captcha-reward-nanos" // tutorial.go RoutePathGetTutorialCreators = "/api/v0/get-tutorial-creators" @@ -1109,6 +1111,13 @@ func (fes *APIServer) NewRouter() *muxtrace.Router { fes.VerifyEmail, PublicAccess, }, + { + "VerifyCaptcha", + []string{"POST", "OPTIONS"}, + RoutePathVerifyCaptcha, + fes.HandleCaptchaVerificationRequest, + PublicAccess, + }, { "GetUserDerivedKeys", []string{"POST", "OPTIONS"}, @@ -1729,6 +1738,13 @@ func (fes *APIServer) NewRouter() *muxtrace.Router { fes.AdminResetPhoneNumber, SuperAdminAccess, }, + { + "SetCaptchaRewardNanos", + []string{"POST", "OPTIONS"}, + RoutePathAdminSetCaptchaRewardNanos, + fes.AdminSetCaptchaRewardNanos, + SuperAdminAccess, + }, // End all /admin routes // GET endpoints for managing parameters related to Buying DeSo { diff --git a/routes/transaction.go b/routes/transaction.go index 43ee242a..8abd53dd 100644 --- a/routes/transaction.go +++ b/routes/transaction.go @@ -25,7 +25,7 @@ import ( ) type GetTxnRequest struct { - // TxnHash to fetch. + // TxnHashHex to fetch. TxnHashHex string `safeForLogging:"true"` } @@ -465,7 +465,7 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata } // Only comp create profile fee if frontend server has both twilio and starter deso seed configured and the user // has verified their profile. - if !fes.Config.CompProfileCreation || fes.Config.StarterDESOSeed == "" || fes.Twilio == nil || (userMetadata.PhoneNumber == "" && !userMetadata.JumioVerified && existingMetamaskAirdropMetadata == nil) { + if !fes.Config.CompProfileCreation || fes.Config.StarterDESOSeed == "" || (fes.Config.HCaptchaSecret == "" && fes.Twilio == nil) || (userMetadata.PhoneNumber == "" && !userMetadata.JumioVerified && existingMetamaskAirdropMetadata == nil && userMetadata.LastHcaptchaBlockHeight == 0) { return additionalFees, nil, nil } var currentBalanceNanos uint64 @@ -506,11 +506,16 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata return additionalFees, nil, nil } updateMetamaskAirdropMetadata = true - } else { + } else if userMetadata.JumioVerified { // User has been Jumio verified but should comp profile creation is false, just return if !userMetadata.JumioShouldCompProfileCreation { return additionalFees, nil, nil } + } else if userMetadata.LastHcaptchaBlockHeight != 0 { + // User has been captcha verified but should comp profile creation is false, just return + if !userMetadata.HcaptchaShouldCompProfileCreation { + return additionalFees, nil, nil + } } // Find the minimum starter bit deso amount @@ -529,6 +534,10 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata // We comp the create profile fee minus the minimum starter deso amount divided by 2. // This discourages botting while covering users who verify a phone number. compAmount := createProfileFeeNanos - (minStarterDESONanos / 2) + if (minStarterDESONanos / 2) > createProfileFeeNanos { + compAmount = createProfileFeeNanos + } + // If the user won't have enough deso to cover the fee, this is an error. if currentBalanceNanos+compAmount < createProfileFeeNanos { return 0, nil, fmt.Errorf("Creating a profile requires DeSo. Please purchase some to create a profile.") @@ -546,6 +555,11 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata if err = fes.putPhoneNumberMetadataInGlobalState(newPhoneNumberMetadata, userMetadata.PhoneNumber); err != nil { return 0, nil, fmt.Errorf("UpdateProfile: Error setting ShouldComp to false for phone number metadata: %v", err) } + } else if userMetadata.LastHcaptchaBlockHeight != 0 { + userMetadata.HcaptchaShouldCompProfileCreation = false + if err = fes.putUserMetadataInGlobalState(userMetadata); err != nil { + return 0, nil, fmt.Errorf("UpdateProfile: Error setting ShouldComp to false for jumio user metadata: %v", err) + } } else { // Set JumioShouldCompProfileCreation to false so we don't continue to comp profile creation. userMetadata.JumioShouldCompProfileCreation = false diff --git a/routes/verify.go b/routes/verify.go index 97237d85..e668b593 100644 --- a/routes/verify.go +++ b/routes/verify.go @@ -141,8 +141,9 @@ func (fes *APIServer) canUserCreateProfile(userMetadata *UserMetadata, utxoView return false, err } // User can create a profile if they have a phone number or if they have enough DeSo to cover the create profile fee. + // User can also create a profile if they've successfully filled out a captcha. // The PhoneNumber is only set if the user has passed phone number verification. - if userMetadata.PhoneNumber != "" || totalBalanceNanos >= utxoView.GlobalParamsEntry.CreateProfileFeeNanos { + if userMetadata.PhoneNumber != "" || totalBalanceNanos >= utxoView.GlobalParamsEntry.CreateProfileFeeNanos || userMetadata.LastHcaptchaBlockHeight > 0 { return true, nil } @@ -251,6 +252,213 @@ func (fes *APIServer) validatePhoneNumberNotAlreadyInUse(phoneNumber string, use return nil } +type SubmitCaptchaVerificationRequest struct { + Token string + JWT string + PublicKeyBase58Check string +} + +type SubmitCaptchaVerificationResponse struct { + Success bool + TxnHashHex string +} + +func (fes *APIServer) HandleCaptchaVerificationRequest(ww http.ResponseWriter, req *http.Request) { + decoder := json.NewDecoder(io.LimitReader(req.Body, MaxRequestBodySizeBytes)) + requestData := SubmitCaptchaVerificationRequest{} + if err := decoder.Decode(&requestData); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Problem parsing request body: %v", err)) + return + } + + // Validate their permissions + isValid, err := fes.ValidateJWT(requestData.PublicKeyBase58Check, requestData.JWT) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Error validating JWT: %v", err)) + } + if !isValid { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Invalid token: %v", err)) + return + } + + txnHashHex, err := fes.verifyHCaptchaTokenAndSendStarterDESO(requestData.Token, requestData.PublicKeyBase58Check) + + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Error verifying captcha: %v", err)) + return + } + + res := SubmitCaptchaVerificationResponse{ + Success: true, + TxnHashHex: txnHashHex, + } + if err = json.NewEncoder(ww).Encode(res); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Problem encoding response: %v", err)) + return + } +} + +type AdminUpdateCaptchaRewardRequest struct { + // Amount of nanos to reward for a successful captcha. + RewardNanos uint64 +} + +type AdminUpdateCaptchaRewardResponse struct { + // Amount of nanos to reward for a successful captcha. + RewardNanos uint64 +} + +// HandleAdminUpdateCaptchaRewardRequest allows an admin to update the captcha reward amount. +func (fes *APIServer) AdminSetCaptchaRewardNanos(ww http.ResponseWriter, req *http.Request) { + decoder := json.NewDecoder(io.LimitReader(req.Body, MaxRequestBodySizeBytes)) + requestData := AdminUpdateCaptchaRewardRequest{} + if err := decoder.Decode(&requestData); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Problem parsing request body: %v", err)) + return + } + + // Ensure that the reward amount is not greater than the starter deso amount flag. + if requestData.RewardNanos > fes.Config.StarterDESONanos { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Reward amount %v exceeds starter deso amount %v", requestData.RewardNanos, fes.Config.StarterDESONanos)) + return + } + + if err := fes.putCaptchaRewardNanosInGlobalState(requestData.RewardNanos); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Error putting captcha reward in global state: %v", err)) + return + } + + res := AdminUpdateCaptchaRewardResponse{ + RewardNanos: requestData.RewardNanos, + } + if err := json.NewEncoder(ww).Encode(res); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Problem encoding response: %v", err)) + return + } +} + +// getCaptchaRewardNanosFromGlobalState returns the amount of nanos to reward for a successful captcha from global state. +func (fes *APIServer) getCaptchaRewardNanosFromGlobalState() (uint64, error) { + dbKey := GlobalStateKeyForCaptchaRewardAmountNanos() + + rewardNanosBytes, err := fes.GlobalState.Get(dbKey) + if err != nil { + return 0, fmt.Errorf( + "getCaptchaRewardNanosFromGlobalState: Problem with Get: %v", err) + } + + rewardNanos, err := lib.ReadUvarint(bytes.NewReader(rewardNanosBytes)) + + return rewardNanos, nil +} + +// putCaptchaRewardNanosInGlobalState puts the amount of nanos to reward for a successful captcha in global state. +func (fes *APIServer) putCaptchaRewardNanosInGlobalState(rewardNanos uint64) error { + dbKey := GlobalStateKeyForCaptchaRewardAmountNanos() + + rewardNanosBytes := lib.UintToBuf(rewardNanos) + + if err := fes.GlobalState.Put(dbKey, rewardNanosBytes); err != nil { + return fmt.Errorf( + "putCaptchaRewardNanosInGlobalState: Problem with Put: %v", err) + } + + return nil +} + +// verifyHCaptchaTokenAndSendStarterDESO verifies the captcha token and sends the starter DESO to the user. +func (fes *APIServer) verifyHCaptchaTokenAndSendStarterDESO(token string, publicKeyBase58Check string) (txnHashHex string, err error) { + if fes.Config.StarterDESOSeed == "" { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Starter DESO seed not set") + } + + // Retrieve the amount of nanos to reward for a successful captcha. + amountToSendNanos, err := fes.getCaptchaRewardNanosFromGlobalState() + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with getCaptchaRewardNanosFromGlobalState: %v", err) + } + + // Decode the public key. + publicKeyBytes, _, err := lib.Base58CheckDecode(publicKeyBase58Check) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem decoding public key: %v", err) + } + + // Ensure the user has not already received the starter DESO for submitting a successful captcha. + userMetadata, err := fes.getUserMetadataFromGlobalState(publicKeyBase58Check) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with getUserMetadataFromGlobalState: %v", err) + } + + if userMetadata.LastHcaptchaBlockHeight != 0 { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: LastHcaptchaBlockHeight is already set") + } + + // Verify the token with hCaptcha. + verificationSuccess, err := fes.verifyHCaptchaToken(token) + + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Error verifying captcha: %v", err) + } + + if !verificationSuccess { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Captcha verification failed") + } + + // Update the user's metadata to indicate that they have received the starter DESO. + lastBlockheight := fes.blockchain.BlockTip().Height + userMetadata.LastHcaptchaBlockHeight = lastBlockheight + userMetadata.HcaptchaShouldCompProfileCreation = true + + if err = fes.putUserMetadataInGlobalState(userMetadata); err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with putUserMetadataInGlobalState: %v", err) + } + + // Send the starter DESO to the user. + var txnHash *lib.BlockHash + txnHash, err = fes.SendSeedDeSo(publicKeyBytes, amountToSendNanos, false) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Error sending seed DeSo: %v", err) + } + + // Log the transaction to datadog. + if fes.backendServer.GetStatsdClient() != nil { + fes.backendServer.GetStatsdClient().Incr("SEND_STARTER_DESO_CAPTCHA", nil, 1) + } + + return txnHash.String(), nil +} + +type VerificationResponse struct { + Success bool `json:"success"` + ErrorCodes []string `json:"error-codes"` +} + +const VERIFY_URL = "https://hcaptcha.com/siteverify" + +// verifyHCaptchaToken verifies the captcha token via the hCaptcha API. +func (fes *APIServer) verifyHCaptchaToken(token string) (bool, error) { + // Construct and send the request to hcaptcha. + data := url.Values{} + data.Set("secret", fes.Config.HCaptchaSecret) + data.Set("response", token) + + resp, err := http.PostForm(VERIFY_URL, data) + if err != nil { + return false, err + } + defer resp.Body.Close() + + // Parse the response and return the result. + var verificationResponse VerificationResponse + err = json.NewDecoder(resp.Body).Decode(&verificationResponse) + if err != nil { + return false, err + } + + return verificationResponse.Success, nil +} + type SubmitPhoneNumberVerificationCodeRequest struct { JWT string PublicKeyBase58Check string diff --git a/scripts/nodes/n0_test b/scripts/nodes/n0_test index fb9f8855..b12c2682 100755 --- a/scripts/nodes/n0_test +++ b/scripts/nodes/n0_test @@ -36,11 +36,12 @@ rm /tmp/main.*.log --block-producer-seed='essence camp ghost remove document vault ladder swim pupil index apart ring' \ --starter-deso-seed='road congress client market couple bid risk escape artwork rookie artwork food' \ --data-dir=/tmp/n0_test_00000 \ - --access-control-allow-origins=http://localhost:4200,http://localhost:80,http://localhost:18002,http://localhost:4201,http://localhost:18001 \ + --access-control-allow-origins=http://localhost:4200,http://localhost:80,http://localhost:18002,http://localhost:4201,http://localhost:18001,http://localhost:3000,localhost:3000 \ --secure-header-allow-hosts=localhost:4200 \ --secure-header-development=true \ --block-cypher-api-key=092dae962ea44b02809a4c74408b42a1 \ --min-satoshis-for-profile=0 \ --expose-global-state=true \ --show-processing-spinners=true \ + --comp-profile-creation=true \ --metamask-airdrop-deso-nanos-amount=1000 ) From b6dc91ac72093c9c120806d5640181bd3186c397 Mon Sep 17 00:00:00 2001 From: superzordon <88362450+superzordon@users.noreply.github.com> Date: Fri, 8 Sep 2023 18:50:20 -0400 Subject: [PATCH 06/13] Update badger sync settings to optimize memory usage during hypersync (#506) * Update hypersync to use default badger settings, switch to performance settings once hypersync completes * Update test dockerfile to accept core branch name as parameter * Blank commit to trigger build --- routes/hot_feed.go | 24 ++++++++++++++++------- routes/server.go | 25 +++++++++++++++++++----- routes/supply.go | 5 +++++ scripts/nodes/n0 | 1 + scripts/tools/clear_ancestral_records.go | 2 +- test.Dockerfile | 6 ++++++ 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/routes/hot_feed.go b/routes/hot_feed.go index 0f058b2e..14c2e4d6 100644 --- a/routes/hot_feed.go +++ b/routes/hot_feed.go @@ -83,13 +83,23 @@ func (fes *APIServer) StartHotFeedRoutine() { for { select { case <-time.After(30 * time.Second): - resetCache := false - if cacheResetCounter >= ResetCachesIterationLimit { - resetCache = true - cacheResetCounter = 0 - } - fes.UpdateHotFeed(resetCache) - cacheResetCounter += 1 + // Use an inner function to unlock the mutex with a defer statement. + func() { + // If we're syncing a snapshot, we need to lock the DB mutex before updating the hot feed. + // This is because at the end of a snapshot sync, we re-start the DB, which will cause + // the hot feed routine to panic if it's in the middle of updating the hot feed. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } + resetCache := false + if cacheResetCounter >= ResetCachesIterationLimit { + resetCache = true + cacheResetCounter = 0 + } + fes.UpdateHotFeed(resetCache) + cacheResetCounter += 1 + }() case <-fes.quit: break out } diff --git a/routes/server.go b/routes/server.go index 83f75138..189f8093 100644 --- a/routes/server.go +++ b/routes/server.go @@ -2599,11 +2599,20 @@ func (fes *APIServer) StartSeedBalancesMonitoring() { return } tags := []string{} - fes.logBalanceForSeed(fes.Config.StarterDESOSeed, "STARTER_DESO", tags) - fes.logBalanceForSeed(fes.Config.BuyDESOSeed, "BUY_DESO", tags) - for label, publicKey := range fes.Config.PublicKeyBalancesToMonitor { - fes.logBalanceForPublicKey(publicKey, label, tags) - } + // Use an inner function to unlock the mutex with a defer statement. + func() { + // If we're syncing a snapshot, we need to lock the DB in case the DB is restarted. This happens + // at the end of the snapshot sync. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } + fes.logBalanceForSeed(fes.Config.StarterDESOSeed, "STARTER_DESO", tags) + fes.logBalanceForSeed(fes.Config.BuyDESOSeed, "BUY_DESO", tags) + for label, publicKey := range fes.Config.PublicKeyBalancesToMonitor { + fes.logBalanceForPublicKey(publicKey, label, tags) + } + }() case <-fes.quit: break out } @@ -2684,6 +2693,12 @@ func (fes *APIServer) SetGlobalStateCache() { if fes.backendServer == nil { return } + // If we're syncing a snapshot, we need to lock the DB in case the DB is restarted. This happens at + // the end of the snapshot sync. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView() if err != nil { glog.Errorf("SetGlobalStateCache: problem with GetAugmentedUniversalView: %v", err) diff --git a/routes/supply.go b/routes/supply.go index 83743504..f6114dd4 100644 --- a/routes/supply.go +++ b/routes/supply.go @@ -44,6 +44,11 @@ func (fes *APIServer) StartSupplyMonitoring() { } func (fes *APIServer) UpdateSupplyStats() { + // Prevent access to the DB while it's reset. This only happens when we're syncing a snapshot. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } totalSupply := uint64(0) totalKeysWithDESO := uint64(0) // Get all the balances from the DB diff --git a/scripts/nodes/n0 b/scripts/nodes/n0 index 84289258..a80aa764 100755 --- a/scripts/nodes/n0 +++ b/scripts/nodes/n0 @@ -18,6 +18,7 @@ rm /tmp/main.*.log --block-cypher-api-key=092dae962ea44b02809a4c74408b42a1 \ --min-satoshis-for-profile=0 \ --connect-ips=deso-seed-2.io:17000 \ + --hypersync-max-queue-size=10 \ --run-hot-feed-routine=true ) diff --git a/scripts/tools/clear_ancestral_records.go b/scripts/tools/clear_ancestral_records.go index cf7a9edb..7f862488 100644 --- a/scripts/tools/clear_ancestral_records.go +++ b/scripts/tools/clear_ancestral_records.go @@ -16,7 +16,7 @@ func main() { fmt.Printf("Error reading db1 err: %v", err) return } - snap, err, _ := lib.NewSnapshot(db, dbDir, lib.SnapshotBlockHeightPeriod, false, false, &lib.DeSoMainnetParams, false) + snap, err, _ := lib.NewSnapshot(db, dbDir, lib.SnapshotBlockHeightPeriod, false, false, &lib.DeSoMainnetParams, false, lib.HypersyncDefaultMaxQueueSize) if err != nil { fmt.Printf("Error reading snap err: %v", err) return diff --git a/test.Dockerfile b/test.Dockerfile index d078d61c..b8c7d0c9 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -4,6 +4,9 @@ RUN apk update RUN apk upgrade RUN apk add --update go gcc g++ vips-dev git +# Declare an ARG for the branch name with a default value of "main" +ARG BRANCH_NAME=main + WORKDIR /deso/src RUN git clone https://github.com/deso-protocol/core.git @@ -11,6 +14,9 @@ RUN git clone https://github.com/deso-protocol/core.git WORKDIR /deso/src/core RUN git pull +# Try to checkout to the specified branch. If it fails, checkout main. +RUN git checkout ${BRANCH_NAME} || (echo "Branch ${BRANCH_NAME} not found. Falling back to main." && git checkout main) + WORKDIR /deso/src/backend COPY go.mod . From 82ff3b4e98e920ebc7b341f3bbcf24fe6bbb22b3 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:35:48 -0400 Subject: [PATCH 07/13] ln/fix-transaction-info-mempool (#510) Co-authored-by: Lazy Nina <> --- routes/exchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/exchange.go b/routes/exchange.go index 5d8025e8..6fc4f6ed 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -1187,7 +1187,7 @@ func (fes *APIServer) APITransactionInfo(ww http.ResponseWriter, rr *http.Reques } // Skip irrelevant transactions - if !isRelevantTxn { + if !isRelevantTxn && txnMeta.TransactorPublicKeyBase58Check != transactionInfoRequest.PublicKeyBase58Check { continue } From 734c2b61543c77da94aa0a5787bf99bd26ae05fe Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Mon, 2 Oct 2023 17:47:14 -0400 Subject: [PATCH 08/13] ln/no-comp-when-0-create-profile-fee (#511) Co-authored-by: Lazy Nina <> --- routes/transaction.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/transaction.go b/routes/transaction.go index 8abd53dd..b77c9000 100644 --- a/routes/transaction.go +++ b/routes/transaction.go @@ -458,7 +458,9 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata } // Additional fee is set to the create profile fee when we are creating a profile additionalFees := utxoView.GlobalParamsEntry.CreateProfileFeeNanos - + if additionalFees == 0 { + return 0, nil, nil + } existingMetamaskAirdropMetadata, err := fes.GetMetamaskAirdropMetadata(profilePublicKey) if err != nil { return 0, nil, fmt.Errorf("Error geting metamask airdrop metadata from global state: %v", err) From c407d81d68a5fa5b1d1aebde539f2151c23e5a2c Mon Sep 17 00:00:00 2001 From: superzordon <88362450+superzordon@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:48:21 -0500 Subject: [PATCH 09/13] Empty commit to trigger build (#515) From 253ad8c50be65ce9d3fd013fdc2722b30794df57 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:25:16 -0500 Subject: [PATCH 10/13] Add extra data to basic transfer and diamond txn construction endpoints (#516) --- routes/exchange.go | 2 +- routes/transaction.go | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/routes/exchange.go b/routes/exchange.go index 6fc4f6ed..e6ea883a 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -774,7 +774,7 @@ func (fes *APIServer) APITransferDeSo(ww http.ResponseWriter, rr *http.Request) if transferDeSoRequest.AmountNanos < 0 { // Create a MAX transaction txnn, totalInputt, spendAmountt, feeNanoss, err = fes.blockchain.CreateMaxSpend( - senderPublicKeyBytes, recipientPub.SerializeCompressed(), + senderPublicKeyBytes, recipientPub.SerializeCompressed(), nil, uint64(minFeeRateNanosPerKB), fes.backendServer.GetMempool(), additionalOutputs) if err != nil { diff --git a/routes/transaction.go b/routes/transaction.go index b77c9000..66571e32 100644 --- a/routes/transaction.go +++ b/routes/transaction.go @@ -993,10 +993,11 @@ func (fes *APIServer) ExceedsDeSoBalance(nanosPurchased uint64, seed string) (bo // SendDeSoRequest ... type SendDeSoRequest struct { - SenderPublicKeyBase58Check string `safeForLogging:"true"` - RecipientPublicKeyOrUsername string `safeForLogging:"true"` - AmountNanos int64 `safeForLogging:"true"` - MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + SenderPublicKeyBase58Check string `safeForLogging:"true"` + RecipientPublicKeyOrUsername string `safeForLogging:"true"` + AmountNanos int64 `safeForLogging:"true"` + MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + ExtraData map[string]string `safeForLogging:"true"` // No need to specify ProfileEntryResponse in each TransactionFee TransactionFees []TransactionFee `safeForLogging:"true"` @@ -1074,6 +1075,12 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { return } + extraData, err := EncodeExtraDataMap(requestData.ExtraData) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("SendDeSo: Problem encoding ExtraData: %v", err)) + return + } + // If the AmountNanos is less than zero then we have a special case where we create // a transaction with the maximum spend. var txnn *lib.MsgDeSoTxn @@ -1084,7 +1091,7 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { if requestData.AmountNanos < 0 { // Create a MAX transaction txnn, totalInputt, spendAmountt, feeNanoss, err = fes.blockchain.CreateMaxSpend( - senderPkBytes, recipientPkBytes, requestData.MinFeeRateNanosPerKB, + senderPkBytes, recipientPkBytes, extraData, requestData.MinFeeRateNanosPerKB, fes.backendServer.GetMempool(), additionalOutputs) if err != nil { _AddBadRequestError(ww, fmt.Sprintf("SendDeSo: Error processing MAX transaction: %v", err)) @@ -1115,6 +1122,10 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { // inputs and change. } + if len(extraData) > 0 { + txnn.ExtraData = extraData + } + // Add inputs to the transaction and do signing, validation, and broadcast // depending on what the user requested. totalInputt, spendAmountt, changeAmountt, feeNanoss, err = @@ -2116,6 +2127,8 @@ type SendDiamondsRequest struct { MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + ExtraData map[string]string `safeForLogging:"true"` + // No need to specify ProfileEntryResponse in each TransactionFee TransactionFees []TransactionFee `safeForLogging:"true"` @@ -2191,6 +2204,12 @@ func (fes *APIServer) SendDiamonds(ww http.ResponseWriter, req *http.Request) { return } + extraData, err := EncodeExtraDataMap(requestData.ExtraData) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("SendDiamonds: Problem encoding extra data: %v", err)) + return + } + // Try and create the transfer with diamonds for the user. // We give diamonds in DESO if we're past the corresponding block height. blockHeight := fes.blockchain.BlockTip().Height + 1 @@ -2210,6 +2229,7 @@ func (fes *APIServer) SendDiamonds(ww http.ResponseWriter, req *http.Request) { senderPublicKeyBytes, diamondPostHash, requestData.DiamondLevel, + extraData, // Standard transaction fields requestData.MinFeeRateNanosPerKB, fes.backendServer.GetMempool(), additionalOutputs) if err != nil { From e5ef729785587b59dc00a15a7f35f668945a864e Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Thu, 21 Dec 2023 20:18:37 -0500 Subject: [PATCH 11/13] trigger build (#517) Co-authored-by: Lazy Nina <> From 4c52e54ec2ce40c506941810e64015ca92d185b7 Mon Sep 17 00:00:00 2001 From: Lazy Nina <> Date: Wed, 27 Dec 2023 19:06:01 -0500 Subject: [PATCH 12/13] trigger build From 5abd151d5bd27a0f1fd49e472fb6956403648140 Mon Sep 17 00:00:00 2001 From: Lazy Nina <81658138+lazynina@users.noreply.github.com> Date: Thu, 28 Dec 2023 13:26:47 -0500 Subject: [PATCH 13/13] Add RWLock around AllCountryLevelSignUpBonuses (#518) Co-authored-by: Lazy Nina <> --- routes/admin_jumio.go | 6 ++++++ routes/server.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/routes/admin_jumio.go b/routes/admin_jumio.go index adfe40dc..cab84dbf 100644 --- a/routes/admin_jumio.go +++ b/routes/admin_jumio.go @@ -341,6 +341,8 @@ type GetAllCountryLevelSignUpBonusResponse struct { } func (fes *APIServer) AdminGetAllCountryLevelSignUpBonuses(ww http.ResponseWriter, req *http.Request) { + fes.allCountryLevelSignUpBonusesLock.RLock() + defer fes.allCountryLevelSignUpBonusesLock.RUnlock() res := GetAllCountryLevelSignUpBonusResponse{ SignUpBonusMetadata: fes.AllCountryLevelSignUpBonuses, DefaultSignUpBonusMetadata: fes.GetDefaultJumioCountrySignUpBonus(), @@ -373,6 +375,8 @@ func (fes *APIServer) SetAllCountrySignUpBonusMetadata() { // SetSingleCountrySignUpBonus sets the sign up bonus configuration for a given country in the cached map. func (fes *APIServer) SetSingleCountrySignUpBonus(countryDetails countries.Alpha3CountryCodeDetails, signUpBonus CountryLevelSignUpBonus) { + fes.allCountryLevelSignUpBonusesLock.Lock() + defer fes.allCountryLevelSignUpBonusesLock.Unlock() fes.AllCountryLevelSignUpBonuses[countryDetails.Name] = CountrySignUpBonusResponse{ CountryLevelSignUpBonus: signUpBonus, CountryCodeDetails: countryDetails, @@ -385,6 +389,8 @@ func (fes *APIServer) GetSingleCountrySignUpBonus(countryCode string) CountryLev countryCodeDetails := countries.Alpha3CountryCodes[strings.ToUpper(countryCode)] // If we can't find the signup bonus from the map, return the default. Else, return the sign up bonus we found in // the map. + fes.allCountryLevelSignUpBonusesLock.RLock() + defer fes.allCountryLevelSignUpBonusesLock.RUnlock() if countrySignUpBonusResponse, exists := fes.AllCountryLevelSignUpBonuses[countryCodeDetails.Name]; !exists { return fes.GetDefaultJumioCountrySignUpBonus() } else { diff --git a/routes/server.go b/routes/server.go index 189f8093..8c16761a 100644 --- a/routes/server.go +++ b/routes/server.go @@ -451,7 +451,8 @@ type APIServer struct { CountKeysWithDESO uint64 // map of country name to sign up bonus data - AllCountryLevelSignUpBonuses map[string]CountrySignUpBonusResponse + allCountryLevelSignUpBonusesLock sync.RWMutex + AllCountryLevelSignUpBonuses map[string]CountrySignUpBonusResponse // Frequently accessed data from global state USDCentsToDESOReserveExchangeRate uint64