diff --git a/cmd/run.go b/cmd/run.go index aeac0335..02486a57 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -177,6 +177,7 @@ func init() { runCmd.PersistentFlags().String("metamask-airdrop-eth-minimum", "100000000000000", "In Wei, amount of Eth required to receive an airdrop during Metamask signup.") runCmd.PersistentFlags().Uint64("metamask-airdrop-deso-nanos-amount", 0, "Amount of DESO in nanos to send to metamask users as an airdrop") + runCmd.PersistentFlags().String("hcaptcha-secret", "", "Secret key for hcaptcha service. Used to verify captcha token verifications.") runCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { viper.BindPFlag(flag.Name, flag) }) diff --git a/config/config.go b/config/config.go index 55ecc326..81b9b47a 100644 --- a/config/config.go +++ b/config/config.go @@ -94,6 +94,9 @@ type Config struct { MetamaskAirdropEthMinimum *uint256.Int // Amount of DESO in nanos metamask users receive as an airdrop MetamaskAirdropDESONanosAmount uint64 + + // Secret used to validate hCaptcha tokens. + HCaptchaSecret string } func LoadConfig(coreConfig *coreCmd.Config) *Config { @@ -194,6 +197,9 @@ func LoadConfig(coreConfig *coreCmd.Config) *Config { // Node source ID config.NodeSource = viper.GetUint64("node-source") + // hCaptcha secret + config.HCaptchaSecret = viper.GetString("hcaptcha-secret") + // Public keys that need their balances monitored. Map of Label to Public key labelsToPublicKeys := viper.GetString("public-key-balances-to-monitor") if len(labelsToPublicKeys) > 0 { diff --git a/routes/admin_jumio.go b/routes/admin_jumio.go index adfe40dc..cab84dbf 100644 --- a/routes/admin_jumio.go +++ b/routes/admin_jumio.go @@ -341,6 +341,8 @@ type GetAllCountryLevelSignUpBonusResponse struct { } func (fes *APIServer) AdminGetAllCountryLevelSignUpBonuses(ww http.ResponseWriter, req *http.Request) { + fes.allCountryLevelSignUpBonusesLock.RLock() + defer fes.allCountryLevelSignUpBonusesLock.RUnlock() res := GetAllCountryLevelSignUpBonusResponse{ SignUpBonusMetadata: fes.AllCountryLevelSignUpBonuses, DefaultSignUpBonusMetadata: fes.GetDefaultJumioCountrySignUpBonus(), @@ -373,6 +375,8 @@ func (fes *APIServer) SetAllCountrySignUpBonusMetadata() { // SetSingleCountrySignUpBonus sets the sign up bonus configuration for a given country in the cached map. func (fes *APIServer) SetSingleCountrySignUpBonus(countryDetails countries.Alpha3CountryCodeDetails, signUpBonus CountryLevelSignUpBonus) { + fes.allCountryLevelSignUpBonusesLock.Lock() + defer fes.allCountryLevelSignUpBonusesLock.Unlock() fes.AllCountryLevelSignUpBonuses[countryDetails.Name] = CountrySignUpBonusResponse{ CountryLevelSignUpBonus: signUpBonus, CountryCodeDetails: countryDetails, @@ -385,6 +389,8 @@ func (fes *APIServer) GetSingleCountrySignUpBonus(countryCode string) CountryLev countryCodeDetails := countries.Alpha3CountryCodes[strings.ToUpper(countryCode)] // If we can't find the signup bonus from the map, return the default. Else, return the sign up bonus we found in // the map. + fes.allCountryLevelSignUpBonusesLock.RLock() + defer fes.allCountryLevelSignUpBonusesLock.RUnlock() if countrySignUpBonusResponse, exists := fes.AllCountryLevelSignUpBonuses[countryCodeDetails.Name]; !exists { return fes.GetDefaultJumioCountrySignUpBonus() } else { diff --git a/routes/base.go b/routes/base.go index 8649c30e..59d6555e 100644 --- a/routes/base.go +++ b/routes/base.go @@ -358,6 +358,7 @@ type GetAppStateResponse struct { JumioKickbackUSDCents uint64 // CountrySignUpBonus is the sign-up bonus configuration for the country inferred from a request's IP address. CountrySignUpBonus CountryLevelSignUpBonus + CaptchaDeSoNanos uint64 DefaultFeeRateNanosPerKB uint64 TransactionFeeMap map[string][]TransactionFee @@ -394,6 +395,11 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) { defaultFeeRateNanosPerKB = globalParams.MinimumNetworkFeeNanosPerKB } + captchaDesoNanos, err := fes.getCaptchaRewardNanosFromGlobalState() + if err != nil { + captchaDesoNanos = 0 + } + res := &GetAppStateResponse{ MinSatoshisBurnedForProfileCreation: fes.Config.MinSatoshisForProfile, BlockHeight: fes.backendServer.GetBlockchain().BlockTip().Height, @@ -417,6 +423,7 @@ func (fes *APIServer) GetAppState(ww http.ResponseWriter, req *http.Request) { TransactionFeeMap: fes.TxnFeeMapToResponse(true), BuyETHAddress: fes.Config.BuyDESOETHAddress, Nodes: lib.NODES, + CaptchaDeSoNanos: captchaDesoNanos, // Deprecated USDCentsPerBitCloutExchangeRate: fes.GetExchangeDeSoPrice(), diff --git a/routes/exchange.go b/routes/exchange.go index 84a99649..a65cdb88 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -31,6 +31,8 @@ var ( IsBlacklisted = []byte{1} ) +const NodeVersion = "3.4.5" + const ( // RoutePathAPIBase ... RoutePathAPIBase = "/api/v1" @@ -46,6 +48,8 @@ const ( RoutePathAPINodeInfo = "/api/v1/node-info" // RoutePathAPIBlock ... RoutePathAPIBlock = "/api/v1/block" + // RoutePathAPINodeVersion ... + RoutePathAPINodeVersion = "/api/v1/node-version" ) // APIRoutes returns the routes for the public-facing API. @@ -100,6 +104,13 @@ func (fes *APIServer) APIRoutes() []Route { fes.APIBlock, PublicAccess, }, + { + "APINodeVersion", + []string{"GET"}, + RoutePathAPINodeVersion, + fes.APINodeVersion, + PublicAccess, + }, } return APIRoutes @@ -764,7 +775,7 @@ func (fes *APIServer) APITransferDeSo(ww http.ResponseWriter, rr *http.Request) if transferDeSoRequest.AmountNanos < 0 { // Create a MAX transaction txnn, totalInputt, spendAmountt, feeNanoss, err = fes.blockchain.CreateMaxSpend( - senderPublicKeyBytes, recipientPub.SerializeCompressed(), + senderPublicKeyBytes, recipientPub.SerializeCompressed(), nil, uint64(minFeeRateNanosPerKB), fes.backendServer.GetMempool(), additionalOutputs, fes.backendServer.GetFeeEstimator()) if err != nil { @@ -1169,7 +1180,7 @@ func (fes *APIServer) APITransactionInfo(ww http.ResponseWriter, rr *http.Reques } // Skip irrelevant transactions - if !isRelevantTxn { + if !isRelevantTxn && txnMeta.TransactorPublicKeyBase58Check != transactionInfoRequest.PublicKeyBase58Check { continue } @@ -1369,6 +1380,21 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) { } } +type APINodeVersionResponse struct { + Version string +} + +// APINodeVersion returns the version of the node. +func (fes *APIServer) APINodeVersion(ww http.ResponseWriter, rr *http.Request) { + if err := json.NewEncoder(ww).Encode(&APINodeVersionResponse{ + Version: NodeVersion, + }); err != nil { + APIAddError(ww, fmt.Sprintf("APINodeVersion: Problem encoding response "+ + "as JSON: %v", err)) + return + } +} + // TODO: This is a somewhat redundant version of processTransaction It exists // because the API needed to cut out the derivation of the public key from the // user object, among other things. diff --git a/routes/exchange_test.go b/routes/exchange_test.go index 94518e99..c14592cd 100644 --- a/routes/exchange_test.go +++ b/routes/exchange_test.go @@ -73,7 +73,106 @@ func GetTestBadgerDb(t *testing.T) (_db *badger.DB, _dir string) { return db, dir } -func newTestAPIServer(t *testing.T, globalStateRemoteNode string, txindex bool) (*APIServer, *APIServer, *lib.DeSoMiner) { +func NewLowDifficultyBlockchain(t *testing.T) (*lib.Blockchain, *lib.DeSoParams, *badger.DB, string) { + + // Set the number of txns per view regeneration to one while creating the txns + lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1 + + return NewLowDifficultyBlockchainWithParams(t, &lib.DeSoTestnetParams) +} + +func NewLowDifficultyBlockchainWithParams(t *testing.T, params *lib.DeSoParams) ( + *lib.Blockchain, *lib.DeSoParams, *badger.DB, string) { + + // Set the number of txns per view regeneration to one while creating the txns + lib.ReadOnlyUtxoViewRegenerationIntervalTxns = 1 + + db, dir := GetTestBadgerDb(t) + timesource := chainlib.NewMedianTime() + + // Set some special parameters for testing. If the blocks above are changed + // these values should be updated to reflect the latest testnet values. + paramsCopy := *params + paramsCopy.GenesisBlock = &lib.MsgDeSoBlock{ + Header: &lib.MsgDeSoHeader{ + Version: 0, + PrevBlockHash: lib.MustDecodeHexBlockHash("0000000000000000000000000000000000000000000000000000000000000000"), + TransactionMerkleRoot: lib.MustDecodeHexBlockHash("097158f0d27e6d10565c4dc696c784652c3380e0ff8382d3599a4d18b782e965"), + TstampSecs: uint64(1560735050), + Height: uint64(0), + Nonce: uint64(0), + // No ExtraNonce is set in the genesis block + }, + Txns: []*lib.MsgDeSoTxn{ + { + TxInputs: []*lib.DeSoInput{}, + TxOutputs: []*lib.DeSoOutput{}, + TxnMeta: &lib.BlockRewardMetadataa{ + ExtraData: []byte("They came here, to the new world. World 2.0, version 1776."), + }, + // A signature is not required for BLOCK_REWARD transactions since they + // don't spend anything. + }, + }, + } + paramsCopy.MinDifficultyTargetHex = "999999948931e5874cf66a74c0fda790dd8c7458243d400324511a4c71f54faa" + paramsCopy.MinChainWorkHex = "0000000000000000000000000000000000000000000000000000000000000000" + paramsCopy.MiningIterationsPerCycle = 500 + // Set maturity to 2 blocks so we can test spending on short chains. The + // tests rely on the maturity equaling exactly two blocks (i.e. being + // two times the time between blocks). + paramsCopy.TimeBetweenBlocks = 2 * time.Second + paramsCopy.BlockRewardMaturity = time.Second * 4 + paramsCopy.TimeBetweenDifficultyRetargets = 100 * time.Second + paramsCopy.MaxDifficultyRetargetFactor = 2 + paramsCopy.SeedBalances = []*lib.DeSoOutput{ + { + PublicKey: lib.MustBase58CheckDecode(moneyPkString), + AmountNanos: uint64(2000000 * lib.NanosPerUnit), + }, + } + + // Temporarily modify the seed balances to make a specific public + // key have some DeSo + chain, err := lib.NewBlockchain([]string{blockSignerPk}, 0, 0, + ¶msCopy, timesource, db, nil, nil, nil, false) + if err != nil { + log.Fatal(err) + } + + return chain, ¶msCopy, db, dir +} + +func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, isSender bool) (*lib.DeSoMempool, *lib.DeSoMiner) { + assert := assert.New(t) + require := require.New(t) + _ = assert + _ = require + + mempool := lib.NewDeSoMempool( + chain, 0, /* rateLimitFeeRateNanosPerKB */ + 0 /* minFeeRateNanosPerKB */, "", true, + "" /*dataDir*/, "", true) + minerPubKeys := []string{} + if isSender { + minerPubKeys = append(minerPubKeys, senderPkString) + } else { + minerPubKeys = append(minerPubKeys, recipientPkString) + } + + blockProducer, err := lib.NewDeSoBlockProducer( + 0, 1, + blockSignerSeed, + mempool, chain, + params, nil) + require.NoError(err) + + newMiner, err := lib.NewDeSoMiner(minerPubKeys, 1 /*numThreads*/, blockProducer, params) + require.NoError(err) + return mempool, newMiner +} + +func newTestAPIServer(t *testing.T, globalStateRemoteNode string) (*APIServer, *APIServer, *lib.DeSoMiner) { assert := assert.New(t) require := require.New(t) _, _ = assert, require diff --git a/routes/global_state.go b/routes/global_state.go index 88c88acc..c478695c 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -240,8 +240,11 @@ var ( // -> _GlobalStatePrefixUsernameToBlacklistState = []byte{47} - // NEXT_TAG: 48 + // The prefix for modifying the starter nanos reward for solving a captcha on signup. + // -> + _GlobalStatePrefixToCaptchaReward = []byte{48} + // NEXT_TAG: 49 ) type HotFeedApprovedPostOp struct { @@ -389,6 +392,12 @@ type UserMetadata struct { UnreadNotifications uint64 // The most recently scanned notification transaction index in the database. Stored in order to prevent unnecessary re-scanning. LatestUnreadNotificationIndex int64 + + // The last block height that the user has submitted hcaptcha verification for. + LastHcaptchaBlockHeight uint32 + // HcaptchaShouldCompProfileCreation = True if we should comp the create profile fee because the user went through the + // Captcha flow. + HcaptchaShouldCompProfileCreation bool } type TutorialStatus string @@ -673,6 +682,12 @@ func GlobalStateKeyForBlacklistedProfileByUsername(username string) []byte { return key } +// Key for accessing the captcha reward amount. +func GlobalStateKeyForCaptchaRewardAmountNanos() []byte { + key := append([]byte{}, _GlobalStatePrefixToCaptchaReward...) + return key +} + // Key for accessing the blacklist audit logs associated with a user. func GlobalStateKeyForBlacklistAuditLogs(username string) []byte { key := append([]byte{}, _GlobalStatePrefixBlacklistAuditLog...) diff --git a/routes/hot_feed.go b/routes/hot_feed.go index 853cfc87..9a884efa 100644 --- a/routes/hot_feed.go +++ b/routes/hot_feed.go @@ -83,13 +83,23 @@ func (fes *APIServer) StartHotFeedRoutine() { for { select { case <-time.After(30 * time.Second): - resetCache := false - if cacheResetCounter >= ResetCachesIterationLimit { - resetCache = true - cacheResetCounter = 0 - } - fes.UpdateHotFeed(resetCache) - cacheResetCounter += 1 + // Use an inner function to unlock the mutex with a defer statement. + func() { + // If we're syncing a snapshot, we need to lock the DB mutex before updating the hot feed. + // This is because at the end of a snapshot sync, we re-start the DB, which will cause + // the hot feed routine to panic if it's in the middle of updating the hot feed. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } + resetCache := false + if cacheResetCounter >= ResetCachesIterationLimit { + resetCache = true + cacheResetCounter = 0 + } + fes.UpdateHotFeed(resetCache) + cacheResetCounter += 1 + }() case <-fes.quit: break out } diff --git a/routes/server.go b/routes/server.go index 7c46af86..710a0d4c 100644 --- a/routes/server.go +++ b/routes/server.go @@ -155,10 +155,12 @@ const ( RoutePathSubmitPhoneNumberVerificationCode = "/api/v0/submit-phone-number-verification-code" RoutePathResendVerifyEmail = "/api/v0/resend-verify-email" RoutePathVerifyEmail = "/api/v0/verify-email" + RoutePathVerifyCaptcha = "/api/v0/verify-captcha" RoutePathJumioBegin = "/api/v0/jumio-begin" RoutePathJumioCallback = "/api/v0/jumio-callback" RoutePathJumioFlowFinished = "/api/v0/jumio-flow-finished" RoutePathGetJumioStatusForPublicKey = "/api/v0/get-jumio-status-for-public-key" + RoutePathAdminSetCaptchaRewardNanos = "/api/v0/admin/set-captcha-reward-nanos" // tutorial.go RoutePathGetTutorialCreators = "/api/v0/get-tutorial-creators" @@ -452,7 +454,8 @@ type APIServer struct { CountKeysWithDESO uint64 // map of country name to sign up bonus data - AllCountryLevelSignUpBonuses map[string]CountrySignUpBonusResponse + allCountryLevelSignUpBonusesLock sync.RWMutex + AllCountryLevelSignUpBonuses map[string]CountrySignUpBonusResponse // Frequently accessed data from global state USDCentsToDESOReserveExchangeRate uint64 @@ -1112,6 +1115,13 @@ func (fes *APIServer) NewRouter() *muxtrace.Router { fes.VerifyEmail, PublicAccess, }, + { + "VerifyCaptcha", + []string{"POST", "OPTIONS"}, + RoutePathVerifyCaptcha, + fes.HandleCaptchaVerificationRequest, + PublicAccess, + }, { "GetUserDerivedKeys", []string{"POST", "OPTIONS"}, @@ -1753,6 +1763,13 @@ func (fes *APIServer) NewRouter() *muxtrace.Router { fes.AdminResetPhoneNumber, SuperAdminAccess, }, + { + "SetCaptchaRewardNanos", + []string{"POST", "OPTIONS"}, + RoutePathAdminSetCaptchaRewardNanos, + fes.AdminSetCaptchaRewardNanos, + SuperAdminAccess, + }, // End all /admin routes // GET endpoints for managing parameters related to Buying DeSo { @@ -2607,11 +2624,20 @@ func (fes *APIServer) StartSeedBalancesMonitoring() { return } tags := []string{} - fes.logBalanceForSeed(fes.Config.StarterDESOSeed, "STARTER_DESO", tags) - fes.logBalanceForSeed(fes.Config.BuyDESOSeed, "BUY_DESO", tags) - for label, publicKey := range fes.Config.PublicKeyBalancesToMonitor { - fes.logBalanceForPublicKey(publicKey, label, tags) - } + // Use an inner function to unlock the mutex with a defer statement. + func() { + // If we're syncing a snapshot, we need to lock the DB in case the DB is restarted. This happens + // at the end of the snapshot sync. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } + fes.logBalanceForSeed(fes.Config.StarterDESOSeed, "STARTER_DESO", tags) + fes.logBalanceForSeed(fes.Config.BuyDESOSeed, "BUY_DESO", tags) + for label, publicKey := range fes.Config.PublicKeyBalancesToMonitor { + fes.logBalanceForPublicKey(publicKey, label, tags) + } + }() case <-fes.quit: break out } @@ -2692,6 +2718,12 @@ func (fes *APIServer) SetGlobalStateCache() { if fes.backendServer == nil { return } + // If we're syncing a snapshot, we need to lock the DB in case the DB is restarted. This happens at + // the end of the snapshot sync. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView() if err != nil { glog.Errorf("SetGlobalStateCache: problem with GetAugmentedUniversalView: %v", err) diff --git a/routes/supply.go b/routes/supply.go index 83743504..f6114dd4 100644 --- a/routes/supply.go +++ b/routes/supply.go @@ -44,6 +44,11 @@ func (fes *APIServer) StartSupplyMonitoring() { } func (fes *APIServer) UpdateSupplyStats() { + // Prevent access to the DB while it's reset. This only happens when we're syncing a snapshot. + if fes.backendServer.GetBlockchain().ChainState() == lib.SyncStateSyncingSnapshot { + fes.backendServer.DbMutex.Lock() + defer fes.backendServer.DbMutex.Unlock() + } totalSupply := uint64(0) totalKeysWithDESO := uint64(0) // Get all the balances from the DB diff --git a/routes/transaction.go b/routes/transaction.go index b3355b50..11667c73 100644 --- a/routes/transaction.go +++ b/routes/transaction.go @@ -25,7 +25,7 @@ import ( ) type GetTxnRequest struct { - // TxnHash to fetch. + // TxnHashHex to fetch. TxnHashHex string `safeForLogging:"true"` } @@ -459,14 +459,16 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata } // Additional fee is set to the create profile fee when we are creating a profile additionalFees := utxoView.GlobalParamsEntry.CreateProfileFeeNanos - + if additionalFees == 0 { + return 0, nil, nil + } existingMetamaskAirdropMetadata, err := fes.GetMetamaskAirdropMetadata(profilePublicKey) if err != nil { return 0, nil, fmt.Errorf("Error geting metamask airdrop metadata from global state: %v", err) } // Only comp create profile fee if frontend server has both twilio and starter deso seed configured and the user // has verified their profile. - if !fes.Config.CompProfileCreation || fes.Config.StarterDESOSeed == "" || fes.Twilio == nil || (userMetadata.PhoneNumber == "" && !userMetadata.JumioVerified && existingMetamaskAirdropMetadata == nil) { + if !fes.Config.CompProfileCreation || fes.Config.StarterDESOSeed == "" || (fes.Config.HCaptchaSecret == "" && fes.Twilio == nil) || (userMetadata.PhoneNumber == "" && !userMetadata.JumioVerified && existingMetamaskAirdropMetadata == nil && userMetadata.LastHcaptchaBlockHeight == 0) { return additionalFees, nil, nil } var currentBalanceNanos uint64 @@ -507,11 +509,16 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata return additionalFees, nil, nil } updateMetamaskAirdropMetadata = true - } else { + } else if userMetadata.JumioVerified { // User has been Jumio verified but should comp profile creation is false, just return if !userMetadata.JumioShouldCompProfileCreation { return additionalFees, nil, nil } + } else if userMetadata.LastHcaptchaBlockHeight != 0 { + // User has been captcha verified but should comp profile creation is false, just return + if !userMetadata.HcaptchaShouldCompProfileCreation { + return additionalFees, nil, nil + } } // Find the minimum starter bit deso amount @@ -530,6 +537,10 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata // We comp the create profile fee minus the minimum starter deso amount divided by 2. // This discourages botting while covering users who verify a phone number. compAmount := createProfileFeeNanos - (minStarterDESONanos / 2) + if (minStarterDESONanos / 2) > createProfileFeeNanos { + compAmount = createProfileFeeNanos + } + // If the user won't have enough deso to cover the fee, this is an error. if currentBalanceNanos+compAmount < createProfileFeeNanos { return 0, nil, fmt.Errorf("Creating a profile requires DeSo. Please purchase some to create a profile.") @@ -547,6 +558,11 @@ func (fes *APIServer) CompProfileCreation(profilePublicKey []byte, userMetadata if err = fes.putPhoneNumberMetadataInGlobalState(newPhoneNumberMetadata, userMetadata.PhoneNumber); err != nil { return 0, nil, fmt.Errorf("UpdateProfile: Error setting ShouldComp to false for phone number metadata: %v", err) } + } else if userMetadata.LastHcaptchaBlockHeight != 0 { + userMetadata.HcaptchaShouldCompProfileCreation = false + if err = fes.putUserMetadataInGlobalState(userMetadata); err != nil { + return 0, nil, fmt.Errorf("UpdateProfile: Error setting ShouldComp to false for jumio user metadata: %v", err) + } } else { // Set JumioShouldCompProfileCreation to false so we don't continue to comp profile creation. userMetadata.JumioShouldCompProfileCreation = false @@ -978,10 +994,11 @@ func (fes *APIServer) ExceedsDeSoBalance(nanosPurchased uint64, seed string) (bo // SendDeSoRequest ... type SendDeSoRequest struct { - SenderPublicKeyBase58Check string `safeForLogging:"true"` - RecipientPublicKeyOrUsername string `safeForLogging:"true"` - AmountNanos int64 `safeForLogging:"true"` - MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + SenderPublicKeyBase58Check string `safeForLogging:"true"` + RecipientPublicKeyOrUsername string `safeForLogging:"true"` + AmountNanos int64 `safeForLogging:"true"` + MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + ExtraData map[string]string `safeForLogging:"true"` // No need to specify ProfileEntryResponse in each TransactionFee TransactionFees []TransactionFee `safeForLogging:"true"` @@ -1059,6 +1076,12 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { return } + extraData, err := EncodeExtraDataMap(requestData.ExtraData) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("SendDeSo: Problem encoding ExtraData: %v", err)) + return + } + // If the AmountNanos is less than zero then we have a special case where we create // a transaction with the maximum spend. var txnn *lib.MsgDeSoTxn @@ -1069,8 +1092,8 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { if requestData.AmountNanos < 0 { // Create a MAX transaction txnn, totalInputt, spendAmountt, feeNanoss, err = fes.blockchain.CreateMaxSpend( - senderPkBytes, recipientPkBytes, requestData.MinFeeRateNanosPerKB, - fes.backendServer.GetMempool(), additionalOutputs, fes.backendServer.GetFeeEstimator()) + senderPkBytes, recipientPkBytes, extraData, requestData.MinFeeRateNanosPerKB, + fes.backendServer.GetMempool(), additionalOutputs) if err != nil { _AddBadRequestError(ww, fmt.Sprintf("SendDeSo: Error processing MAX transaction: %v", err)) return @@ -1100,6 +1123,10 @@ func (fes *APIServer) SendDeSo(ww http.ResponseWriter, req *http.Request) { // inputs and change. } + if len(extraData) > 0 { + txnn.ExtraData = extraData + } + // Add inputs to the transaction and do signing, validation, and broadcast // depending on what the user requested. totalInputt, spendAmountt, changeAmountt, feeNanoss, err = @@ -2107,6 +2134,8 @@ type SendDiamondsRequest struct { MinFeeRateNanosPerKB uint64 `safeForLogging:"true"` + ExtraData map[string]string `safeForLogging:"true"` + // No need to specify ProfileEntryResponse in each TransactionFee TransactionFees []TransactionFee `safeForLogging:"true"` @@ -2182,6 +2211,12 @@ func (fes *APIServer) SendDiamonds(ww http.ResponseWriter, req *http.Request) { return } + extraData, err := EncodeExtraDataMap(requestData.ExtraData) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("SendDiamonds: Problem encoding extra data: %v", err)) + return + } + // Try and create the transfer with diamonds for the user. // We give diamonds in DESO if we're past the corresponding block height. blockHeight := fes.blockchain.BlockTip().Height + 1 @@ -2201,6 +2236,7 @@ func (fes *APIServer) SendDiamonds(ww http.ResponseWriter, req *http.Request) { senderPublicKeyBytes, diamondPostHash, requestData.DiamondLevel, + extraData, // Standard transaction fields requestData.MinFeeRateNanosPerKB, fes.backendServer.GetMempool(), additionalOutputs, fes.backendServer.GetFeeEstimator()) diff --git a/routes/verify.go b/routes/verify.go index 97237d85..e668b593 100644 --- a/routes/verify.go +++ b/routes/verify.go @@ -141,8 +141,9 @@ func (fes *APIServer) canUserCreateProfile(userMetadata *UserMetadata, utxoView return false, err } // User can create a profile if they have a phone number or if they have enough DeSo to cover the create profile fee. + // User can also create a profile if they've successfully filled out a captcha. // The PhoneNumber is only set if the user has passed phone number verification. - if userMetadata.PhoneNumber != "" || totalBalanceNanos >= utxoView.GlobalParamsEntry.CreateProfileFeeNanos { + if userMetadata.PhoneNumber != "" || totalBalanceNanos >= utxoView.GlobalParamsEntry.CreateProfileFeeNanos || userMetadata.LastHcaptchaBlockHeight > 0 { return true, nil } @@ -251,6 +252,213 @@ func (fes *APIServer) validatePhoneNumberNotAlreadyInUse(phoneNumber string, use return nil } +type SubmitCaptchaVerificationRequest struct { + Token string + JWT string + PublicKeyBase58Check string +} + +type SubmitCaptchaVerificationResponse struct { + Success bool + TxnHashHex string +} + +func (fes *APIServer) HandleCaptchaVerificationRequest(ww http.ResponseWriter, req *http.Request) { + decoder := json.NewDecoder(io.LimitReader(req.Body, MaxRequestBodySizeBytes)) + requestData := SubmitCaptchaVerificationRequest{} + if err := decoder.Decode(&requestData); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Problem parsing request body: %v", err)) + return + } + + // Validate their permissions + isValid, err := fes.ValidateJWT(requestData.PublicKeyBase58Check, requestData.JWT) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Error validating JWT: %v", err)) + } + if !isValid { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Invalid token: %v", err)) + return + } + + txnHashHex, err := fes.verifyHCaptchaTokenAndSendStarterDESO(requestData.Token, requestData.PublicKeyBase58Check) + + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Error verifying captcha: %v", err)) + return + } + + res := SubmitCaptchaVerificationResponse{ + Success: true, + TxnHashHex: txnHashHex, + } + if err = json.NewEncoder(ww).Encode(res); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleCaptchaVerificationRequest: Problem encoding response: %v", err)) + return + } +} + +type AdminUpdateCaptchaRewardRequest struct { + // Amount of nanos to reward for a successful captcha. + RewardNanos uint64 +} + +type AdminUpdateCaptchaRewardResponse struct { + // Amount of nanos to reward for a successful captcha. + RewardNanos uint64 +} + +// HandleAdminUpdateCaptchaRewardRequest allows an admin to update the captcha reward amount. +func (fes *APIServer) AdminSetCaptchaRewardNanos(ww http.ResponseWriter, req *http.Request) { + decoder := json.NewDecoder(io.LimitReader(req.Body, MaxRequestBodySizeBytes)) + requestData := AdminUpdateCaptchaRewardRequest{} + if err := decoder.Decode(&requestData); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Problem parsing request body: %v", err)) + return + } + + // Ensure that the reward amount is not greater than the starter deso amount flag. + if requestData.RewardNanos > fes.Config.StarterDESONanos { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Reward amount %v exceeds starter deso amount %v", requestData.RewardNanos, fes.Config.StarterDESONanos)) + return + } + + if err := fes.putCaptchaRewardNanosInGlobalState(requestData.RewardNanos); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Error putting captcha reward in global state: %v", err)) + return + } + + res := AdminUpdateCaptchaRewardResponse{ + RewardNanos: requestData.RewardNanos, + } + if err := json.NewEncoder(ww).Encode(res); err != nil { + _AddBadRequestError(ww, fmt.Sprintf("HandleAdminUpdateCaptchaRewardRequest: Problem encoding response: %v", err)) + return + } +} + +// getCaptchaRewardNanosFromGlobalState returns the amount of nanos to reward for a successful captcha from global state. +func (fes *APIServer) getCaptchaRewardNanosFromGlobalState() (uint64, error) { + dbKey := GlobalStateKeyForCaptchaRewardAmountNanos() + + rewardNanosBytes, err := fes.GlobalState.Get(dbKey) + if err != nil { + return 0, fmt.Errorf( + "getCaptchaRewardNanosFromGlobalState: Problem with Get: %v", err) + } + + rewardNanos, err := lib.ReadUvarint(bytes.NewReader(rewardNanosBytes)) + + return rewardNanos, nil +} + +// putCaptchaRewardNanosInGlobalState puts the amount of nanos to reward for a successful captcha in global state. +func (fes *APIServer) putCaptchaRewardNanosInGlobalState(rewardNanos uint64) error { + dbKey := GlobalStateKeyForCaptchaRewardAmountNanos() + + rewardNanosBytes := lib.UintToBuf(rewardNanos) + + if err := fes.GlobalState.Put(dbKey, rewardNanosBytes); err != nil { + return fmt.Errorf( + "putCaptchaRewardNanosInGlobalState: Problem with Put: %v", err) + } + + return nil +} + +// verifyHCaptchaTokenAndSendStarterDESO verifies the captcha token and sends the starter DESO to the user. +func (fes *APIServer) verifyHCaptchaTokenAndSendStarterDESO(token string, publicKeyBase58Check string) (txnHashHex string, err error) { + if fes.Config.StarterDESOSeed == "" { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Starter DESO seed not set") + } + + // Retrieve the amount of nanos to reward for a successful captcha. + amountToSendNanos, err := fes.getCaptchaRewardNanosFromGlobalState() + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with getCaptchaRewardNanosFromGlobalState: %v", err) + } + + // Decode the public key. + publicKeyBytes, _, err := lib.Base58CheckDecode(publicKeyBase58Check) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem decoding public key: %v", err) + } + + // Ensure the user has not already received the starter DESO for submitting a successful captcha. + userMetadata, err := fes.getUserMetadataFromGlobalState(publicKeyBase58Check) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with getUserMetadataFromGlobalState: %v", err) + } + + if userMetadata.LastHcaptchaBlockHeight != 0 { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: LastHcaptchaBlockHeight is already set") + } + + // Verify the token with hCaptcha. + verificationSuccess, err := fes.verifyHCaptchaToken(token) + + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Error verifying captcha: %v", err) + } + + if !verificationSuccess { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Captcha verification failed") + } + + // Update the user's metadata to indicate that they have received the starter DESO. + lastBlockheight := fes.blockchain.BlockTip().Height + userMetadata.LastHcaptchaBlockHeight = lastBlockheight + userMetadata.HcaptchaShouldCompProfileCreation = true + + if err = fes.putUserMetadataInGlobalState(userMetadata); err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Problem with putUserMetadataInGlobalState: %v", err) + } + + // Send the starter DESO to the user. + var txnHash *lib.BlockHash + txnHash, err = fes.SendSeedDeSo(publicKeyBytes, amountToSendNanos, false) + if err != nil { + return "", fmt.Errorf("HandleCaptchaVerificationRequest: Error sending seed DeSo: %v", err) + } + + // Log the transaction to datadog. + if fes.backendServer.GetStatsdClient() != nil { + fes.backendServer.GetStatsdClient().Incr("SEND_STARTER_DESO_CAPTCHA", nil, 1) + } + + return txnHash.String(), nil +} + +type VerificationResponse struct { + Success bool `json:"success"` + ErrorCodes []string `json:"error-codes"` +} + +const VERIFY_URL = "https://hcaptcha.com/siteverify" + +// verifyHCaptchaToken verifies the captcha token via the hCaptcha API. +func (fes *APIServer) verifyHCaptchaToken(token string) (bool, error) { + // Construct and send the request to hcaptcha. + data := url.Values{} + data.Set("secret", fes.Config.HCaptchaSecret) + data.Set("response", token) + + resp, err := http.PostForm(VERIFY_URL, data) + if err != nil { + return false, err + } + defer resp.Body.Close() + + // Parse the response and return the result. + var verificationResponse VerificationResponse + err = json.NewDecoder(resp.Body).Decode(&verificationResponse) + if err != nil { + return false, err + } + + return verificationResponse.Success, nil +} + type SubmitPhoneNumberVerificationCodeRequest struct { JWT string PublicKeyBase58Check string diff --git a/scripts/nodes/n0 b/scripts/nodes/n0 index 84289258..a80aa764 100755 --- a/scripts/nodes/n0 +++ b/scripts/nodes/n0 @@ -18,6 +18,7 @@ rm /tmp/main.*.log --block-cypher-api-key=092dae962ea44b02809a4c74408b42a1 \ --min-satoshis-for-profile=0 \ --connect-ips=deso-seed-2.io:17000 \ + --hypersync-max-queue-size=10 \ --run-hot-feed-routine=true ) diff --git a/scripts/nodes/n0_test b/scripts/nodes/n0_test index fb9f8855..b12c2682 100755 --- a/scripts/nodes/n0_test +++ b/scripts/nodes/n0_test @@ -36,11 +36,12 @@ rm /tmp/main.*.log --block-producer-seed='essence camp ghost remove document vault ladder swim pupil index apart ring' \ --starter-deso-seed='road congress client market couple bid risk escape artwork rookie artwork food' \ --data-dir=/tmp/n0_test_00000 \ - --access-control-allow-origins=http://localhost:4200,http://localhost:80,http://localhost:18002,http://localhost:4201,http://localhost:18001 \ + --access-control-allow-origins=http://localhost:4200,http://localhost:80,http://localhost:18002,http://localhost:4201,http://localhost:18001,http://localhost:3000,localhost:3000 \ --secure-header-allow-hosts=localhost:4200 \ --secure-header-development=true \ --block-cypher-api-key=092dae962ea44b02809a4c74408b42a1 \ --min-satoshis-for-profile=0 \ --expose-global-state=true \ --show-processing-spinners=true \ + --comp-profile-creation=true \ --metamask-airdrop-deso-nanos-amount=1000 ) diff --git a/scripts/tools/clear_ancestral_records.go b/scripts/tools/clear_ancestral_records.go index cf7a9edb..7f862488 100644 --- a/scripts/tools/clear_ancestral_records.go +++ b/scripts/tools/clear_ancestral_records.go @@ -16,7 +16,7 @@ func main() { fmt.Printf("Error reading db1 err: %v", err) return } - snap, err, _ := lib.NewSnapshot(db, dbDir, lib.SnapshotBlockHeightPeriod, false, false, &lib.DeSoMainnetParams, false) + snap, err, _ := lib.NewSnapshot(db, dbDir, lib.SnapshotBlockHeightPeriod, false, false, &lib.DeSoMainnetParams, false, lib.HypersyncDefaultMaxQueueSize) if err != nil { fmt.Printf("Error reading snap err: %v", err) return diff --git a/test.Dockerfile b/test.Dockerfile index a1ed5cd2..a7970f17 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -7,6 +7,9 @@ RUN apk add --update bash cmake g++ gcc git make vips-dev COPY --from=golang:1.20-alpine /usr/local/go/ /usr/local/go/ ENV PATH="/usr/local/go/bin:${PATH}" +# Declare an ARG for the branch name with a default value of "main" +ARG BRANCH_NAME=main + WORKDIR /deso/src RUN git clone https://github.com/deso-protocol/core.git @@ -19,6 +22,9 @@ RUN git pull && \ RUN go mod download RUN ./scripts/install-relic.sh +# Try to checkout to the specified branch. If it fails, checkout main. +RUN git checkout ${BRANCH_NAME} || (echo "Branch ${BRANCH_NAME} not found. Falling back to main." && git checkout main) + WORKDIR /deso/src/backend COPY go.mod .