diff --git a/cmd/node.go b/cmd/node.go index 06f3cbedc..8c123066b 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -2,6 +2,7 @@ package cmd import ( "encoding/hex" + "errors" "flag" "fmt" "net" @@ -26,6 +27,40 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/profiler" ) +var ( + // ErrAlreadyStarted is returned when somebody tries to start an already + // running service. + ErrAlreadyStarted = errors.New("already started") + // ErrAlreadyStopped is returned when somebody tries to stop an already + // stopped service (without resetting it). + ErrAlreadyStopped = errors.New("already stopped") + // ErrNodeNeverStarted is returned when somebody tries to find or change the + // running status of the server without never starting it once. + ErrNodeNeverStarted = errors.New("never started the node instance") + // Cannot set node status to NEVERSTARTED. + // NEVERSTARTED is the default status, cannot set deliberately. + ErrCannotSetToNeverStarted = errors.New("cannot set the node status to neverstarted") + // Erorr when invalid status is set for node + ErrInvalidNodeStatus = errors.New("invalid node status. check cmd/node.go for valid states") + // Invalid Status return value for the *Node.GetStatus()/*Node.getStatusWithoutLock helper functions. + // byte(255) is a reserved status for invalid status code for the Node + INVALIDNODESTATUS = NodeStatus(255) +) + +// Valid status codes for the Node. +// Status should be retrieved using the helper functions *Node.GetStatus() +const ( + // Status when the node is not initialized or started for the first time + NEVERSTARTED NodeStatus = iota // byte(0) + // Status when the node is initialized using *Node.Start + RUNNING // byte(1) + // Status when the node is stopped + STOPPED // byte(2) +) + +// custom byte type to indicate the running status of the Node. +type NodeStatus byte + type Node struct { Server *lib.Server ChainDB *badger.DB @@ -34,12 +69,16 @@ type Node struct { Config *Config Postgres *lib.Postgres - // IsRunning is false when a NewNode is created, set to true on Start(), set to false - // after Stop() is called. Mainly used in testing. - IsRunning bool - // runningMutex is held whenever we call Start() or Stop() on the node. - runningMutex sync.Mutex + // status is nil when a NewNode is created, it is initialized and set to RUNNING [byte(1)] on node.Start(), + // set to STOPPED [byte(2)] after Stop() is called. + // Use the convenience methods SetStatus/SetStatusRunningWithoutLock/SetStatusStoppedWithoutLock to change node status. + // Use *Node.IsRunning() to check if the node is running. + // Use *Node.GetStatus() to retrieve the status of the node. + // Use *Node.getStatusWithoutLock() if the statusMutex is held by the caller. + status *NodeStatus + // Held whenever the status of the node is read or altered. + statusMutex sync.Mutex // internalExitChan is used internally to signal that a node should close. internalExitChan chan struct{} // nodeMessageChan is passed to the core engine and used to trigger node actions such as a restart or database reset. @@ -50,6 +89,7 @@ type Node struct { func NewNode(config *Config) *Node { result := Node{} + result.Config = config result.Params = config.Params result.internalExitChan = make(chan struct{}) @@ -68,8 +108,8 @@ func (node *Node) Start(exitChannels ...*chan struct{}) { flag.Set("alsologtostderr", "true") flag.Parse() glog.CopyStandardLogTo("INFO") - node.runningMutex.Lock() - defer node.runningMutex.Unlock() + node.statusMutex.Lock() + defer node.statusMutex.Unlock() node.internalExitChan = make(chan struct{}) node.nodeMessageChan = make(chan lib.NodeMessage) @@ -251,7 +291,39 @@ func (node *Node) Start(exitChannels ...*chan struct{}) { } } } - node.IsRunning = true + + // Load the node status. + // This is to identify whether the node is initialized for the first time or it's a restart. + status, err := node.getStatusWithoutLock() + if err != nil { + glog.Fatal("Failed to load node status") + } + // Handling the first time initialization and node restart cases separately. + // This allows us to log the events and even run exclusive logic if any. + switch status { + case NEVERSTARTED: + glog.Info("Changing node status from NEVERSTARTED to RUNNING...") + // The node status is changed from NEVERSTARTED to RUNNING only once + // when the node is started for the first time. + err = node.SetStatusRunningWithoutLock() + if err != nil { + glog.Fatalf("Error running Node -- %v", err) + } + case STOPPED: + // This case is called during a node restart. + // During restart the Node is first STOPPED before setting the + // status again to RUNNING. + glog.Info("Changing node status from STOP to RUNNING...") + err = node.SetStatusRunningWithoutLock() + if err != nil { + glog.Fatalf("Error running Node -- %v", err) + } + default: + // Rare occurrence. Happens if you set an invalid node status while restarting a node. + // cannot start the node if the status of the Node is already set to RUNNING. + panic(fmt.Sprintf("Cannot change node status to RUNNING from the current status %v", status)) + + } if shouldRestart { if node.nodeMessageChan != nil { @@ -265,6 +337,10 @@ func (node *Node) Start(exitChannels ...*chan struct{}) { go func() { // If an internalExitChan is triggered then we won't immediately signal a shutdown to the parent context through // the exitChannels. When internal exit is called, we will just restart the node in the background. + + // Example: First call Node.Stop(), then call Node.Start() to internally restart the service. + // Since Stop() closes the internalExitChain, the exitChannels sent by the users of the + // core library will not closed. Thus ensuring that an internal restart doesn't affect the users of the core library. select { case _, open := <-node.internalExitChan: if !open { @@ -274,6 +350,7 @@ func (node *Node) Start(exitChannels ...*chan struct{}) { } node.Stop() + for _, channel := range exitChannels { if *channel != nil { close(*channel) @@ -284,14 +361,154 @@ func (node *Node) Start(exitChannels ...*chan struct{}) { }() } -func (node *Node) Stop() { - node.runningMutex.Lock() - defer node.runningMutex.Unlock() +// Changes node status to STOPPED. +// Cannot transition from NEVERSTARTED to STOPPED. +// Valid transition sequence is NEVERSTARTED->RUNNING->STOPPED. +func (node *Node) SetStatusStoppedWithoutLock() error { + if err := node.setStatusWithoutLock(STOPPED); err != nil { + return fmt.Errorf("failed to set status to stop: %w", err) + } + return nil +} + +// Changes node status to RUNNING. +func (node *Node) SetStatusRunningWithoutLock() error { + if err := node.setStatusWithoutLock(RUNNING); err != nil { + return fmt.Errorf("failed to set status to running: %w", err) + } + return nil +} - if !node.IsRunning { - return +// Loads the status of Node. +// Modifying getStatusWithoutLock() and the validateStatus() functions and their tests are hard requirements +// to add new status codes for the node. +func (node *Node) getStatusWithoutLock() (NodeStatus, error) { + // Never initialized the server using *Node.Start() + if node.status == nil { + return NEVERSTARTED, nil + } + // using switch and case prevents from adding new invalid codes + // without changing the getStatusWithoutLock() function and its unit test. + switch *node.status { + // Node is started using *Node.Start() and not stopped yet. + case RUNNING: + return RUNNING, nil + // Node was once initialized, but currently stopped. + // Set to STOPPED on calling *Node.Stop() + case STOPPED: + return STOPPED, nil + // Any other status code apart from the cases above are considered INVALID!!!! + default: + return INVALIDNODESTATUS, ErrInvalidNodeStatus + } +} + +// Wrapper function to get the status of the node with statusMutex. +// Use private method getStatusWithoutLock(), in case the locks are held by the caller. +func (node *Node) GetStatus() (NodeStatus, error) { + node.statusMutex.Lock() + defer node.statusMutex.Unlock() + + return node.getStatusWithoutLock() +} + +// Verifies whether a status code of the node is a valid status code. +// Used while changing the status of the node +// Modifying getStatusWithoutLock() and the validateStatus() functions and their tests are hard requirements +// to add new status codes for the node. +func ValidateNodeStatus(status NodeStatus) error { + switch status { + // Instance of the *Node is created, but not yet initialized using *Node.Start() + case NEVERSTARTED: + return nil + // Node is started using *Node.Start() and not stopped yet. + case RUNNING: + return nil + // Node was once initialized, but currently stopped. + // Set to STOPPED on calling *Node.Stop() + case STOPPED: + return nil + // Any other status code apart from the cases above are considered INVALID, + default: + return ErrInvalidNodeStatus + } +} + +// changes the running status of the node +// Always use this function to change the status of the node. +func (node *Node) setStatusWithoutLock(newStatus NodeStatus) error { + if err := ValidateNodeStatus(newStatus); err != nil { + return err + } + + // Cannot deliberately change the status to NEVERSTARTED. + // NEVERSTARTED is the default status of the node before the + // node is started using *Node.Start(). + if newStatus == NEVERSTARTED { + return ErrCannotSetToNeverStarted + } + // Load the current status of the Node. + status, err := node.getStatusWithoutLock() + if err != nil { + return err + } + + // Cannot change the status of the server to STOPPED while it was never initialized + // in the first place. Can stop the never only after it's started using *Node.Start(). + // Valid status transition is NEVERSTARTED -> RUNNING -> STOPPED -> RUNNING + if status == NEVERSTARTED && newStatus == STOPPED { + return ErrNodeNeverStarted + } + + // No need to change the status if the new status is same as current status. + if newStatus == status { + switch newStatus { + case RUNNING: + return ErrAlreadyStarted + case STOPPED: + return ErrAlreadyStopped + } + } + + node.status = &newStatus + + return nil +} + +// Wrapper function for changing node status with statusMutex +// use SetStatusWithoutLock if the lock is held by the caller. +// calling this function while the statusMutex lock is already held will lead to deadlocks! +func (node *Node) SetStatus(newStatus NodeStatus) error { + node.statusMutex.Lock() + defer node.statusMutex.Unlock() + + return node.setStatusWithoutLock(newStatus) + +} + +// Helper function to check if the node is running. +// returns false if node status is not set to RUNNING +func (node *Node) IsRunning() bool { + status, err := node.GetStatus() + + if err != nil { + return false } - node.IsRunning = false + + return status == RUNNING +} + +func (node *Node) Stop() error { + node.statusMutex.Lock() + defer node.statusMutex.Unlock() + + // Change nodes running status to stop + // Node's current status has to be RUNNING to be able to STOP! + if err := node.SetStatusStoppedWithoutLock(); err != nil { + glog.Errorf("Error stopping Node -- %v", err) + return err + } + glog.Infof(lib.CLog(lib.Yellow, "Node is shutting down. This might take a minute. Please don't "+ "close the node now or else you might corrupt the state.")) @@ -327,6 +544,12 @@ func (node *Node) Stop() { close(node.internalExitChan) node.internalExitChan = nil } + return nil +} + +// Return internal exit channel to wait for the node to stop. +func (node *Node) Quit() chan struct{} { + return node.internalExitChan } // Close a database and handle the stopWaitGroup accordingly. We close databases in a go routine to speed up the process. @@ -353,26 +576,45 @@ func (node *Node) listenToNodeMessages(exitChannels ...*chan struct{}) { case <-node.internalExitChan: break case operation := <-node.nodeMessageChan: - if !node.IsRunning { - panic("Node.listenToNodeMessages: Node is currently not running, nodeMessageChan should've not been called!") - } - glog.Infof("Node.listenToNodeMessages: Stopping node") - node.Stop() - glog.Infof("Node.listenToNodeMessages: Finished stopping node") switch operation { + case lib.NodeRestart: + // Using Mutex while accessing the Node status to avoid any race conditions + glog.Infof("Node.listenToNodeMessages: Restarting node.") + glog.Infof("Node.listenToNodeMessages: Stopping node") + // Stopping node + // Stop only works if the current state of the node is RUNNING. + if err := node.Stop(); err != nil { + panic(fmt.Sprintf("Error stopping node: %v", err)) + } + + glog.Infof("Node.listenToNodeMessages: Finished stopping node") + case lib.NodeErase: + glog.Infof("Node.listenToNodeMessages: Restarting node with a Database erase.") + glog.Infof("Node.listenToNodeMessages: Stopping node") + // cannot stop the node if the node is not already in RUNNING state. + // Stop the node and remove the data directory if the server is in RUNNING state. + + // Stopping node. + // When restart with database erase fails when the node starts for the first time + // This is because the status of the node is still NEVERSTARTED. + // We log the Stop failure, and still go ahead with the hard restart, a.k.a restart with database erase! + if err := node.Stop(); err != nil { + glog.Infof("Node.listenToNodeMessages: Node Stop operation failed.") + glog.Infof("Still going ahead with the database erase.") + } else { + glog.Infof("Node.listenToNodeMessages: Finished stopping node") + } + if err := os.RemoveAll(node.Config.DataDirectory); err != nil { glog.Fatal(lib.CLog(lib.Red, fmt.Sprintf("IMPORTANT: Problem removing the directory (%v), you "+ "should run `rm -rf %v` to delete it manually. Error: (%v)", node.Config.DataDirectory, node.Config.DataDirectory, err))) - return } } - glog.Infof("Node.listenToNodeMessages: Restarting node") // Wait a few seconds so that all peer messages we've sent while closing the node get propagated in the network. go node.Start(exitChannels...) - break } } diff --git a/integration_testing/blocksync_test.go b/integration_testing/blocksync_test.go index af1bc3637..435cfcf51 100644 --- a/integration_testing/blocksync_test.go +++ b/integration_testing/blocksync_test.go @@ -2,19 +2,20 @@ package integration_testing import ( "fmt" + "os" + "testing" + "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" "github.com/stretchr/testify/require" - "os" - "testing" ) // TestSimpleBlockSync test if a node can successfully sync from another node: -// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. -// 3. bridge node1 and node2 -// 4. node2 syncs MaxSyncBlockHeight blocks from node1. -// 5. compare node1 db matches node2 db. +// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. +// 3. bridge node1 and node2 +// 4. node2 syncs MaxSyncBlockHeight blocks from node1. +// 5. compare node1 db matches node2 db. func TestSimpleBlockSync(t *testing.T) { require := require.New(t) _ = require @@ -54,13 +55,13 @@ func TestSimpleBlockSync(t *testing.T) { } // TestSimpleSyncRestart tests if a node can successfully restart while syncing blocks. -// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. -// 3. bridge node1 and node2 -// 4. node2 syncs between 10 and MaxSyncBlockHeight blocks from node1. +// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. +// 3. bridge node1 and node2 +// 4. node2 syncs between 10 and MaxSyncBlockHeight blocks from node1. // 5. node2 disconnects from node1 and reboots. // 6. node2 reconnects with node1 and syncs remaining blocks. -// 7. compare node1 db matches node2 db. +// 7. compare node1 db matches node2 db. func TestSimpleSyncRestart(t *testing.T) { require := require.New(t) _ = require @@ -105,14 +106,14 @@ func TestSimpleSyncRestart(t *testing.T) { // TestSimpleSyncDisconnectWithSwitchingToNewPeer tests if a node can successfully restart while syncing blocks, and // then connect to a different node and sync the remaining blocks. -// 1. Spawn three nodes node1, node2, node3 with max block height of MaxSyncBlockHeight blocks. -// 2. node1 and node3 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. -// 3. bridge node1 and node2 -// 4. node2 syncs between 10 and MaxSyncBlockHeight blocks from node1. +// 1. Spawn three nodes node1, node2, node3 with max block height of MaxSyncBlockHeight blocks. +// 2. node1 and node3 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. +// 3. bridge node1 and node2 +// 4. node2 syncs between 10 and MaxSyncBlockHeight blocks from node1. // 5. node2 disconnects from node1 and reboots. // 6. node2 reconnects with node3 and syncs remaining blocks. -// 7. compare node1 state matches node2 state. -// 8. compare node3 state matches node2 state. +// 7. compare node1 state matches node2 state. +// 8. compare node3 state matches node2 state. func TestSimpleSyncDisconnectWithSwitchingToNewPeer(t *testing.T) { require := require.New(t) _ = require diff --git a/integration_testing/hypersync_test.go b/integration_testing/hypersync_test.go index fc0b9bd87..d413eeeb7 100644 --- a/integration_testing/hypersync_test.go +++ b/integration_testing/hypersync_test.go @@ -2,19 +2,20 @@ package integration_testing import ( "fmt" + "os" + "testing" + "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" "github.com/stretchr/testify/require" - "os" - "testing" ) // TestSimpleHyperSync test if a node can successfully hyper sync from another node: -// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod. -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. -// 3. bridge node1 and node2. -// 4. node2 hypersyncs from node1 -// 5. once done, compare node1 state, db, and checksum matches node2. +// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod. +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. +// 3. bridge node1 and node2. +// 4. node2 hypersyncs from node1 +// 5. once done, compare node1 state, db, and checksum matches node2. func TestSimpleHyperSync(t *testing.T) { require := require.New(t) _ = require @@ -58,12 +59,12 @@ func TestSimpleHyperSync(t *testing.T) { } // TestHyperSyncFromHyperSyncedNode test if a node can successfully hypersync from another hypersynced node: -// 1. Spawn three nodes node1, node2, node3 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. -// 3. bridge node1 and node2. -// 4. node2 hypersyncs state. -// 5. once done, bridge node3 and node2 so that node3 hypersyncs from node2. -// 6. compare node1 state, db, and checksum matches node2, and node3. +// 1. Spawn three nodes node1, node2, node3 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. +// 3. bridge node1 and node2. +// 4. node2 hypersyncs state. +// 5. once done, bridge node3 and node2 so that node3 hypersyncs from node2. +// 6. compare node1 state, db, and checksum matches node2, and node3. func TestHyperSyncFromHyperSyncedNode(t *testing.T) { require := require.New(t) _ = require @@ -128,12 +129,12 @@ func TestHyperSyncFromHyperSyncedNode(t *testing.T) { } // TestSimpleHyperSyncRestart test if a node can successfully hyper sync from another node: -// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod. -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. -// 3. bridge node1 and node2. -// 4. node2 hyper syncs a portion of the state from node1 and then restarts. -// 5. node2 reconnects to node1 and hypersyncs again. -// 6. Once node2 finishes sync, compare node1 state, db, and checksum matches node2. +// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks, and snapshot period of HyperSyncSnapshotPeriod. +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator and builds ancestral records. +// 3. bridge node1 and node2. +// 4. node2 hyper syncs a portion of the state from node1 and then restarts. +// 5. node2 reconnects to node1 and hypersyncs again. +// 6. Once node2 finishes sync, compare node1 state, db, and checksum matches node2. func TestSimpleHyperSyncRestart(t *testing.T) { require := require.New(t) _ = require @@ -183,12 +184,12 @@ func TestSimpleHyperSyncRestart(t *testing.T) { } // TestSimpleHyperSyncDisconnectWithSwitchingToNewPeer tests if a node can successfully restart while hypersyncing. -// 1. Spawn three nodes node1, node2, and node3 with max block height of MaxSyncBlockHeight blocks. -// 2. node1, node3 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. -// 3. bridge node1 and node2 -// 4. node2 hypersyncs from node1 but we restart node2 midway. -// 5. after restart, bridge node2 with node3 and resume hypersync. -// 6. once node2 finishes, compare node1, node2, node3 state, db, and checksums are identical. +// 1. Spawn three nodes node1, node2, and node3 with max block height of MaxSyncBlockHeight blocks. +// 2. node1, node3 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator. +// 3. bridge node1 and node2 +// 4. node2 hypersyncs from node1 but we restart node2 midway. +// 5. after restart, bridge node2 with node3 and resume hypersync. +// 6. once node2 finishes, compare node1, node2, node3 state, db, and checksums are identical. func TestSimpleHyperSyncDisconnectWithSwitchingToNewPeer(t *testing.T) { require := require.New(t) _ = require diff --git a/integration_testing/migrations_test.go b/integration_testing/migrations_test.go index b0a692b52..f9a74de19 100644 --- a/integration_testing/migrations_test.go +++ b/integration_testing/migrations_test.go @@ -2,11 +2,12 @@ package integration_testing import ( "fmt" + "os" + "testing" + "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" "github.com/stretchr/testify/require" - "os" - "testing" ) // TODO: Add an encoder migration height in constants.go then modify some diff --git a/integration_testing/mining_test.go b/integration_testing/mining_test.go index 49a23333c..7ec262613 100644 --- a/integration_testing/mining_test.go +++ b/integration_testing/mining_test.go @@ -1,11 +1,12 @@ package integration_testing import ( + "os" + "testing" + "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" "github.com/stretchr/testify/require" - "os" - "testing" ) // TestSimpleBlockSync test if a node can mine blocks on regtest diff --git a/integration_testing/node_test.go b/integration_testing/node_test.go new file mode 100644 index 000000000..8d6c84c68 --- /dev/null +++ b/integration_testing/node_test.go @@ -0,0 +1,293 @@ +package integration_testing + +import ( + "fmt" + "os" + "syscall" + "testing" + "time" + + "github.com/deso-protocol/core/cmd" + + "github.com/stretchr/testify/require" +) + +func TestNodeIsRunning(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + + // expected running status should be false before the node is started + require.False(t, testNode.IsRunning()) + + // Start the node + testNode.Start() + // expected running status should be true after the server is started + require.True(t, testNode.IsRunning()) + + // stop the node + testNode.Stop() + require.False(t, testNode.IsRunning()) + +} + +func TestNodeStatusRunningWithoutLock(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + + // Can change the status to RUNNING from the state NEVERSTARTED + actualErr := testNode.SetStatusRunningWithoutLock() + require.NoError(t, actualErr) + + // Change status from RUNNING to STOPPED + actualErr = testNode.SetStatusStoppedWithoutLock() + require.NoError(t, actualErr) + + // start the server + // Cannot change status to RUNNING while the node is already RUNNING! + testNode.Start() + expErr := cmd.ErrAlreadyStarted + actualErr = testNode.SetStatusRunningWithoutLock() + require.ErrorIs(t, actualErr, expErr) + + // Stop the node + testNode.Stop() + // Should be able to change status to RUNNING from STOP. + actualErr = testNode.SetStatusRunningWithoutLock() + require.NoError(t, actualErr) + // Once the running flag is changed, the isRunning function should return true + require.True(t, testNode.IsRunning()) + +} + +func TestNodeSetStatusStoppedWithoutLock(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + + // Need to call node.start() to atleast once to be able to change node status + // Cannot change status of node which never got initialized in the first place + expErr := cmd.ErrNodeNeverStarted + actualErr := testNode.SetStatusStoppedWithoutLock() + require.ErrorIs(t, actualErr, expErr) + + // start the node + // Should be able to successfully change the status of the node + // Once the server is started + testNode.Start() + + actualErr = testNode.SetStatusStoppedWithoutLock() + require.NoError(t, actualErr) + + // stop the node + testNode.Stop() + + expErr = cmd.ErrAlreadyStopped + actualErr = testNode.SetStatusStoppedWithoutLock() + require.ErrorIs(t, actualErr, expErr) +} + +// Node status is change in the following sequence, +// NEVERSTARTED -> RUNNING -> STOP -> RUNNING +// In each state change it's tested for valid change in status. +func TestNodeSetStatus(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + + // Node is in NEVERSTARTED STATE + + // Changing status from NEVERSTARTED to STOPPED + // This is an invalid status transition. + // Node status cannot needs to transitioned to RUNNING before changing to STOPPED + expError := cmd.ErrNodeNeverStarted + actualError := testNode.SetStatus(cmd.STOPPED) + require.ErrorIs(t, actualError, expError) + + // Cannot set the status to NEVERSTARTED, + // It's the default value before the Node is initialized. + expError = cmd.ErrCannotSetToNeverStarted + actualError = testNode.SetStatus(cmd.NEVERSTARTED) + require.ErrorIs(t, actualError, expError) + // Starting the node. + // The current status of the node is RUNNING. + testNode.Start() + // The status should be changed to RUNNING. + // This successfully tests the transition of status from NEVERSTARTED to RUNNING + expectedStatus := cmd.RUNNING + actualStatus, err := testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + + // Cannot set the status to NEVERSTARTED, + // It's the default value before the Node is initialized. + expError = cmd.ErrCannotSetToNeverStarted + actualError = testNode.SetStatus(cmd.NEVERSTARTED) + require.ErrorIs(t, actualError, expError) + + // Cannot expect the Node status to changed from STOPPED to RUNNING, + // while it's current state is RUNNING + expError = cmd.ErrAlreadyStarted + actualError = testNode.SetStatus(cmd.RUNNING) + require.ErrorIs(t, actualError, expError) + + // Stopping the node. + // This should transition the Node state from RUNNING to STOPPED. + testNode.Stop() + expectedStatus = cmd.STOPPED + actualStatus, err = testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + + // Cannot set the status to NEVERSTARTED, + // It's the default value before the Node is initialized. + expError = cmd.ErrCannotSetToNeverStarted + actualError = testNode.SetStatus(cmd.NEVERSTARTED) + require.ErrorIs(t, actualError, expError) + + // Cannot expect the Node status to changed from NEVERSTARTED to STOPPED, + // while it's current state is STOPPED + expError = cmd.ErrAlreadyStopped + actualError = testNode.SetStatus(cmd.STOPPED) + require.ErrorIs(t, actualError, expError) + + // Changing status from STOPPED to RUNNING + testNode.Start() + // The following tests validates a successful transition of state from STOP to RUNNING + expectedStatus = cmd.RUNNING + actualStatus, err = testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + testNode.Stop() + + // Set the Node status to an invalid status code + // protects from the ignorance of adding new status codes to the iota sequence! + expError = cmd.ErrInvalidNodeStatus + actualError = testNode.SetStatus(cmd.NodeStatus(3)) + require.ErrorIs(t, actualError, expError) + + expError = cmd.ErrInvalidNodeStatus + actualError = testNode.SetStatus(cmd.NodeStatus(4)) + require.ErrorIs(t, actualError, expError) + + expError = cmd.ErrInvalidNodeStatus + actualError = testNode.SetStatus(cmd.NodeStatus(5)) + require.ErrorIs(t, actualError, expError) + +} + +// Tests for *Node.GetStatus() +// Loads the status of node after node operations and tests its correctness. +func TestGetStatus(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + + // Status is set to NEVERSTARTED before the node is started. + expectedStatus := cmd.NEVERSTARTED + actualStatus, err := testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + + // Start the node + testNode.Start() + + // The status is expected to be RUNNING once the node is started. + expectedStatus = cmd.RUNNING + actualStatus, err = testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + + // Stop the node. + testNode.Stop() + + // The status is expected to be STOPPED once the node is stopped. + expectedStatus = cmd.STOPPED + actualStatus, err = testNode.GetStatus() + require.NoError(t, err) + require.Equal(t, actualStatus, expectedStatus) + + // set an invalid status + wrongStatus := cmd.NodeStatus(5) + err = testNode.SetStatus(wrongStatus) + require.Error(t, err) + // Commenting this out as I can't set a wrong status + // maybe there's a way to do this to get more coverage, + // but I'd rather have everything in the integration testing file + //// expect error and invalid status code + //expectedStatus = cmd.INVALIDNODESTATUS + //actualStatus, err = testNode.GetStatus() + //require.ErrorIs(t, err, cmd.ErrInvalidNodeStatus) + //require.Equal(t, actualStatus, expectedStatus) +} + +func TestValidateNodeStatus(t *testing.T) { + + inputs := []cmd.NodeStatus{cmd.NEVERSTARTED, cmd.RUNNING, cmd.STOPPED, cmd.NodeStatus(3), cmd.NodeStatus(4)} + errors := []error{nil, nil, nil, cmd.ErrInvalidNodeStatus, cmd.ErrInvalidNodeStatus} + + var err error + for i := 0; i < len(inputs); i++ { + err = cmd.ValidateNodeStatus(inputs[i]) + require.ErrorIs(t, err, errors[i]) + } +} + +// Stop the node and test whether the internalExitChan fires as expected. +func TestNodeStop(t *testing.T) { + testDir := getDirectory(t) + defer os.RemoveAll(testDir) + + testConfig := generateConfig(t, 18000, testDir, 10) + + testConfig.ConnectIPs = []string{"deso-seed-2.io:17000"} + + testNode := cmd.NewNode(testConfig) + testNode.Start() + + // stop the node + go func() { + err := testNode.Stop() + require.NoError(t, err) + }() + + // Test whether the node stops successfully under three seconds. + select { + case <-testNode.Quit(): + case <-time.After(3 * time.Second): + pid := os.Getpid() + p, err := os.FindProcess(pid) + if err != nil { + panic(err) + } + err = p.Signal(syscall.SIGABRT) + fmt.Println(err) + t.Fatal("timed out waiting for shutdown") + } +} diff --git a/integration_testing/rollback_test.go b/integration_testing/rollback_test.go index 154a392c4..8b6808859 100644 --- a/integration_testing/rollback_test.go +++ b/integration_testing/rollback_test.go @@ -1,12 +1,13 @@ package integration_testing import ( - "github.com/deso-protocol/core/cmd" - "github.com/deso-protocol/core/lib" - "github.com/stretchr/testify/require" "os" "reflect" "testing" + + "github.com/deso-protocol/core/cmd" + "github.com/deso-protocol/core/lib" + "github.com/stretchr/testify/require" ) // Start blocks to height 5000 and then disconnect diff --git a/integration_testing/tools.go b/integration_testing/tools.go index b393c0e44..3e7c26be4 100644 --- a/integration_testing/tools.go +++ b/integration_testing/tools.go @@ -3,6 +3,13 @@ package integration_testing import ( "encoding/hex" "fmt" + "io/ioutil" + "os" + "reflect" + "sort" + "testing" + "time" + "github.com/btcsuite/btcd/wire" "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" @@ -10,12 +17,6 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "github.com/stretchr/testify/require" - "io/ioutil" - "os" - "reflect" - "sort" - "testing" - "time" ) // This testing suite is the first serious attempt at making a comprehensive functional testing framework for DeSo nodes. @@ -43,7 +44,7 @@ const HyperSyncSnapshotPeriod = 1000 // get a random temporary directory. func getDirectory(t *testing.T) string { require := require.New(t) - dbDir, err := ioutil.TempDir("", "badgerdb") + dbDir, err := ioutil.TempDir("", t.Name()) if err != nil { require.NoError(err) } @@ -353,7 +354,7 @@ func computeNodeStateChecksum(t *testing.T, node *cmd.Node, blockHeight uint64) // Stop the provided node. func shutdownNode(t *testing.T, node *cmd.Node) *cmd.Node { - if !node.IsRunning { + if !node.IsRunning() { t.Fatalf("shutdownNode: can't shutdown, node is already down") } @@ -364,7 +365,7 @@ func shutdownNode(t *testing.T, node *cmd.Node) *cmd.Node { // Start the provided node. func startNode(t *testing.T, node *cmd.Node) *cmd.Node { - if node.IsRunning { + if node.IsRunning() { t.Fatalf("startNode: node is already running") } // Start the node. @@ -372,9 +373,9 @@ func startNode(t *testing.T, node *cmd.Node) *cmd.Node { return node } -// Restart the provided node.A +// Restart the provided node. func restartNode(t *testing.T, node *cmd.Node) *cmd.Node { - if !node.IsRunning { + if !node.IsRunning() { t.Fatalf("shutdownNode: can't restart, node already down") } diff --git a/integration_testing/txindex_test.go b/integration_testing/txindex_test.go index b01f7d3b2..f9bccbbd4 100644 --- a/integration_testing/txindex_test.go +++ b/integration_testing/txindex_test.go @@ -2,19 +2,20 @@ package integration_testing import ( "fmt" + "os" + "testing" + "github.com/deso-protocol/core/cmd" "github.com/deso-protocol/core/lib" "github.com/stretchr/testify/require" - "os" - "testing" ) // TestSimpleTxIndex test if a node can successfully build txindex after block syncing from another node: -// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. -// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator, and builds txindex afterwards. -// 3. bridge node1 and node2 -// 4. node2 syncs MaxSyncBlockHeight blocks from node1, and builds txindex afterwards. -// 5. compare node1 db and txindex matches node2. +// 1. Spawn two nodes node1, node2 with max block height of MaxSyncBlockHeight blocks. +// 2. node1 syncs MaxSyncBlockHeight blocks from the "deso-seed-2.io" generator, and builds txindex afterwards. +// 3. bridge node1 and node2 +// 4. node2 syncs MaxSyncBlockHeight blocks from node1, and builds txindex afterwards. +// 5. compare node1 db and txindex matches node2. func TestSimpleTxIndex(t *testing.T) { require := require.New(t) _ = require