Race condition in persistence-state-restart-all

[ckreibich]

This test failed here, because ...

--- /tmp/test-diff.22131.output.nodes.baseline.tmp      2022-10-27 19:58:46.957533766 +0000
+++ /tmp/test-diff.22131.output.nodes.tmp       2022-10-27 19:58:46.949533621 +0000
@@ -17,18 +17,18 @@
         "cluster_role": "LOGGER",
         "mgmt_role": null,
         "port": 5001,
-        "state": "RUNNING"
+        "state": "PENDING"
       },
       "manager": {
         "cluster_role": "MANAGER",
         "mgmt_role": null,
         "port": 5000,
-        "state": "RUNNING"
+        "state": "PENDING"
       },
       "worker": {
         "cluster_role": "WORKER",
         "mgmt_role": null,
-        "state": "RUNNING"
+        "state": "PENDING"
       }
     }
   }

... The nodes output may catch nodes that haven't fully booted. This may affect other tests, too.

[awelzel]

I had one here, too:
https://github.com/zeek/zeek/actions/runs/3340194784/jobs/5530618329

The .diag file from the artifacts has the following:

  1 == File ===============================                                         
  2 {                                                                               
  3   "errors": [],                                                                 
  4   "results": {                                                                  
  5     "instance-1": {                                                             
  6       "controller": {                                                           
  7         "cluster_role": null,                                                   
  8         "mgmt_role": "CONTROLLER",                                              
  9         "port": 2151,                                                           
 10         "state": "RUNNING"                                                      
 11       },                                                                        
 12       "instance-1": {                                                           
 13         "cluster_role": null,                                                   
 14         "mgmt_role": "AGENT",                                                   
 15         "state": "RUNNING"                                                      
 16       },                                                                        
 17       "logger": {                                                               
 18         "cluster_role": "LOGGER",                                               
 19         "mgmt_role": null,                                                      
 20         "port": 5001,                                                           
 21         "state": "RUNNING"                                                      
 22       },                                                                        
 23       "manager": {                                                              
 24         "cluster_role": "MANAGER",                                              
 25         "mgmt_role": null,                                                      
 26         "port": 5000,                                                           
 27         "state": "RUNNING"                                                      
 28       },                                                                        
 29       "worker": {                                                               
 30         "cluster_role": "WORKER",                                               
 31         "mgmt_role": null,                                                      
 32         "state": "PENDING"                                                      
 33       }                                                                         
 34     }                                                                           
 35   }                                                                             
 36 }                                                                               
 37 == Diff ===============================                                         
 38 --- /tmp/test-diff.22136.output.nodes.baseline.tmp      2022-11-02 11:56:10.423187717 +0000
 39 +++ /tmp/test-diff.22136.output.nodes.tmp       2022-11-02 11:56:10.415187602 +0000
 40 @@ -28,7 +28,7 @@                                                               
 41        "worker": {                                                              
 42          "cluster_role": "WORKER",                                              
 43          "mgmt_role": null,                                                     
 44 -        "state": "RUNNING"                                                     
 45 +        "state": "PENDING"                                                     
 46        }                                                                        
 47      }                                                                          
 48    }                                                                            
 49 =======================================