Docs and logging improvements

buildstockbatch/gcp/gcp.py

@@ -672,20 +672,30 @@ def start_batch_job(self, batch_info):
         # Start the job!
         created_job = client.create_job(create_request)
         job_name = created_job.name
-
-        logger.info("Newly created GCP Batch job")
-        logger.info(f"  Job name: {job_name}")
-        logger.info(f"  Job UID: {created_job.uid}")
         job_url = (
             "https://console.cloud.google.com/batch/jobsDetail/regions/"
             f"{self.region}/jobs/{self.job_identifier}/details?project={self.gcp_project}"
         )
+
         logger.info(
-            "Simulation output browser (Cloud Console): "
-            f"https://console.cloud.google.com/storage/browser/{self.gcs_bucket}/{self.gcs_prefix}"
-            "/results/simulation_output"
+            f"""
+╔══════════════════════════════════════════════════════════════════════════════╗
+║ GCP Batch Job for running simulations has started!                           ║
+║                                                                              ║
+║ You may interrupt the script and the job will continue to run, though        ║
+║ post-processing will not be triggered.                                       ║
+╚══════════════════════════════════════════════════════════════════════════════╝
+
+Job name:
+    {job_name}
+Job UID:
+    {created_job.uid}
+🔗 See status at:
+    {job_url}
+Results output browser (Cloud Console):
+    https://console.cloud.google.com/storage/browser/{self.gcs_bucket}/{self.gcs_prefix}/results/simulation_output
+"""
         )
-        logger.info(f"View GCP Batch job at {job_url}")
 
         # Monitor job status while waiting for the job to complete
         n_completed_last_time = 0
@@ -710,7 +720,10 @@ def start_batch_job(self, batch_info):
         logger.info(f"Batch job status: {job_status}")
         logger.info(f"Batch job tasks: {dict(task_counts)}")
         if job_status != "SUCCEEDED":
-            raise RuntimeError(f"Batch job failed. See GCP logs at {job_url}")
+            raise RuntimeError(
+                f"Batch job failed. See GCP logs at {job_url}. "
+                "Re-run this script with --missingonly to retry failed tasks."
+            )
         else:
             task_group = job_info.task_groups[0]
             task_spec = task_group.task_spec
@@ -918,8 +931,10 @@ def start_combine_results_job_on_cloud(self, results_dir, do_timeseries=True):
 ║ You may interrupt the script and the job will continue to run.               ║
 ╚══════════════════════════════════════════════════════════════════════════════╝
 
-🔗 See status at: {self.postprocessing_job_console_url}.
-Results output browser (Cloud Console): https://console.cloud.google.com/storage/browser/{self.gcs_bucket}/{self.gcs_prefix}/results/
+🔗 See status at:
+    {self.postprocessing_job_console_url}
+Results output browser (Cloud Console):
+    https://console.cloud.google.com/storage/browser/{self.gcs_bucket}/{self.gcs_prefix}/results/
 
 Run this script with --clean to clean up the GCP environment after post-processing is complete."""
                 )

docs/installation.rst

@@ -261,10 +261,11 @@ One-time GCP setup that can be shared by multiple users.
 
     * **Option 2**: Terraform
 
-      * Install `Terraform`_
-      * From the buildstockbatch/gcp/ directory, run the following with your chosen GCP project and region.
-        You can optionally specify the names of the storage bucket and artifact registery repository. See
-        `main.tf` for more details.
+      * Follow the :ref:`per-user-gcp` instructions below to install BuildStockBatch and the Google Cloud CLI.
+      * Install `Terraform`_.
+      * From the buildstockbatch/gcp/ directory, run the following with your chosen GCP project and region
+        (e.g. "us-central1"). You can optionally specify the names of the storage bucket and
+        artifact registery repository. See `main.tf` for more details.
 
         ::
 
@@ -283,6 +284,8 @@ One-time GCP setup that can be shared by multiple users.
 .. _Terraform: https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli
 
 
+.. _per-user-gcp:
+
 Per-user setup
 ..............
 One-time setup that each user needs to do on the workstation from which they'll launch and
@@ -293,7 +296,7 @@ manage BuildStockBatch runs.
    above (i.e., create a Python virtual environment, activate the venv, and install buildstockbatch
    to it).
 3. Download/Clone ResStock or ComStock.
-4. Set up GCP authentication
+4. Set up GCP authentication.
 
    * **Option 1**: Create and download a `Service Account Key`_.
 

docs/run_sims.rst

@@ -143,7 +143,8 @@ file, something like this:
       gcs:
         bucket: buildstockbatch
         prefix: national01_run01
-      use_spot: true
+      job_environment:
+        use_spot: true
       batch_array_size: 10000
 
 See :ref:`gcp-config` for details and other optional settings.