From 9a95aec9593fe9b49d935381ae4db96b31524627 Mon Sep 17 00:00:00 2001 From: Nick Frasser <1693461+nfrasser@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:45:53 -0500 Subject: [PATCH 1/5] docs(tools): notebook fixes to work with updated APIs --- .../3dflex-custom-latent-trajectory.ipynb | 2 +- docs/examples/cryolo.ipynb | 18 +- docs/examples/custom-workflow.ipynb | 168 +++++++++++++++--- docs/examples/hi-res-2d-classes.ipynb | 16 +- docs/examples/recenter-particles.ipynb | 5 +- docs/examples/xml-exposure-groups.ipynb | 13 +- docs/guides/jobs.ipynb | 98 +++++----- 7 files changed, 212 insertions(+), 108 deletions(-) diff --git a/docs/examples/3dflex-custom-latent-trajectory.ipynb b/docs/examples/3dflex-custom-latent-trajectory.ipynb index 03458c97..30efa4c6 100644 --- a/docs/examples/3dflex-custom-latent-trajectory.ipynb +++ b/docs/examples/3dflex-custom-latent-trajectory.ipynb @@ -608,7 +608,7 @@ "# so we need to divide the components_mode fields by two to get the total number of components\n", "num_components = int(len([x for x in particles.fields() if \"components_mode\" in x]) / 2)\n", "\n", - "slot_spec = [{\"dtype\": \"components\", \"prefix\": f\"components_mode_{k}\", \"required\": True} for k in range(num_components)]\n", + "slot_spec = [{\"dtype\": \"components\", \"name\": f\"components_mode_{k}\"} for k in range(num_components)]\n", "job = project.create_external_job(\"W5\", \"Custom Latents\")\n", "job.connect(\"particles\", \"J243\", \"particles\", slots=slot_spec)" ] diff --git a/docs/examples/cryolo.ipynb b/docs/examples/cryolo.ipynb index 0b565748..af135389 100644 --- a/docs/examples/cryolo.ipynb +++ b/docs/examples/cryolo.ipynb @@ -32,16 +32,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem0.sbi:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem0.sbi:40003\n", - "Connection succeeded to CryoSPARC command_rtp at http://cryoem0.sbi:40005\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()\n", "\n", "project = cs.find_project(\"P251\")" @@ -175,12 +173,12 @@ "\n", "for mic in all_micrographs.rows():\n", " source = mic[\"micrograph_blob/path\"]\n", - " target = job.uid + \"/full_data/\" + source.split(\"/\")[-1]\n", + " target = job.uid + \"/full_data/\"\n", " project.symlink(source, target)\n", "\n", "for mic in train_micrographs.rows():\n", " source = mic[\"micrograph_blob/path\"]\n", - " target = job.uid + \"/train_image/\" + source.split(\"/\")[-1]\n", + " target = job.uid + \"/train_image/\"\n", " project.symlink(source, target)" ] }, @@ -293,7 +291,11 @@ "\n", "cryosparc-tools provides a `job.subprocess` function to run arbitrary processes, including `cryolo_*.py` scripts installed in the active conda environment.\n", "\n", - "Use `job.subprocess` to generate a crYOLO configuration file with the `cryolo_gui.py config` command. Specify a box size of 130 for this dataset." + "Use `job.subprocess` to generate a crYOLO configuration file with the `cryolo_gui.py config` command. Specify a box size of 130 for this dataset.\n", + "\n", + "```{note}\n", + "When connecting to a remote CryoSPARC instance, note that `job.subprocess` processes will run on the local machine, not remotely.\n", + "```" ] }, { @@ -368,7 +370,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This creates a `cryolo_model.h5` trained model file in the job directory.\n", + "Open the External job's \"Events\" tab from CryoSPARC's web interface to view crYOLO's output. When the process completes, crYOLO creates a `cryolo_model.h5` trained model file in the job directory.\n", "\n", "## Picking\n", "\n", diff --git a/docs/examples/custom-workflow.ipynb b/docs/examples/custom-workflow.ipynb index 1ef8a5e2..deb87aac 100644 --- a/docs/examples/custom-workflow.ipynb +++ b/docs/examples/custom-workflow.ipynb @@ -42,16 +42,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem0.sbi:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem0.sbi:40003\n", - "Connection succeeded to CryoSPARC command_rtp at http://cryoem0.sbi:40005\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()\n", "\n", "project = cs.find_project(\"P251\")\n", @@ -64,16 +62,123 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Import the movies with an Import Movies job. Note that you may use the `CryoSPARC.get_job_sections` method to inspect available job type keys to use with `Workspace.create_job`." + "Import the movies with an Import Movies job. Note that you may use the `CryoSPARC.print_job_types` method to inspect available job type keys to use with `Workspace.create_job`." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Category | Job | Title | Stability\n", + "====================================================================================================\n", + "import | import_movies | Import Movies | stable \n", + " | import_micrographs | Import Micrographs | stable \n", + " | import_particles | Import Particle Stack | stable \n", + " | import_volumes | Import 3D Volumes | stable \n", + " | import_templates | Import Templates | stable \n", + " | import_result_group | Import Result Group | stable \n", + " | import_beam_shift | Import Beam Shift | stable \n", + "motion_correction | patch_motion_correction_multi | Patch Motion Correction | stable \n", + " | rigid_motion_correction_multi | Full-frame Motion Correction | stable \n", + " | rigid_motion_correction | Full-frame Motion Correction | develop \n", + " | local_motion_correction | Local Motion Correction | stable \n", + " | local_motion_correction_multi | Local Motion Correction | stable \n", + " | motion_correction_motioncor2 | MotionCor2 | beta \n", + " | reference_motion_correction | Reference Based Motion Correction | beta \n", + " | local_applytraj | Apply Trajectories | develop \n", + " | patch_to_local | Patch Motion to Local Motion | develop \n", + " | recenter_trajectories | Recenter Trajectories | develop \n", + "ctf_estimation | patch_ctf_estimation_multi | Patch CTF Estimation | stable \n", + " | patch_ctf_extract | Patch CTF Extraction | stable \n", + " | ctf_estimation | CTF Estimation (CTFFIND4) | stable \n", + "exposure_curation | denoise_train | Micrograph Denoiser | beta \n", + " | curate_exposures_v2 | Manually Curate Exposures | stable \n", + "particle_picking | manual_picker_v2 | Manual Picker | stable \n", + " | blob_picker_gpu | Blob Picker | stable \n", + " | template_picker_gpu | Template Picker | stable \n", + " | filament_tracer_gpu | Filament Tracer | stable \n", + " | auto_blob_picker_gpu | Blob Picker Tuner | stable \n", + " | inspect_picks_v2 | Inspect Particle Picks | stable \n", + " | create_templates | Create Templates | stable \n", + "extraction | extract_micrographs_multi | Extract From Micrographs (GPU) | stable \n", + " | extract_micrographs_cpu_parallel | Extract From Micrographs (CPU) | stable \n", + " | downsample_particles | Downsample Particles | stable \n", + " | restack_particles | Restack Particles | stable \n", + "deep_picker | topaz_train | Topaz Train | stable \n", + " | topaz_cross_validation | Topaz Cross Validation (BETA) | beta \n", + " | topaz_extract | Topaz Extract | stable \n", + " | topaz_denoise | Topaz Denoise | stable \n", + "particle_curation | class_2D_new | 2D Classification (NEW) | stable \n", + " | select_2D | Select 2D Classes | stable \n", + " | reference_select_2D | Reference Based Auto Select 2D | beta \n", + " | reconstruct_2D | Reconstruct 2D Classes | stable \n", + " | rebalance_classes_2D | Rebalance 2D Classes | stable \n", + " | class_probability_filter | Class Probability Filter | stable \n", + " | rebalance_3D | Rebalance Orientations | stable \n", + "reconstruction | homo_abinit | Ab-Initio Reconstruction | stable \n", + "refinement | homo_refine_new | Homogeneous Refinement | stable \n", + " | hetero_refine | Heterogeneous Refinement | stable \n", + " | nonuniform_refine_new | Non-uniform Refinement | stable \n", + " | homo_reconstruct | Homogeneous Reconstruction Only | stable \n", + " | hetero_reconstruct_new | Heterogenous Reconstruction Only | stable \n", + "ctf_refinement | ctf_refine_global | Global CTF Refinement | stable \n", + " | ctf_refine_local | Local CTF Refinement | stable \n", + " | exposure_groups | Exposure Group Utilities | stable \n", + "variability | var_3D | 3D Variability | stable \n", + " | var_3D_disp | 3D Variability Display | stable \n", + " | class_3D | 3D Classification | stable \n", + " | regroup_3D_new | Regroup 3D Classes | stable \n", + " | reference_select_3D | Reference Based Auto Select 3D | beta \n", + " | reorder_3D | Reorder 3D Classes | beta \n", + "flexibility | flex_prep | 3D Flex Data Prep | beta \n", + " | flex_meshprep | 3D Flex Mesh Prep | beta \n", + " | flex_train | 3D Flex Training | beta \n", + " | flex_highres | 3D Flex Reconstruction | beta \n", + " | flex_generate | 3D Flex Generator | beta \n", + "postprocessing | sharpen | Sharpening Tools | stable \n", + " | deepemhancer | DeepEMhancer | stable \n", + " | validation | Validation (FSC) | stable \n", + " | local_resolution | Local Resolution Estimation | stable \n", + " | local_filter | Local Filtering | stable \n", + " | reslog | ResLog Analysis | stable \n", + "local_refinement | new_local_refine | Local Refinement | stable \n", + " | particle_subtract | Particle Subtraction | stable \n", + "helix | helix_refine | Helical Refinement | stable \n", + " | helix_search | Symmetry Search Utility | stable \n", + " | helix_initmodel | Helical Initial Model Utility | develop \n", + " | helix_symmetrize | Apply Helical Symmetry | develop \n", + " | helix_average_power_spectra | Average Power Spectra | stable \n", + "utilities | exposure_sets | Exposure Sets Tool | stable \n", + " | exposure_tools | Exposure Tools | stable \n", + " | generate_thumbs | Generate Micrograph Thumbnails | stable \n", + " | cache_particles | Cache Particles on SSD | stable \n", + " | check_corrupt_particles | Check For Corrupt Particles | stable \n", + " | check_corrupt_micrographs | Check For Corrupt Micrographs | stable \n", + " | particle_sets | Particle Sets Tool | stable \n", + " | reassign_particles_mics | Reassign Particles to Micrographs | stable \n", + " | remove_duplicate_particles | Remove Duplicate Particles | stable \n", + " | sym_expand | Symmetry Expansion | stable \n", + " | volume_tools | Volume Tools | stable \n", + " | volume_alignment_tools | Volume Alignment Tools | stable \n", + " | align_3D_new | Align 3D Maps | stable \n", + " | split_volumes_group | Split Volumes Group | stable \n", + " | orientation_diagnostics | Orientation Diagnostics | stable \n", + "simulations | simulator_gpu | Simulate Data | stable \n", + "instance_testing | instance_launch_test | Test Job Launch | stable \n", + " | worker_ssd_test | Test Worker SSD | stable \n", + " | worker_gpu_test | Test Worker GPUs | stable \n", + " | worker_benchmark | Benchmark | stable \n", + "workflows | extensive_workflow_bench | Extensive Validation | stable \n" + ] + } + ], "source": [ - "job_sections = cs.get_job_sections() # [{'contains': ['import_movies', 'import_micrographs', ...] ... }, ...]\n", + "job_sections = cs.print_job_types()\n", "import_movies_job = workspace.create_job(\n", " \"import_movies\",\n", " params={\n", @@ -104,25 +209,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "Param | Title | Type | Default\n", - "=============================================================================\n", - "accel_kv | Accelerating Voltage (kV) | number | None \n", - "blob_paths | Movies data path | path | None \n", - "cs_mm | Spherical Aberration (mm) | number | None \n", - "defect_path | Defect file path | path | None \n", - "eer_num_fractions | EER Number of Fractions | number | 40 \n", - "eer_upsamp_factor | EER Upsampling Factor | number | 2 \n", - "gainref_flip_x | Flip gain ref & defect file in X? | boolean | False \n", - "gainref_flip_y | Flip gain ref & defect file in Y? | boolean | False \n", - "gainref_path | Gain reference path | path | None \n", - "gainref_rotate_num | Rotate gain ref? | number | 0 \n", - "negative_stain_data | Negative Stain Data | boolean | False \n", - "output_constant_ctf | Output Constant CTF | boolean | False \n", - "override_exp_group_id | Override Exposure Group ID | number | None \n", - "phase_plate_data | Phase Plate Data | boolean | False \n", - "psize_A | Raw pixel size (A) | number | None \n", - "skip_header_check | Skip Header Check | boolean | False \n", - "total_dose_e_per_A2 | Total exposure dose (e/A^2) | number | None \n" + "Param | Title | Type | Default\n", + "=========================================================================================================\n", + "blob_paths | Movies data path | string | None \n", + "gainref_path | Gain reference path | string | None \n", + "defect_path | Defect file path | string | None \n", + "gainref_flip_x | Flip gain ref & defect file in X? | boolean | False \n", + "gainref_flip_y | Flip gain ref & defect file in Y? | boolean | False \n", + "gainref_rotate_num | Rotate gain ref? | integer | 0 \n", + "psize_A | Pixel size (A) | number | None \n", + "accel_kv | Accelerating Voltage (kV) | number | None \n", + "cs_mm | Spherical Aberration (mm) | number | None \n", + "total_dose_e_per_A2 | Total exposure dose (e/A^2) | number | None \n", + "negative_stain_data | Negative Stain Data | boolean | False \n", + "phase_plate_data | Phase Plate Data | boolean | False \n", + "override_exp_group_id | Override Exposure Group ID | integer | None \n", + "skip_header_check | Skip Header Check | boolean | True \n", + "output_constant_ctf | Output Constant CTF | boolean | False \n", + "eer_num_fractions | EER Number of Fractions | integer | 40 \n", + "eer_upsamp_factor | EER Upsampling Factor | number | 2 \n", + "parse_xml_files | Import Beam Shift Values from XML Files | boolean | False \n", + "xml_paths | EPU XML metadata path | string | None \n", + "mov_cut_prefix_xml | Length of input filename prefix to cut for XML correspondence | integer | None \n", + "mov_cut_suffix_xml | Length of input filename suffix to cut for XML correspondence | integer | None \n", + "xml_cut_prefix_xml | Length of XML filename prefix to cut for input correspondence | integer | None \n", + "xml_cut_suffix_xml | Length of XML filename suffix to cut for input correspondence | integer | 4 \n", + "compute_num_cpus | Number of CPUs to parallelize during header check | integer | 4 \n" ] } ], @@ -155,7 +267,7 @@ } ], "source": [ - "import_movies_job.set_param(\"skip_header_check\", True)" + "import_movies_job.set_param(\"skip_header_check\", False)" ] }, { @@ -489,7 +601,7 @@ ")\n", "\n", "classify_blob_picks_job = workspace.create_job(\n", - " \"class_2D\",\n", + " \"class_2D_new\",\n", " connections={\"particles\": (extract_blob_picks_job.uid, \"particles\")},\n", " params={\"class2D_K\": 10},\n", ")\n", diff --git a/docs/examples/hi-res-2d-classes.ipynb b/docs/examples/hi-res-2d-classes.ipynb index 85289200..c0509791 100644 --- a/docs/examples/hi-res-2d-classes.ipynb +++ b/docs/examples/hi-res-2d-classes.ipynb @@ -13,23 +13,21 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem0.sbi:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem0.sbi:40003\n", - "Connection succeeded to CryoSPARC command_rtp at http://cryoem0.sbi:40005\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()" ] }, @@ -55,7 +53,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This job has 10 selected templates. Each template is stored in a MRC file created by the 2D Classification job. The `templates_selected` contains the location of these. Load all the unique paths (organized by path in a Python dictionary).\n" + "This job has 24 selected templates. Each template is stored in a MRC file created by the 2D Classification job. The `templates_selected` contains the location of these. Load all the unique paths (organized by path in a Python dictionary).\n" ] }, { @@ -100,17 +98,17 @@ "\n", "N = templates_selected[\"blob/shape\"][0][0]\n", "scale = 100 / templates_selected[\"blob/psize_A\"][0] # 100 Å in pixels\n", - "fig, axes = plt.subplots(3, 5, figsize=(5, 3), dpi=400)\n", + "fig, axes = plt.subplots(3, 8, figsize=(8, 3), dpi=400)\n", "plt.margins(x=0, y=0)\n", "\n", "for i, template in enumerate(templates_selected.rows()):\n", " path = template[\"blob/path\"]\n", " index = template[\"blob/idx\"]\n", " blob = all_templates_blobs[path][index]\n", - " ax = axes[i // 5, i % 5]\n", + " ax = axes[i // 8, i % 8]\n", " ax.axis(\"off\")\n", " ax.imshow(blob, cmap=\"gray\", origin=\"lower\")\n", - " if i % 5 > 0:\n", + " if i % 8 > 0:\n", " continue\n", "\n", " # Plot scale bar\n", diff --git a/docs/examples/recenter-particles.ipynb b/docs/examples/recenter-particles.ipynb index c719ed4a..0897d406 100644 --- a/docs/examples/recenter-particles.ipynb +++ b/docs/examples/recenter-particles.ipynb @@ -28,15 +28,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem5:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem5:40003\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem5\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()" ] }, diff --git a/docs/examples/xml-exposure-groups.ipynb b/docs/examples/xml-exposure-groups.ipynb index 7f8470f0..42c622ee 100644 --- a/docs/examples/xml-exposure-groups.ipynb +++ b/docs/examples/xml-exposure-groups.ipynb @@ -20,15 +20,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem5:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem5:40003\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem5\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()\n", "\n", "project = cs.find_project(\"P251\")" @@ -137,7 +136,7 @@ "\n", "from cryosparc.tools import get_exposure_format, get_import_signatures\n", "\n", - "job.start()\n", + "output_datasets = {}\n", "\n", "for i, node in enumerate(doc.getElementsByTagName(\"imageSet\")[:2]):\n", " directory = get_child_value(node, \"directory\")\n", @@ -177,9 +176,11 @@ " dset[\"gain_ref_blob/path\"] = str(gain_path)\n", " dset[\"gain_ref_blob/shape\"] = (image_height, image_width)\n", "\n", - " job.save_output(f\"images_{i}\", dset)\n", + " output_datasets[f\"images_{i}\"] = dset\n", "\n", - "job.stop()" + "with job.run():\n", + " for output_name, dset in output_datasets.items():\n", + " job.save_output(output_name, dset)" ] }, { diff --git a/docs/guides/jobs.ipynb b/docs/guides/jobs.ipynb index 4641f78e..898306e4 100644 --- a/docs/guides/jobs.ipynb +++ b/docs/guides/jobs.ipynb @@ -30,16 +30,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Connection succeeded to CryoSPARC command_core at http://cryoem0.sbi:40002\n", - "Connection succeeded to CryoSPARC command_vis at http://cryoem0.sbi:40003\n", - "Connection succeeded to CryoSPARC command_rtp at http://cryoem0.sbi:40005\n" + "Connection succeeded to CryoSPARC API at http://cryoem0.sbi:61002\n" ] } ], "source": [ "from cryosparc.tools import CryoSPARC\n", "\n", - "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=40000)\n", + "cs = CryoSPARC(host=\"cryoem0.sbi\", base_port=61000)\n", "assert cs.test_connection()" ] }, @@ -66,22 +64,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "Section | Job | Title \n", - "=================================================================================\n", - "extraction | extract_micrographs_multi | Extract From Micrographs (GPU) \n", - " | extract_micrographs_cpu_parallel | Extract From Micrographs (CPU) \n", - " | downsample_particles | Downsample Particles \n", - " | restack_particles | Restack Particles \n", - "refinement | homo_refine_new | Homogeneous Refinement \n", - " | hetero_refine | Heterogeneous Refinement \n", - " | nonuniform_refine_new | Non-uniform Refinement \n", - " | homo_reconstruct | Homogeneous Reconstruction Only \n", - " | hetero_reconstruct | Heterogeneous Reconstruction Only\n" + "Category | Job | Title | Stability\n", + "============================================================================================\n", + "extraction | extract_micrographs_multi | Extract From Micrographs (GPU) | stable \n", + " | extract_micrographs_cpu_parallel | Extract From Micrographs (CPU) | stable \n", + " | downsample_particles | Downsample Particles | stable \n", + " | restack_particles | Restack Particles | stable \n", + "refinement | homo_refine_new | Homogeneous Refinement | stable \n", + " | hetero_refine | Heterogeneous Refinement | stable \n", + " | nonuniform_refine_new | Non-uniform Refinement | stable \n", + " | homo_reconstruct | Homogeneous Reconstruction Only | stable \n", + " | hetero_reconstruct_new | Heterogenous Reconstruction Only | stable \n" ] } ], "source": [ - "cs.print_job_types(section=[\"extraction\", \"refinement\"])" + "cs.print_job_types(category=[\"extraction\", \"refinement\"])" ] }, { @@ -182,16 +180,16 @@ "text": [ "Param | Title | Type | Default\n", "==================================================================================\n", - "bin_size_pix | Fourier-crop to box size (pix) | number | None \n", - "bin_size_pix_small | Second (small) F-crop box size (pix) | number | None \n", - "box_size_pix | Extraction box size (pix) | number | 256 \n", - "compute_num_cores | Number of CPU cores | number | 4 \n", - "flip_x | Flip mic. in x before extract? | boolean | False \n", - "flip_y | Flip mic. in y before extract? | boolean | False \n", - "force_reextract_CTF | Force re-extract CTFs from micrographs | boolean | False \n", - "num_extract | Number of mics to extract | number | None \n", + "compute_num_cores | Number of CPU cores | integer | 4 \n", + "box_size_pix | Extraction box size (pix) | integer | 256 \n", + "bin_size_pix | Fourier-crop to box size (pix) | integer | None \n", + "bin_size_pix_small | Second (small) F-crop box size (pix) | integer | None \n", "output_f16 | Save results in 16-bit floating point | boolean | False \n", + "force_reextract_CTF | Force re-extract CTFs from micrographs | boolean | False \n", "recenter_using_shifts | Recenter using aligned shifts | boolean | True \n", + "num_extract | Number of mics to extract | integer | None \n", + "flip_x | Flip mic. in x before extract? | boolean | False \n", + "flip_y | Flip mic. in y before extract? | boolean | False \n", "scale_const_override | Scale constant (override) | number | None \n" ] } @@ -334,23 +332,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Output | Title | Type | Result Slots | Result Types \n", - "==============================================================================================\n", - "micrographs | Micrographs accepted | exposure | micrograph_blob | micrograph_blob\n", - " | | | ctf | ctf \n", - " | | | mscope_params | mscope_params \n", - " | | | background_blob | stat_blob \n", - " | | | micrograph_thumbnail_blob_1x | thumbnail_blob \n", - " | | | micrograph_thumbnail_blob_2x | thumbnail_blob \n", - " | | | ctf_stats | ctf_stats \n", - " | | | micrograph_blob_non_dw | micrograph_blob\n", - " | | | rigid_motion | motion \n", - " | | | spline_motion | motion \n", - " | | | movie_blob | movie_blob \n", - " | | | gain_ref_blob | gain_ref_blob \n", - "particles | Particles accepted | particle | location | location \n", - " | | | pick_stats | pick_stats \n", - " | | | ctf | ctf \n" + "Output | Title | Type | Result Slots | Result Types | Passthrough?\n", + "=============================================================================================================\n", + "micrographs | Micrographs accepted | exposure | micrograph_blob | micrograph_blob | \n", + " | | | ctf | ctf | ✓ \n", + " | | | ctf_stats | ctf_stats | ✓ \n", + " | | | rigid_motion | motion | ✓ \n", + " | | | spline_motion | motion | ✓ \n", + " | | | mscope_params | mscope_params | ✓ \n", + " | | | background_blob | stat_blob | ✓ \n", + " | | | micrograph_thumbnail_blob_1x | thumbnail_blob | ✓ \n", + " | | | micrograph_thumbnail_blob_2x | thumbnail_blob | ✓ \n", + " | | | micrograph_blob_non_dw | micrograph_blob | ✓ \n", + " | | | micrograph_blob_non_dw_AB | micrograph_blob | ✓ \n", + " | | | movie_blob | movie_blob | ✓ \n", + " | | | gain_ref_blob | gain_ref_blob | ✓ \n", + "particles | Particles accepted | particle | location | location | \n", + " | | | ctf | ctf | ✓ \n", + " | | | pick_stats | pick_stats | ✓ \n" ] } ], @@ -1415,15 +1414,8 @@ { "data": { "text/plain": [ - "{'_id': '6560d183562b2c67c7d35754',\n", - " 'chunkSize': 2096128,\n", - " 'contentType': 'image/png',\n", - " 'filename': 'J96_extracted_coordinates_on_j2motioncorrected009270517818331954156_14sep05c_00024sq_00003hl_00002esframes_patch_aligned_doseweightedmrc.png',\n", - " 'job_uid': 'J96',\n", - " 'length': 867617,\n", - " 'md5': '471ab293b92726043c8277cb6964f70b',\n", - " 'project_uid': 'P251',\n", - " 'uploadDate': '2023-11-24T16:38:27.800000'}" + "('6786df603b5d82f98ab42232',\n", + " 'J478_extracted_coordinates_on_j357motioncorrected005581540094314066188_14sep05c_00024sq_00003hl_00002esframes_patch_aligned_doseweightedmrc.png')" ] }, "execution_count": 31, @@ -1433,7 +1425,7 @@ ], "source": [ "assets = job.list_assets()\n", - "assets[0]" + "assets[0].id, assets[0].filename" ] }, { @@ -1451,7 +1443,7 @@ { "data": { "text/plain": [ - "PosixPath('image.png')" + "'image.png'" ] }, "execution_count": 32, @@ -1460,7 +1452,7 @@ } ], "source": [ - "job.download_asset(assets[0][\"_id\"], \"image.png\")" + "job.download_asset(assets[0].id, \"image.png\")" ] }, { From af4b15bfa1c015ea9eb160b38fada963a1208e5a Mon Sep 17 00:00:00 2001 From: Nick Frasser <1693461+nfrasser@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:49:00 -0500 Subject: [PATCH 2/5] fix(tools): various bugs in new API integration --- cryosparc/controllers/job.py | 13 ++++++++----- cryosparc/tools.py | 12 +++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cryosparc/controllers/job.py b/cryosparc/controllers/job.py index ce7f6e02..613c3c65 100644 --- a/cryosparc/controllers/job.py +++ b/cryosparc/controllers/job.py @@ -988,7 +988,7 @@ def print_param_spec(self): """ headings = ["Param", "Title", "Type", "Default"] rows = [] - for key, details in self.full_spec.params: + for key, details in self.full_spec.params.items(): if details.get("hidden") is True: continue type = (details["anyOf"][0] if "anyOf" in details else details).get("type", "Any") @@ -1062,13 +1062,16 @@ def print_output_spec(self): | | | ctf | ctf """ specs = self.cs.api.jobs.get_output_specs(self.project_uid, self.uid) - headings = ["Output", "Title", "Type", "Result Slots", "Result Types"] + headings = ["Output", "Title", "Type", "Result Slots", "Result Types", "Passthrough?"] rows = [] for key, spec in specs.root.items(): + output = self.model.spec.outputs.root.get(key) + if not output: + warnings.warn(f"No results for input {key}", stacklevel=2) + continue name, title, type = key, spec.title, spec.type - for slot in spec.slots: - slot = as_output_slot(slot) - rows.append([name, title, type, slot.name, slot.dtype]) + for result in output.results: + rows.append([name, title, type, result.name, result.dtype, "✓" if result.passthrough else ""]) name, title, type = "", "", "" # only these print once per group print_table(headings, rows) diff --git a/cryosparc/tools.py b/cryosparc/tools.py index d7eb7e0d..b87994ff 100644 --- a/cryosparc/tools.py +++ b/cryosparc/tools.py @@ -26,7 +26,7 @@ from contextlib import contextmanager from functools import cached_property from hashlib import sha256 -from io import BytesIO +from io import BytesIO, TextIOBase from pathlib import PurePath, PurePosixPath from typing import IO, TYPE_CHECKING, Any, Container, Dict, Iterable, List, Optional, Tuple, Union, get_args @@ -313,7 +313,7 @@ def print_job_types( """ allowed_categories = {category} if isinstance(category, str) else category register = self.job_register - headings = ["Category", "Job", "Title"] + headings = ["Category", "Job", "Title", "Stability"] rows = [] prev_category = None for job_spec in register.specs: @@ -326,7 +326,7 @@ def print_job_types( category = job_spec.category display_category = "" if category == prev_category else category - rows.append([display_category, job_spec.type, job_spec.title]) + rows.append([display_category, job_spec.type, job_spec.title, job_spec.stability]) prev_category = category print_table(headings, rows) @@ -782,13 +782,15 @@ def upload( project_uid (str): Project unique ID, e.g., "P3" target_path (str | Path): Name or path of file to write in project directory. - source (str | bytes | Path | IO | Stream): Local path or file handle to - upload. May also specified as raw bytes. + source (str | bytes | Path | IO | Stream): Local path or file handle + to upload. May also specified as raw bytes. overwrite (bool, optional): If True, overwrite existing files. Defaults to False. """ if isinstance(source, bytes): source = BytesIO(source) + if isinstance(source, TextIOBase): # e.g., open(p, "r") or StringIO() + source = Stream.from_iterator(s.encode() for s in source) if not isinstance(source, Stream): source = Stream.load(source) self.api.projects.upload_file(project_uid, source, path=str(target_path), overwrite=overwrite) From da6b43c1eb5c087e1fd680f4c05d786d186ab7ce Mon Sep 17 00:00:00 2001 From: Nick Frasser <1693461+nfrasser@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:52:15 -0500 Subject: [PATCH 3/5] docs(tools): README and CHANGELOG update --- CHANGELOG.md | 6 ++++++ README.md | 14 +++++--------- docs/intro.md | 1 - 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76293caf..e2c39776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Next - BREAKING: replaced low-level `CryoSPARC.cli`, `CryoSPARC.rtp` and `CryoSPARC.vis` attributes with single unified `CryoSPARC.api` +- BREAKING: When a `job.start()` or `job.run()` is called for an external job, changing the job connections with `job.add_input`, `job.add_output` or `job.connect` will now trigger an error. Please add all inputs and outputs and connect all inputs before running an external job. - BREAKING: `CryoSPARC.download_asset(fileid, target)` no longer accepts a directory target. Must specify a filename. - BREAKING: removed `CryoSPARC.get_job_specs()`. Use `CryoSPARC.job_register` instead - BREAKING: `CryoSPARC.list_assets()` and `Job.list_assets()` return list of models instead of list of dictionaries, accessible with dot-notation @@ -15,10 +16,15 @@ - OLD: `cs.get_targets()[0]['hostname']` - NEW: `cs.get_targets()[0].hostname` - Some top-level target attributes have also been moved into the `.config` attribute +- BREAKING: `CryoSPARC.print_job_types` `section` argument renamed to `category` + - OLD: `cs.print_job_types(section=["extraction", "refinement"])` + - NEW: `cs.print_job_types(category=["extraction", "refinement"])` - BREAKING: Restructured schema for Job models, many `Job.doc` properties have been internally rearranged - Added: `CryoSPARC.job_register` property - Added: `job.load_input()` and `job.load_output()` now accept `"default"`, `"passthrough"` and `"all"` keywords for their `slots` argument - Added: `job.alloc_output()` now accepts `dtype_params` argument for fields with dynamic shapes +- Added: `CryoSPARC.print_job_types` now includes a job stability column +- Added: `Job.print_output_spec` now includes a passthrough indicator column for results - Updated: Improved type definitions - Deprecated: When adding external inputs and outputs, expanded slot definitions now expect `"name"` key instead of `"prefix"`, support for which will be removed in a future release. - OLD: `job.add_input("particle", slots=[{"prefix": "component_mode_1", "dtype": "component", "required": True}])` diff --git a/README.md b/README.md index e4c7b3b4..3852404a 100644 --- a/README.md +++ b/README.md @@ -117,19 +117,16 @@ rm -rf cryosparc/*.so build dist *.egg-info Install dependencies into a new conda environment: ```sh -conda create -n cryosparc-tools-example -c conda-forge \ - python=3 numpy==1.18.5 \ - pyqt=5 libtiff wxPython=4.1.1 adwaita-icon-theme +conda create -n cryosparc-tools-example -c conda-forge python=3 numpy=1.18.5 \ + pyqt=5 libtiff wxPython=4.1.1 adwaita-icon-theme 'setuptools<66' # exclude these dependencies if you don't need cryolo conda activate cryosparc-tools-example -pip install -U pip -pip install nvidia-pyindex matplotlib~=3.4.0 pandas==1.1.4 notebook -pip install "cryolo[c11]" -pip install -e ".[build]" +pip install cryosparc-tools matplotlib~=3.4.0 pandas~=1.1.0 notebook +pip install nvidia-pyindex # exclude last two steps if you don't need cryolo +pip install 'cryolo[c11]' ``` Run the notebook server with the following environment variables: -- `CRYOSPARC_LICENSE_ID` with Structura-issued CryoSPARC license - `CRYOSPARC_EMAIL` with a CryoSPARC user account email - `CRYOSPARC_PASSWORD` with a CryoSPARC user account password @@ -137,7 +134,6 @@ You may also need to include `LD_LIBRARY_PATH` which includes the location of CUDA Toolkit and cuDNN runtime libraries (e.g., `~/miniconda3/envs/tools/lib/python3.8/site-packages/nvidia/*/lib`). ``` -CRYOSPARC_LICENSE_ID="xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxx" \ CRYOSPARC_EMAIL="ali@example.com" \ CRYOSPARC_PASSWORD="password123" \ jupyter notebook diff --git a/docs/intro.md b/docs/intro.md index 13d3d52d..bee7ba51 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -188,7 +188,6 @@ environment variables. To do the same, define `CRYOSPARC_LICENSE`, CryoSPARC license and login credentials: ```sh -CRYOSPARC_LICENSE_ID="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ CRYOSPARC_EMAIL="ali@example.com" \ CRYOSPARC_PASSWORD="password123" \ jupyter notebook --no-browser --ip=0.0.0.0 --port=8888 From 2690f34b6e38e9a3bf1597cb13243ff50322790d Mon Sep 17 00:00:00 2001 From: Nick Frasser <1693461+nfrasser@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:05:54 -0500 Subject: [PATCH 4/5] fix(tools): show x when printing result not passthrough --- cryosparc/controllers/job.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cryosparc/controllers/job.py b/cryosparc/controllers/job.py index 613c3c65..c1613626 100644 --- a/cryosparc/controllers/job.py +++ b/cryosparc/controllers/job.py @@ -1050,16 +1050,16 @@ def print_output_spec(self): >>> job.doc['type'] 'extract_micrographs_multi' >>> job.print_output_spec() - Output | Title | Type | Result Slots | Result Types - ========================================================================================== - micrographs | Micrographs | exposure | micrograph_blob | micrograph_blob - | | | micrograph_blob_non_dw | micrograph_blob - | | | background_blob | stat_blob - | | | ctf | ctf - | | | ctf_stats | ctf_stats - | | | mscope_params | mscope_params - particles | Particles | particle | blob | blob - | | | ctf | ctf + Output | Title | Type | Result Slots | Result Types | Passthrough? + ========================================================================================================= + micrographs | Micrographs | exposure | micrograph_blob | micrograph_blob | ✕ + | | | micrograph_blob_non_dw | micrograph_blob | ✓ + | | | background_blob | stat_blob | ✓ + | | | ctf | ctf | ✓ + | | | ctf_stats | ctf_stats | ✓ + | | | mscope_params | mscope_params | ✓ + particles | Particles | particle | blob | blob | ✕ + | | | ctf | ctf | ✕ """ specs = self.cs.api.jobs.get_output_specs(self.project_uid, self.uid) headings = ["Output", "Title", "Type", "Result Slots", "Result Types", "Passthrough?"] @@ -1071,7 +1071,7 @@ def print_output_spec(self): continue name, title, type = key, spec.title, spec.type for result in output.results: - rows.append([name, title, type, result.name, result.dtype, "✓" if result.passthrough else ""]) + rows.append([name, title, type, result.name, result.dtype, "✓" if result.passthrough else "✕"]) name, title, type = "", "", "" # only these print once per group print_table(headings, rows) From 9746b5911f42cacc8c89fe58c649655f83318366 Mon Sep 17 00:00:00 2001 From: Nick Frasser <1693461+nfrasser@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:07:44 -0500 Subject: [PATCH 5/5] docs(tools): update jobs notebook example output --- docs/guides/jobs.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guides/jobs.ipynb b/docs/guides/jobs.ipynb index 898306e4..7294cc4e 100644 --- a/docs/guides/jobs.ipynb +++ b/docs/guides/jobs.ipynb @@ -334,7 +334,7 @@ "text": [ "Output | Title | Type | Result Slots | Result Types | Passthrough?\n", "=============================================================================================================\n", - "micrographs | Micrographs accepted | exposure | micrograph_blob | micrograph_blob | \n", + "micrographs | Micrographs accepted | exposure | micrograph_blob | micrograph_blob | ✕ \n", " | | | ctf | ctf | ✓ \n", " | | | ctf_stats | ctf_stats | ✓ \n", " | | | rigid_motion | motion | ✓ \n", @@ -347,7 +347,7 @@ " | | | micrograph_blob_non_dw_AB | micrograph_blob | ✓ \n", " | | | movie_blob | movie_blob | ✓ \n", " | | | gain_ref_blob | gain_ref_blob | ✓ \n", - "particles | Particles accepted | particle | location | location | \n", + "particles | Particles accepted | particle | location | location | ✕ \n", " | | | ctf | ctf | ✓ \n", " | | | pick_stats | pick_stats | ✓ \n" ]