From 1eb494c9f4ccaf3d883d2c73e78364c494e29264 Mon Sep 17 00:00:00 2001 From: RobJY Date: Wed, 26 Feb 2025 19:27:34 +0000 Subject: [PATCH] =?UTF-8?q?Deploy=20preview=20for=20PR=2065=20=F0=9F=9B=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pr-preview/pr-65/404.html | 2 +- .../assets/js/{0d7929ce.bb190655.js => 0d7929ce.5637b530.js} | 2 +- .../assets/js/{f8428497.560c841d.js => f8428497.bdfa7c22.js} | 2 +- .../js/{runtime~main.764fde0a.js => runtime~main.d0057ec0.js} | 2 +- pr-preview/pr-65/blog/archive/index.html | 2 +- .../blog/authors/all-sebastien-lorber-articles/index.html | 2 +- pr-preview/pr-65/blog/authors/index.html | 2 +- pr-preview/pr-65/blog/authors/yangshun/index.html | 2 +- pr-preview/pr-65/blog/index.html | 2 +- pr-preview/pr-65/blog/mdx-blog-post/index.html | 2 +- pr-preview/pr-65/blog/tags/docusaurus/index.html | 2 +- pr-preview/pr-65/blog/tags/index.html | 2 +- pr-preview/pr-65/docs/genai/intro/index.html | 2 +- .../docs/hpc/connecting_to_hpc/connecting_to_hpc/index.html | 2 +- pr-preview/pr-65/docs/hpc/connecting_to_hpc/ood/index.html | 2 +- .../ssh_tunneling_and_x11_forwarding/index.html | 2 +- pr-preview/pr-65/docs/hpc/containers/containers/index.html | 4 ++-- pr-preview/pr-65/docs/hpc/containers/intro/index.html | 2 +- .../docs/hpc/containers/singularity_with_conda/index.html | 2 +- .../containers/squash_file_system_and_singularity/index.html | 2 +- pr-preview/pr-65/docs/hpc/datasets/intro/index.html | 2 +- .../datasets/squash_file_system_and_singularity/index.html | 2 +- .../getting_and_renewing_an_account/index.html | 2 +- .../hpc_accounts_external_collaborators/index.html | 2 +- pr-preview/pr-65/docs/hpc/getting_started/intro/index.html | 2 +- .../walkthrough_approve_hpc_account_request/index.html | 2 +- .../walkthrough_renew_hpc_account_iiq/index.html | 2 +- pr-preview/pr-65/docs/hpc/ml_ai_hpc/intro/index.html | 2 +- .../pr-65/docs/hpc/ml_ai_hpc/llm_fine_tuning/index.html | 2 +- pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_on_hpc/index.html | 2 +- pr-preview/pr-65/docs/hpc/ml_ai_hpc/pytorch_on_hpc/index.html | 2 +- pr-preview/pr-65/docs/hpc/ood/open_on_demand/index.html | 2 +- pr-preview/pr-65/docs/hpc/spec_sheet/index.html | 2 +- .../docs/hpc/storage/available_storage_systems/index.html | 2 +- pr-preview/pr-65/docs/hpc/storage/best_practices/index.html | 2 +- pr-preview/pr-65/docs/hpc/storage/data_management/index.html | 4 ++-- pr-preview/pr-65/docs/hpc/storage/data_transfers/index.html | 2 +- pr-preview/pr-65/docs/hpc/storage/globus/index.html | 2 +- pr-preview/pr-65/docs/hpc/storage/intro/index.html | 2 +- .../docs/hpc/storage/large_number_of_small_files/index.html | 2 +- .../pr-65/docs/hpc/storage/research_project_space/index.html | 2 +- .../pr-65/docs/hpc/storage/sharing_data_on_hpc/index.html | 2 +- pr-preview/pr-65/docs/hpc/storage/system_status/index.html | 2 +- .../transferring_cloud_storage_data_with_rclone/index.html | 2 +- .../docs/hpc/submitting_jobs/slurm_main_commands/index.html | 2 +- .../docs/hpc/submitting_jobs/slurm_submitting_jobs/index.html | 2 +- .../pr-65/docs/hpc/submitting_jobs/slurm_tutorial/index.html | 2 +- pr-preview/pr-65/docs/hpc/system_status/index.html | 2 +- .../docs/hpc/tools_and_software/conda_environments/index.html | 2 +- pr-preview/pr-65/docs/hpc/tools_and_software/intro/index.html | 2 +- .../python_packages_with_virtual_environments/index.html | 2 +- .../hpc/tools_and_software/r_packages_with_renv/index.html | 2 +- .../index.html | 2 +- .../docs/hpc/tools_and_software/software_on_greene/index.html | 2 +- .../pr-65/docs/hpc/training_and_support/support/index.html | 2 +- .../docs/hpc/training_and_support/training_at_nyu/index.html | 2 +- .../docs/hpc/training_and_support/training_outside/index.html | 2 +- .../pr-65/docs/hpc/tutorials/hpc_foundations/index.html | 2 +- pr-preview/pr-65/docs/hpc/tutorials/linux_tutorial/index.html | 2 +- pr-preview/pr-65/docs/hsrn/intro/index.html | 2 +- pr-preview/pr-65/docs/rtc/intro/index.html | 2 +- pr-preview/pr-65/docs/srde/dlp/dlp/index.html | 2 +- pr-preview/pr-65/docs/srde/faq/basics/index.html | 2 +- pr-preview/pr-65/docs/srde/faq/env_roles/index.html | 2 +- pr-preview/pr-65/docs/srde/faq/using_srde/index.html | 2 +- .../docs/srde/getting_started/eligibility_accounts/index.html | 2 +- pr-preview/pr-65/docs/srde/getting_started/intro/index.html | 2 +- pr-preview/pr-65/docs/srde/support/support/index.html | 2 +- .../pr-65/docs/srde/user_guide/best_practices/index.html | 2 +- pr-preview/pr-65/docs/srde/user_guide/connecting/index.html | 2 +- pr-preview/pr-65/docs/srde/user_guide/data_access/index.html | 2 +- .../pr-65/docs/srde/user_guide/data_transfers/index.html | 2 +- .../pr-65/docs/srde/user_guide/troubleshooting/index.html | 2 +- pr-preview/pr-65/index.html | 2 +- pr-preview/pr-65/markdown-page/index.html | 2 +- 75 files changed, 77 insertions(+), 77 deletions(-) rename pr-preview/pr-65/assets/js/{0d7929ce.bb190655.js => 0d7929ce.5637b530.js} (59%) rename pr-preview/pr-65/assets/js/{f8428497.560c841d.js => f8428497.bdfa7c22.js} (91%) rename pr-preview/pr-65/assets/js/{runtime~main.764fde0a.js => runtime~main.d0057ec0.js} (98%) diff --git a/pr-preview/pr-65/404.html b/pr-preview/pr-65/404.html index be673c3c..131469e1 100644 --- a/pr-preview/pr-65/404.html +++ b/pr-preview/pr-65/404.html @@ -5,7 +5,7 @@ Page Not Found | Research Technology Servcies - + diff --git a/pr-preview/pr-65/assets/js/0d7929ce.bb190655.js b/pr-preview/pr-65/assets/js/0d7929ce.5637b530.js similarity index 59% rename from pr-preview/pr-65/assets/js/0d7929ce.bb190655.js rename to pr-preview/pr-65/assets/js/0d7929ce.5637b530.js index 28e08c01..47115ce3 100644 --- a/pr-preview/pr-65/assets/js/0d7929ce.bb190655.js +++ b/pr-preview/pr-65/assets/js/0d7929ce.5637b530.js @@ -1 +1 @@ -"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[2540],{3866:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>a,contentTitle:()=>c,default:()=>p,frontMatter:()=>i,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"hpc/containers/containers","title":"Using Containers on HPC","description":"","source":"@site/docs/hpc/07_containers/02_containers.md","sourceDirName":"hpc/07_containers","slug":"/hpc/containers/containers","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/containers","draft":false,"unlisted":false,"editUrl":"https://github.com/NYU-ITS/rts-docs-dev/blob/main/docs/hpc/07_containers/02_containers.md","tags":[],"version":"current","sidebarPosition":2,"frontMatter":{},"sidebar":"hpcSidebar","previous":{"title":"Apptainer/Singularity","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/intro"},"next":{"title":"Singularity with Conda","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda"}}');var s=t(5105),o=t(3881);const i={},c="Using Containers on HPC",a={},d=[];function l(e){const n={h1:"h1",header:"header",...(0,o.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"using-containers-on-hpc",children:"Using Containers on HPC"})}),"\n",(0,s.jsx)("div",{children:(0,s.jsx)("iframe",{src:"https://docs.google.com/presentation/d/e/2PACX-1vRqcEPcx3N3tZtHlemuRoktVhRT-b3MQyVi8S7SJkzbCKWEt45hF3oeMyB6WHZA9WJsjeHcpsD_I_ao/embed?start=false&loop=false&delayms=3000",frameborder:"0",width:"960",height:"569",allowfullscreen:"true",mozallowfullscreen:"true",webkitallowfullscreen:"true"})})]})}function p(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(l,{...e})}):l(e)}},3881:(e,n,t)=>{t.d(n,{R:()=>i,x:()=>c});var r=t(8101);const s={},o=r.createContext(s);function i(e){const n=r.useContext(o);return r.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function c(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:i(e.components),r.createElement(o.Provider,{value:n},e.children)}}}]); \ No newline at end of file +"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[2540],{3866:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>a,contentTitle:()=>c,default:()=>p,frontMatter:()=>i,metadata:()=>r,toc:()=>d});const r=JSON.parse('{"id":"hpc/containers/containers","title":"Using Containers on HPC","description":"","source":"@site/docs/hpc/07_containers/02_containers.md","sourceDirName":"hpc/07_containers","slug":"/hpc/containers/containers","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/containers","draft":false,"unlisted":false,"editUrl":"https://github.com/NYU-ITS/rts-docs-dev/blob/main/docs/hpc/07_containers/02_containers.md","tags":[],"version":"current","sidebarPosition":2,"frontMatter":{},"sidebar":"hpcSidebar","previous":{"title":"Apptainer/Singularity","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/intro"},"next":{"title":"Singularity with Conda","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda"}}');var s=t(5105),o=t(3881);const i={},c="Using Containers on HPC",a={},d=[];function l(e){const n={h1:"h1",header:"header",...(0,o.R)(),...e.components};return(0,s.jsxs)(s.Fragment,{children:[(0,s.jsx)(n.header,{children:(0,s.jsx)(n.h1,{id:"using-containers-on-hpc",children:"Using Containers on HPC"})}),"\n",(0,s.jsx)("iframe",{src:"https://docs.google.com/presentation/d/e/2PACX-1vRqcEPcx3N3tZtHlemuRoktVhRT-b3MQyVi8S7SJkzbCKWEt45hF3oeMyB6WHZA9WJsjeHcpsD_I_ao/embed?start=false&loop=false&delayms=3000",frameborder:"0",width:"960",height:"569",allowfullscreen:"true",mozallowfullscreen:"true",webkitallowfullscreen:"true"})]})}function p(e={}){const{wrapper:n}={...(0,o.R)(),...e.components};return n?(0,s.jsx)(n,{...e,children:(0,s.jsx)(l,{...e})}):l(e)}},3881:(e,n,t)=>{t.d(n,{R:()=>i,x:()=>c});var r=t(8101);const s={},o=r.createContext(s);function i(e){const n=r.useContext(o);return r.useMemo((function(){return"function"==typeof e?e(n):{...n,...e}}),[n,e])}function c(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(s):e.components||s:i(e.components),r.createElement(o.Provider,{value:n},e.children)}}}]); \ No newline at end of file diff --git a/pr-preview/pr-65/assets/js/f8428497.560c841d.js b/pr-preview/pr-65/assets/js/f8428497.bdfa7c22.js similarity index 91% rename from pr-preview/pr-65/assets/js/f8428497.560c841d.js rename to pr-preview/pr-65/assets/js/f8428497.bdfa7c22.js index 12b86e60..7906002b 100644 --- a/pr-preview/pr-65/assets/js/f8428497.560c841d.js +++ b/pr-preview/pr-65/assets/js/f8428497.bdfa7c22.js @@ -1 +1 @@ -"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[9359],{7404:(e,s,r)=>{r.r(s),r.d(s,{assets:()=>c,contentTitle:()=>o,default:()=>h,frontMatter:()=>i,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"hpc/storage/data_management","title":"Data Management","description":"Introduction to HPC Data Management","source":"@site/docs/hpc/03_storage/06_data_management.md","sourceDirName":"hpc/03_storage","slug":"/hpc/storage/data_management","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/data_management","draft":false,"unlisted":false,"editUrl":"https://github.com/NYU-ITS/rts-docs-dev/blob/main/docs/hpc/03_storage/06_data_management.md","tags":[],"version":"current","sidebarPosition":6,"frontMatter":{},"sidebar":"hpcSidebar","previous":{"title":"Research Project Space (RPS)","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space"},"next":{"title":"Best Practices on HPC Storage","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices"}}');var n=r(5105),a=r(3881);const i={},o="Data Management",c={},d=[{value:"Introduction to HPC Data Management",id:"introduction-to-hpc-data-management",level:2},{value:"Data Security Warning",id:"data-security-warning",level:2},{value:"Moderate Risk Data - HPC Approved",id:"moderate-risk-data---hpc-approved",level:3},{value:"High Risk Data - Secure Research Data Environments (SRDE) Approved",id:"high-risk-data---secure-research-data-environments-srde-approved",level:3},{value:"Data Storage options in the HPC Environment",id:"data-storage-options-in-the-hpc-environment",level:2},{value:"User Home Directories",id:"user-home-directories",level:3},{value:"HPC Scratch",id:"hpc-scratch",level:3},{value:"HPC Vast",id:"hpc-vast",level:3},{value:"HPC Research Project Space",id:"hpc-research-project-space",level:3},{value:"HPC Work",id:"hpc-work",level:3},{value:"HPC Archive",id:"hpc-archive",level:3},{value:"NYU (Google) Drive",id:"nyu-google-drive",level:3},{value:"HPC Storage Mounts Comparison Table",id:"hpc-storage-mounts-comparison-table",level:3},{value:"Data Management Subpages",id:"data-management-subpages",level:2},{value:"Data Transfers",id:"data-transfers",level:3},{value:"Sharing Data on HPC",id:"sharing-data-on-hpc",level:3},{value:"Research Project Space",id:"research-project-space",level:3},{value:"Squash File System and Singularity",id:"squash-file-system-and-singularity",level:3}];function l(e){const s={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(s.header,{children:(0,n.jsx)(s.h1,{id:"data-management",children:"Data Management"})}),"\n",(0,n.jsx)(s.h2,{id:"introduction-to-hpc-data-management",children:"Introduction to HPC Data Management"}),"\n",(0,n.jsxs)(s.p,{children:["The NYU HPC Environment provides access to a number of ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"file systems"})})," to better serve the needs of researchers managing data during the various stages of the research data lifecycle (data capture, analysis, archiving, etc.). Each HPC file system comes with different features, policies, and availability."]}),"\n",(0,n.jsxs)(s.p,{children:["In addition, a number of ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"data management tools"})})," are available that enable data transfers and data sharing, recommended best practices, and various scenarios and use cases of managing data in the HPC Environment."]}),"\n",(0,n.jsxs)(s.p,{children:["Multiple ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"public data sets"})})," are available to all users of the HPC environment, such as a subset of The Cancer Genome Atlas (TCGA), the Million Song Database, ImageNet, and Reference Genomes."]}),"\n",(0,n.jsxs)(s.p,{children:["Below is a list of file systems with their characteristics and a summary table. Reviewing the list of available file systems and the various Scenarios/Use cases that are presented below, can help select the right file systems for a research project. As always, if you have any questions about data storage in the HPC environment, you can request a consultation with the HPC team by sending email to ",(0,n.jsx)(s.a,{href:"mailto:hpc@nyu.edu",children:"hpc@nyu.edu"}),"."]}),"\n",(0,n.jsx)(s.h2,{id:"data-security-warning",children:"Data Security Warning"}),"\n",(0,n.jsx)(s.h3,{id:"moderate-risk-data---hpc-approved",children:"Moderate Risk Data - HPC Approved"}),"\n",(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["The HPC Environment has been approved for storing and analyzing ",(0,n.jsx)(s.strong,{children:"Moderate Risk research data"}),", as defined in the ",(0,n.jsx)(s.a,{href:"https://www.nyu.edu/about/policies-guidelines-compliance/policies-and-guidelines/electronic-data-and-system-risk-classification.html",children:"NYU Electronic Data and System Risk Classification Policy"}),"."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"High Risk"})," research data, such as those that include Personal Identifiable Information (",(0,n.jsx)(s.strong,{children:"PII"}),") or electronic Protected Health Information (",(0,n.jsx)(s.strong,{children:"ePHI"}),") or Controlled Unclassified Information (",(0,n.jsx)(s.strong,{children:"CUI"}),") ",(0,n.jsx)(s.strong,{children:"should NOT be stored in the HPC Environment"}),"."]}),"\n",(0,n.jsx)(s.li,{children:"Please note that only the Office of Sponsored Projects (OSP) and Global Office of Information Security (GOIS) are empowered to classify the risk categories of data."}),"\n"]}),"\n",(0,n.jsx)(s.h3,{id:"high-risk-data---secure-research-data-environments-srde-approved",children:"High Risk Data - Secure Research Data Environments (SRDE) Approved"}),"\n",(0,n.jsxs)(s.p,{children:["Because the HPC system is not approved for High Risk data, we recommend using an approved system like the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/srde/getting_started/intro",children:"Secure Research Data Environments (SRDE)"}),"."]}),"\n",(0,n.jsx)(s.h2,{id:"data-storage-options-in-the-hpc-environment",children:"Data Storage options in the HPC Environment"}),"\n",(0,n.jsx)(s.h3,{id:"user-home-directories",children:"User Home Directories"}),"\n",(0,n.jsxs)(s.p,{children:["Every individual user has a home directory (under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/home/$USER"})}),", environment variable ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$HOME"})}),") for permanently storing code and important configuration files. Home Directories provide limited storage space (",(0,n.jsx)(s.strong,{children:"50 GB"}),") and inodes (files) ",(0,n.jsx)(s.strong,{children:"30,000"})," per user. Users can check their quota utilization using the ",(0,n.jsx)(s.a,{href:"http://www.info-ren.org/projects/ckp/tech/software/version/myquota.html",children:"myquota"})," command."]}),"\n",(0,n.jsxs)(s.p,{children:["User home directories are backed up daily and old files under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$HOME"})})," are not purged."]}),"\n",(0,n.jsx)(s.p,{children:"The User home directories are available on all HPC clusters (Greene) and on every cluster node (login nodes, compute nodes) as well as and Data Transfer Node (gDTN)."}),"\n",(0,n.jsx)(s.admonition,{type:"note",children:(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Avoid changing file and directory permissions in your home directory to allow other users to access files."})," User Home Directories are not ideal for sharing files and folders with other users. HPC Scratch of Research Project Space (RPS) are better file systems for sharing data."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"One of the common issues that users report regarding their home directories is running out of inodes,"})," i.e. the number of files stored under their home exceeds the inode limit, which by default is set to 30,000 files. This typically occurs when users install software under their home directories, for example, when working with Conda and Julia environments, that involve many small files."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the current space and inode quota utilization and the distribution of files under your home directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Working with Conda environments:"})," To avoid running out of inode limits in home directories, the HPC team recommends ",(0,n.jsx)(s.strong,{children:"setting up conda environments with Singularity overlay images"})]}),"\n"]})}),"\n",(0,n.jsx)(s.h3,{id:"hpc-scratch",children:"HPC Scratch"}),"\n",(0,n.jsxs)(s.p,{children:["The HPC scratch file system is the HPC file system where most of the users store research data needed during the analysis phase of their research projects. The scratch file system provides ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"temporary"})})," storage for datasets needed for running jobs."]}),"\n",(0,n.jsxs)(s.p,{children:["Files stored in the HPC scratch file system are subject to the ",(0,n.jsxs)("ins",{children:[(0,n.jsx)(s.strong,{children:"HPC Scratch old file purging policy:"})," Files on the /scratch file system that have not been accessed for 60 or more days will be purged."]})]}),"\n",(0,n.jsxs)(s.p,{children:["Every user has a dedicated scratch directory (",(0,n.jsx)(s.strong,{children:"/scratch/$USER"}),") with ",(0,n.jsx)(s.strong,{children:"5 TB"})," disk quota and ",(0,n.jsx)(s.strong,{children:"1,000,000 inodes"})," (files) limit per user."]}),"\n",(0,n.jsx)(s.p,{children:"The scratch file system is available on all nodes (compute, login, etc.) on Greene as well as Data Transfer Node (gDTN)."}),"\n",(0,n.jsx)(s.admonition,{type:"note",children:(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["There are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"No Back ups"})})," of the scratch file system. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Files that were deleted accidentally or removed due to storage system failures CAN NOT be recovered."})})]}),"\n",(0,n.jsxs)(s.li,{children:["Since there are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"no back ups of HPC Scratch file system"})}),", users should not put important source code, scripts, libraries, executables in ",(0,n.jsx)(s.code,{children:"/scratch"}),". These important files should be stored in file systems that are backed up, such as ",(0,n.jsx)(s.code,{children:"/home"})," or ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Research Project Space (RPS)"}),". Code can also be stored in a ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"git"})})," repository."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Old file purging policy on HPC Scratch:"})})," ",(0,n.jsxs)("ins",{children:["All files on the HPC Scratch file system that have not been accessed ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"for more than 60 days"})})," will be removed. It is a policy violation to use scripts to change the file access time."]})," Any user found to be violating this policy will have their HPC account locked. A second violation may result in your HPC account being turned off."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the user's current disk space and inode quota utilization and the distribution of files under your scratch directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota Limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:["Once a research project completes, users should archive their important files in the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/intro",children:"HPC Archive file system"}),"."]}),"\n"]})}),"\n",(0,n.jsx)(s.h3,{id:"hpc-vast",children:"HPC Vast"}),"\n",(0,n.jsxs)(s.p,{children:["The HPC Vast all-flash file system is the HPC file system where users store research data needed during the analysis phase of their research projects, particuarly for high I/O data that can bottleneck on the scratch file system. The Vast file system provides ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"temporary"})})," storage for datasets needed for running jobs."]}),"\n",(0,n.jsxs)(s.p,{children:["Files stored in the HPC vast file system are subject to the ",(0,n.jsxs)("ins",{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"HPC Vast old file purging policy:"})})," Files on the ",(0,n.jsx)(s.code,{children:"/vast"})," file system that have not been accessed for ",(0,n.jsx)(s.strong,{children:"60 or more days"})," will be purged."]})]}),"\n",(0,n.jsxs)(s.p,{children:["Every user has a dedicated vast directory (",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/vast/$USER"})}),") with ",(0,n.jsx)(s.strong,{children:"2 TB"})," disk quota and ",(0,n.jsx)(s.strong,{children:"5,000,000 inodes"})," (files) limit per user."]}),"\n",(0,n.jsx)(s.p,{children:"The vast file system is available on all nodes (compute, login, etc.) on Greene as well as Data Transfer Node (gDTN)."}),"\n",(0,n.jsxs)(s.admonition,{type:"note",children:[(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["There are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"No Back ups"})})," of the vastsc file system. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Files that were deleted accidentally or removed due to storage system failures CAN NOT be recovered."})})]}),"\n",(0,n.jsxs)(s.li,{children:["Since there are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"no back ups of HPC Vast file system"})}),", users should not put important source code, scripts, libraries, executables in ",(0,n.jsx)(s.code,{children:"/vast"}),". These important files should be stored in file systems that are backed up, such as ",(0,n.jsx)(s.code,{children:"/home"})," or ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Research Project Space (RPS)"}),". Code can also be stored in a ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"git"})})," repository."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Old file purging policy on HPC Vast:"})})," ",(0,n.jsxs)("ins",{children:["All files on the HPC Vast file system that have not been accessed ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"for more than 60 days will be removed."})})," It is a policy violation to use scripts to change the file access time."]})," Any user found to be violating this policy will have their HPC account locked. A second violation may result in your HPC account being turned off."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the user's current disk space and inode quota utilization and the distribution of files under your vast directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota Limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:["Once a research project completes, users should archive their important files in the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/intro",children:"HPC Archive file system"}),"."]}),"\n"]}),(0,n.jsx)(s.h3,{id:"hpc-research-project-space",children:"HPC Research Project Space"}),(0,n.jsxs)(s.p,{children:["The HPC Research Project Space (RPS) provides data storage space for research projects that is easily shared amongst collaborators, ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"backed up"})}),", and ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"not subject to the old file purging policy"})}),". HPC RPS was introduced to ease data management in the HPC environment and eliminate the need of having to frequently copying files between Scratch and Archive file systems by having all projects files under one area. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"These benefits of the HPC RPS come at a cost"})}),". The cost is determined by the allocated disk space and the number of files (inodes)."]}),(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["For detailed information about RPS see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"HPC Research Project Space"})]}),"\n"]}),(0,n.jsx)(s.h3,{id:"hpc-work",children:"HPC Work"}),(0,n.jsxs)(s.p,{children:["The HPC team makes available a number of public sets that are commonly used in analysis jobs. The data sets are available Read-Only under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/scratch/work/public"})}),"."]}),(0,n.jsx)(s.p,{children:"For some of the datasets users must provide a signed usage agreement before accessing."}),(0,n.jsxs)(s.p,{children:["Public datasets available on the HPC clusters can be viewed on the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/getting_started/intro",children:"Datasets page"}),"."]}),(0,n.jsx)(s.h3,{id:"hpc-archive",children:"HPC Archive"}),(0,n.jsxs)(s.p,{children:["Once the Analysis stage of the research data lifecycle has completed, ",(0,n.jsxs)("ins",{children:[(0,n.jsxs)(s.em,{children:["HPC users should ",(0,n.jsx)(s.strong,{children:"tar"})," their data and code into a single tar.gz file and then copy the file to their archive directory (",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/archive/$USER"})})]}),")."]})," The HPC Archive file system is not accessible by running jobs; it is suitable for long-term data storage. Each user has access to a default disk quota of ",(0,n.jsx)(s.strong,{children:"2TB"})," and ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"20,000 inode (files) limit"})}),". The rather low limit on the number of inodes per user is intentional. The archive file system is available only ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"on login nodes"})})," of Greene. The archive file system is backed up daily."]}),(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["Here is an example ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"tar"})})," command that combines the data in a directory named ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"my_run_dir"})})," under ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$SCRATCH"})})})," and outputs the tar file in the user's ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$ARCHIVE"})})}),":"]}),"\n"]}),(0,n.jsx)(s.pre,{children:(0,n.jsx)(s.code,{className:"language-sh",children:"# to archive `$SCRATCH/my_run_dir`\ntar cvf $ARCHIVE/simulation_01.tar -C $SCRATCH my_run_dir\n"})}),(0,n.jsx)(s.h3,{id:"nyu-google-drive",children:"NYU (Google) Drive"}),(0,n.jsxs)(s.p,{children:["Google Drive (",(0,n.jsx)(s.a,{href:"https://www.nyu.edu/life/information-technology/communication-and-collaboration/document-collaboration-and-sharing/nyu-drive.html",children:"NYU Drive"}),") is accessible from the NYU HPC environment and provides an option to users who wish to archive data or share data with external collaborators who do not have access to the NYU HPC environment."]}),(0,n.jsx)(s.p,{children:"Currently (March 2021) there is no limit on the amount of data a user can store on Google Drive and there is no cost associated with storing data on Google Drive (although we hear rumors that free storage on Google Drive may be ending soon)."}),(0,n.jsx)(s.p,{children:"However, there are limits to the data transfer rate in moving to/from Google Drive. Thus, moving many small files to Google Drive is not going to be efficient."}),(0,n.jsxs)(s.p,{children:["Please read the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone",children:"Instructions on how to use cloud storage within the NYU HPC Environment"}),"."]}),(0,n.jsx)(s.h3,{id:"hpc-storage-mounts-comparison-table",children:"HPC Storage Mounts Comparison Table"}),(0,n.jsx)("div",{children:(0,n.jsx)("iframe",{width:"100%",height:"300em",src:"https://docs.google.com/spreadsheets/d/e/2PACX-1vT-q0rRueYg1Be_gcWSghB-GGFDonP8DaXNnm8Qi036w-Vi_l7CCOav4IPxi1yZy8TSnTRFy7S5dNTJ/pubhtml?widget=true&headers=false"})}),(0,n.jsx)(s.h2,{id:"data-management-subpages",children:"Data Management Subpages"}),(0,n.jsx)(s.h3,{id:"data-transfers",children:"Data Transfers"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/data_transfers",children:"Link to Data Transfers Page"})}),(0,n.jsx)(s.h3,{id:"sharing-data-on-hpc",children:"Sharing Data on HPC"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc",children:"Link to Sharing Data on HPC Page"})}),(0,n.jsx)(s.h3,{id:"research-project-space",children:"Research Project Space"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Link to Research Project Space Page"})}),(0,n.jsx)(s.h3,{id:"squash-file-system-and-singularity",children:"Squash File System and Singularity"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity",children:"Link to Squash File System and Singularity Page"})})]})]})}function h(e={}){const{wrapper:s}={...(0,a.R)(),...e.components};return s?(0,n.jsx)(s,{...e,children:(0,n.jsx)(l,{...e})}):l(e)}},3881:(e,s,r)=>{r.d(s,{R:()=>i,x:()=>o});var t=r(8101);const n={},a=t.createContext(n);function i(e){const s=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(s):{...s,...e}}),[s,e])}function o(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:i(e.components),t.createElement(a.Provider,{value:s},e.children)}}}]); \ No newline at end of file +"use strict";(self.webpackChunkmy_website=self.webpackChunkmy_website||[]).push([[9359],{7404:(e,s,r)=>{r.r(s),r.d(s,{assets:()=>c,contentTitle:()=>o,default:()=>h,frontMatter:()=>i,metadata:()=>t,toc:()=>d});const t=JSON.parse('{"id":"hpc/storage/data_management","title":"Data Management","description":"Introduction to HPC Data Management","source":"@site/docs/hpc/03_storage/06_data_management.md","sourceDirName":"hpc/03_storage","slug":"/hpc/storage/data_management","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/data_management","draft":false,"unlisted":false,"editUrl":"https://github.com/NYU-ITS/rts-docs-dev/blob/main/docs/hpc/03_storage/06_data_management.md","tags":[],"version":"current","sidebarPosition":6,"frontMatter":{},"sidebar":"hpcSidebar","previous":{"title":"Research Project Space (RPS)","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space"},"next":{"title":"Best Practices on HPC Storage","permalink":"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices"}}');var n=r(5105),a=r(3881);const i={},o="Data Management",c={},d=[{value:"Introduction to HPC Data Management",id:"introduction-to-hpc-data-management",level:2},{value:"Data Security Warning",id:"data-security-warning",level:2},{value:"Moderate Risk Data - HPC Approved",id:"moderate-risk-data---hpc-approved",level:3},{value:"High Risk Data - Secure Research Data Environments (SRDE) Approved",id:"high-risk-data---secure-research-data-environments-srde-approved",level:3},{value:"Data Storage options in the HPC Environment",id:"data-storage-options-in-the-hpc-environment",level:2},{value:"User Home Directories",id:"user-home-directories",level:3},{value:"HPC Scratch",id:"hpc-scratch",level:3},{value:"HPC Vast",id:"hpc-vast",level:3},{value:"HPC Research Project Space",id:"hpc-research-project-space",level:3},{value:"HPC Work",id:"hpc-work",level:3},{value:"HPC Archive",id:"hpc-archive",level:3},{value:"NYU (Google) Drive",id:"nyu-google-drive",level:3},{value:"HPC Storage Mounts Comparison Table",id:"hpc-storage-mounts-comparison-table",level:3},{value:"Data Management Subpages",id:"data-management-subpages",level:2},{value:"Data Transfers",id:"data-transfers",level:3},{value:"Sharing Data on HPC",id:"sharing-data-on-hpc",level:3},{value:"Research Project Space",id:"research-project-space",level:3},{value:"Squash File System and Singularity",id:"squash-file-system-and-singularity",level:3}];function l(e){const s={a:"a",admonition:"admonition",code:"code",em:"em",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",p:"p",pre:"pre",strong:"strong",ul:"ul",...(0,a.R)(),...e.components};return(0,n.jsxs)(n.Fragment,{children:[(0,n.jsx)(s.header,{children:(0,n.jsx)(s.h1,{id:"data-management",children:"Data Management"})}),"\n",(0,n.jsx)(s.h2,{id:"introduction-to-hpc-data-management",children:"Introduction to HPC Data Management"}),"\n",(0,n.jsxs)(s.p,{children:["The NYU HPC Environment provides access to a number of ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"file systems"})})," to better serve the needs of researchers managing data during the various stages of the research data lifecycle (data capture, analysis, archiving, etc.). Each HPC file system comes with different features, policies, and availability."]}),"\n",(0,n.jsxs)(s.p,{children:["In addition, a number of ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"data management tools"})})," are available that enable data transfers and data sharing, recommended best practices, and various scenarios and use cases of managing data in the HPC Environment."]}),"\n",(0,n.jsxs)(s.p,{children:["Multiple ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"public data sets"})})," are available to all users of the HPC environment, such as a subset of The Cancer Genome Atlas (TCGA), the Million Song Database, ImageNet, and Reference Genomes."]}),"\n",(0,n.jsxs)(s.p,{children:["Below is a list of file systems with their characteristics and a summary table. Reviewing the list of available file systems and the various Scenarios/Use cases that are presented below, can help select the right file systems for a research project. As always, if you have any questions about data storage in the HPC environment, you can request a consultation with the HPC team by sending email to ",(0,n.jsx)(s.a,{href:"mailto:hpc@nyu.edu",children:"hpc@nyu.edu"}),"."]}),"\n",(0,n.jsx)(s.h2,{id:"data-security-warning",children:"Data Security Warning"}),"\n",(0,n.jsx)(s.h3,{id:"moderate-risk-data---hpc-approved",children:"Moderate Risk Data - HPC Approved"}),"\n",(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["The HPC Environment has been approved for storing and analyzing ",(0,n.jsx)(s.strong,{children:"Moderate Risk research data"}),", as defined in the ",(0,n.jsx)(s.a,{href:"https://www.nyu.edu/about/policies-guidelines-compliance/policies-and-guidelines/electronic-data-and-system-risk-classification.html",children:"NYU Electronic Data and System Risk Classification Policy"}),"."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"High Risk"})," research data, such as those that include Personal Identifiable Information (",(0,n.jsx)(s.strong,{children:"PII"}),") or electronic Protected Health Information (",(0,n.jsx)(s.strong,{children:"ePHI"}),") or Controlled Unclassified Information (",(0,n.jsx)(s.strong,{children:"CUI"}),") ",(0,n.jsx)(s.strong,{children:"should NOT be stored in the HPC Environment"}),"."]}),"\n",(0,n.jsx)(s.li,{children:"Please note that only the Office of Sponsored Projects (OSP) and Global Office of Information Security (GOIS) are empowered to classify the risk categories of data."}),"\n"]}),"\n",(0,n.jsx)(s.h3,{id:"high-risk-data---secure-research-data-environments-srde-approved",children:"High Risk Data - Secure Research Data Environments (SRDE) Approved"}),"\n",(0,n.jsxs)(s.p,{children:["Because the HPC system is not approved for High Risk data, we recommend using an approved system like the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/srde/getting_started/intro",children:"Secure Research Data Environments (SRDE)"}),"."]}),"\n",(0,n.jsx)(s.h2,{id:"data-storage-options-in-the-hpc-environment",children:"Data Storage options in the HPC Environment"}),"\n",(0,n.jsx)(s.h3,{id:"user-home-directories",children:"User Home Directories"}),"\n",(0,n.jsxs)(s.p,{children:["Every individual user has a home directory (under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/home/$USER"})}),", environment variable ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$HOME"})}),") for permanently storing code and important configuration files. Home Directories provide limited storage space (",(0,n.jsx)(s.strong,{children:"50 GB"}),") and inodes (files) ",(0,n.jsx)(s.strong,{children:"30,000"})," per user. Users can check their quota utilization using the ",(0,n.jsx)(s.a,{href:"http://www.info-ren.org/projects/ckp/tech/software/version/myquota.html",children:"myquota"})," command."]}),"\n",(0,n.jsxs)(s.p,{children:["User home directories are backed up daily and old files under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$HOME"})})," are not purged."]}),"\n",(0,n.jsx)(s.p,{children:"The User home directories are available on all HPC clusters (Greene) and on every cluster node (login nodes, compute nodes) as well as and Data Transfer Node (gDTN)."}),"\n",(0,n.jsx)(s.admonition,{type:"note",children:(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Avoid changing file and directory permissions in your home directory to allow other users to access files."})," User Home Directories are not ideal for sharing files and folders with other users. HPC Scratch of Research Project Space (RPS) are better file systems for sharing data."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"One of the common issues that users report regarding their home directories is running out of inodes,"})," i.e. the number of files stored under their home exceeds the inode limit, which by default is set to 30,000 files. This typically occurs when users install software under their home directories, for example, when working with Conda and Julia environments, that involve many small files."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the current space and inode quota utilization and the distribution of files under your home directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.strong,{children:"Working with Conda environments:"})," To avoid running out of inode limits in home directories, the HPC team recommends ",(0,n.jsx)(s.strong,{children:"setting up conda environments with Singularity overlay images"})]}),"\n"]})}),"\n",(0,n.jsx)(s.h3,{id:"hpc-scratch",children:"HPC Scratch"}),"\n",(0,n.jsxs)(s.p,{children:["The HPC scratch file system is the HPC file system where most of the users store research data needed during the analysis phase of their research projects. The scratch file system provides ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"temporary"})})," storage for datasets needed for running jobs."]}),"\n",(0,n.jsxs)(s.p,{children:["Files stored in the HPC scratch file system are subject to the ",(0,n.jsxs)("ins",{children:[(0,n.jsx)(s.strong,{children:"HPC Scratch old file purging policy:"})," Files on the /scratch file system that have not been accessed for 60 or more days will be purged."]})]}),"\n",(0,n.jsxs)(s.p,{children:["Every user has a dedicated scratch directory (",(0,n.jsx)(s.strong,{children:"/scratch/$USER"}),") with ",(0,n.jsx)(s.strong,{children:"5 TB"})," disk quota and ",(0,n.jsx)(s.strong,{children:"1,000,000 inodes"})," (files) limit per user."]}),"\n",(0,n.jsx)(s.p,{children:"The scratch file system is available on all nodes (compute, login, etc.) on Greene as well as Data Transfer Node (gDTN)."}),"\n",(0,n.jsx)(s.admonition,{type:"note",children:(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["There are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"No Back ups"})})," of the scratch file system. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Files that were deleted accidentally or removed due to storage system failures CAN NOT be recovered."})})]}),"\n",(0,n.jsxs)(s.li,{children:["Since there are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"no back ups of HPC Scratch file system"})}),", users should not put important source code, scripts, libraries, executables in ",(0,n.jsx)(s.code,{children:"/scratch"}),". These important files should be stored in file systems that are backed up, such as ",(0,n.jsx)(s.code,{children:"/home"})," or ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Research Project Space (RPS)"}),". Code can also be stored in a ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"git"})})," repository."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Old file purging policy on HPC Scratch:"})})," ",(0,n.jsxs)("ins",{children:["All files on the HPC Scratch file system that have not been accessed ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"for more than 60 days"})})," will be removed. It is a policy violation to use scripts to change the file access time."]})," Any user found to be violating this policy will have their HPC account locked. A second violation may result in your HPC account being turned off."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the user's current disk space and inode quota utilization and the distribution of files under your scratch directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota Limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:["Once a research project completes, users should archive their important files in the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/intro",children:"HPC Archive file system"}),"."]}),"\n"]})}),"\n",(0,n.jsx)(s.h3,{id:"hpc-vast",children:"HPC Vast"}),"\n",(0,n.jsxs)(s.p,{children:["The HPC Vast all-flash file system is the HPC file system where users store research data needed during the analysis phase of their research projects, particuarly for high I/O data that can bottleneck on the scratch file system. The Vast file system provides ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"temporary"})})," storage for datasets needed for running jobs."]}),"\n",(0,n.jsxs)(s.p,{children:["Files stored in the HPC vast file system are subject to the ",(0,n.jsxs)("ins",{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"HPC Vast old file purging policy:"})})," Files on the ",(0,n.jsx)(s.code,{children:"/vast"})," file system that have not been accessed for ",(0,n.jsx)(s.strong,{children:"60 or more days"})," will be purged."]})]}),"\n",(0,n.jsxs)(s.p,{children:["Every user has a dedicated vast directory (",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/vast/$USER"})}),") with ",(0,n.jsx)(s.strong,{children:"2 TB"})," disk quota and ",(0,n.jsx)(s.strong,{children:"5,000,000 inodes"})," (files) limit per user."]}),"\n",(0,n.jsx)(s.p,{children:"The vast file system is available on all nodes (compute, login, etc.) on Greene as well as Data Transfer Node (gDTN)."}),"\n",(0,n.jsxs)(s.admonition,{type:"note",children:[(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["There are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"No Back ups"})})," of the vastsc file system. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Files that were deleted accidentally or removed due to storage system failures CAN NOT be recovered."})})]}),"\n",(0,n.jsxs)(s.li,{children:["Since there are ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"no back ups of HPC Vast file system"})}),", users should not put important source code, scripts, libraries, executables in ",(0,n.jsx)(s.code,{children:"/vast"}),". These important files should be stored in file systems that are backed up, such as ",(0,n.jsx)(s.code,{children:"/home"})," or ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Research Project Space (RPS)"}),". Code can also be stored in a ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"git"})})," repository."]}),"\n",(0,n.jsxs)(s.li,{children:[(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"Old file purging policy on HPC Vast:"})})," ",(0,n.jsxs)("ins",{children:["All files on the HPC Vast file system that have not been accessed ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"for more than 60 days will be removed."})})," It is a policy violation to use scripts to change the file access time."]})," Any user found to be violating this policy will have their HPC account locked. A second violation may result in your HPC account being turned off."]}),"\n",(0,n.jsxs)(s.li,{children:["To find out the user's current disk space and inode quota utilization and the distribution of files under your vast directory, please see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/best_practices#user-quota-limits-and-the-myquota-command",children:"Understanding user quota Limits and the myquota command."})]}),"\n",(0,n.jsxs)(s.li,{children:["Once a research project completes, users should archive their important files in the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/intro",children:"HPC Archive file system"}),"."]}),"\n"]}),(0,n.jsx)(s.h3,{id:"hpc-research-project-space",children:"HPC Research Project Space"}),(0,n.jsxs)(s.p,{children:["The HPC Research Project Space (RPS) provides data storage space for research projects that is easily shared amongst collaborators, ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"backed up"})}),", and ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"not subject to the old file purging policy"})}),". HPC RPS was introduced to ease data management in the HPC environment and eliminate the need of having to frequently copying files between Scratch and Archive file systems by having all projects files under one area. ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"These benefits of the HPC RPS come at a cost"})}),". The cost is determined by the allocated disk space and the number of files (inodes)."]}),(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["For detailed information about RPS see: ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"HPC Research Project Space"})]}),"\n"]}),(0,n.jsx)(s.h3,{id:"hpc-work",children:"HPC Work"}),(0,n.jsxs)(s.p,{children:["The HPC team makes available a number of public sets that are commonly used in analysis jobs. The data sets are available Read-Only under ",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/scratch/work/public"})}),"."]}),(0,n.jsx)(s.p,{children:"For some of the datasets users must provide a signed usage agreement before accessing."}),(0,n.jsxs)(s.p,{children:["Public datasets available on the HPC clusters can be viewed on the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/getting_started/intro",children:"Datasets page"}),"."]}),(0,n.jsx)(s.h3,{id:"hpc-archive",children:"HPC Archive"}),(0,n.jsxs)(s.p,{children:["Once the Analysis stage of the research data lifecycle has completed, ",(0,n.jsxs)("ins",{children:[(0,n.jsxs)(s.em,{children:["HPC users should ",(0,n.jsx)(s.strong,{children:"tar"})," their data and code into a single tar.gz file and then copy the file to their archive directory (",(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"/archive/$USER"})})]}),")."]})," The HPC Archive file system is not accessible by running jobs; it is suitable for long-term data storage. Each user has access to a default disk quota of ",(0,n.jsx)(s.strong,{children:"2TB"})," and ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"20,000 inode (files) limit"})}),". The rather low limit on the number of inodes per user is intentional. The archive file system is available only ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"on login nodes"})})," of Greene. The archive file system is backed up daily."]}),(0,n.jsxs)(s.ul,{children:["\n",(0,n.jsxs)(s.li,{children:["Here is an example ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"tar"})})," command that combines the data in a directory named ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:"my_run_dir"})})," under ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$SCRATCH"})})})," and outputs the tar file in the user's ",(0,n.jsx)(s.em,{children:(0,n.jsx)(s.strong,{children:(0,n.jsx)(s.code,{children:"$ARCHIVE"})})}),":"]}),"\n"]}),(0,n.jsx)(s.pre,{children:(0,n.jsx)(s.code,{className:"language-sh",children:"# to archive `$SCRATCH/my_run_dir`\ntar cvf $ARCHIVE/simulation_01.tar -C $SCRATCH my_run_dir\n"})}),(0,n.jsx)(s.h3,{id:"nyu-google-drive",children:"NYU (Google) Drive"}),(0,n.jsxs)(s.p,{children:["Google Drive (",(0,n.jsx)(s.a,{href:"https://www.nyu.edu/life/information-technology/communication-and-collaboration/document-collaboration-and-sharing/nyu-drive.html",children:"NYU Drive"}),") is accessible from the NYU HPC environment and provides an option to users who wish to archive data or share data with external collaborators who do not have access to the NYU HPC environment."]}),(0,n.jsx)(s.p,{children:"Currently (March 2021) there is no limit on the amount of data a user can store on Google Drive and there is no cost associated with storing data on Google Drive (although we hear rumors that free storage on Google Drive may be ending soon)."}),(0,n.jsx)(s.p,{children:"However, there are limits to the data transfer rate in moving to/from Google Drive. Thus, moving many small files to Google Drive is not going to be efficient."}),(0,n.jsxs)(s.p,{children:["Please read the ",(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone",children:"Instructions on how to use cloud storage within the NYU HPC Environment"}),"."]}),(0,n.jsx)(s.h3,{id:"hpc-storage-mounts-comparison-table",children:"HPC Storage Mounts Comparison Table"}),(0,n.jsx)("iframe",{width:"100%",height:"300em",src:"https://docs.google.com/spreadsheets/d/e/2PACX-1vT-q0rRueYg1Be_gcWSghB-GGFDonP8DaXNnm8Qi036w-Vi_l7CCOav4IPxi1yZy8TSnTRFy7S5dNTJ/pubhtml?widget=true&headers=false"}),(0,n.jsx)(s.h2,{id:"data-management-subpages",children:"Data Management Subpages"}),(0,n.jsx)(s.h3,{id:"data-transfers",children:"Data Transfers"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/data_transfers",children:"Link to Data Transfers Page"})}),(0,n.jsx)(s.h3,{id:"sharing-data-on-hpc",children:"Sharing Data on HPC"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc",children:"Link to Sharing Data on HPC Page"})}),(0,n.jsx)(s.h3,{id:"research-project-space",children:"Research Project Space"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/storage/research_project_space",children:"Link to Research Project Space Page"})}),(0,n.jsx)(s.h3,{id:"squash-file-system-and-singularity",children:"Squash File System and Singularity"}),(0,n.jsx)(s.p,{children:(0,n.jsx)(s.a,{href:"/rts-docs-dev/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity",children:"Link to Squash File System and Singularity Page"})})]})]})}function h(e={}){const{wrapper:s}={...(0,a.R)(),...e.components};return s?(0,n.jsx)(s,{...e,children:(0,n.jsx)(l,{...e})}):l(e)}},3881:(e,s,r)=>{r.d(s,{R:()=>i,x:()=>o});var t=r(8101);const n={},a=t.createContext(n);function i(e){const s=t.useContext(a);return t.useMemo((function(){return"function"==typeof e?e(s):{...s,...e}}),[s,e])}function o(e){let s;return s=e.disableParentContext?"function"==typeof e.components?e.components(n):e.components||n:i(e.components),t.createElement(a.Provider,{value:s},e.children)}}}]); \ No newline at end of file diff --git a/pr-preview/pr-65/assets/js/runtime~main.764fde0a.js b/pr-preview/pr-65/assets/js/runtime~main.d0057ec0.js similarity index 98% rename from pr-preview/pr-65/assets/js/runtime~main.764fde0a.js rename to pr-preview/pr-65/assets/js/runtime~main.d0057ec0.js index 040c03fd..4eeafd2b 100644 --- a/pr-preview/pr-65/assets/js/runtime~main.764fde0a.js +++ b/pr-preview/pr-65/assets/js/runtime~main.d0057ec0.js @@ -1 +1 @@ -(()=>{"use strict";var e,a,f,d,b,c={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var f=t[e]={exports:{}};return c[e].call(f.exports,f,f.exports,r),f.exports}r.m=c,e=[],r.O=(a,f,d,b)=>{if(!f){var c=1/0;for(i=0;i=b)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,b0&&e[i-1][2]>b;i--)e[i]=e[i-1];e[i]=[f,d,b]},r.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return r.d(a,{a:a}),a},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,d){if(1&d&&(e=this(e)),8&d)return e;if("object"==typeof e&&e){if(4&d&&e.__esModule)return e;if(16&d&&"function"==typeof e.then)return e}var b=Object.create(null);r.r(b);var c={};a=a||[null,f({}),f([]),f(f)];for(var t=2&d&&e;"object"==typeof t&&!~a.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((a=>c[a]=()=>e[a]));return c.default=()=>e,r.d(b,c),b},r.d=(e,a)=>{for(var f in a)r.o(a,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:a[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((a,f)=>(r.f[f](e,a),a)),[])),r.u=e=>"assets/js/"+({697:"07dc3999",824:"29e9f0fb",867:"33fc5bb8",941:"709608ee",1045:"3bd1cd05",1083:"02759b6a",1159:"78ec2d69",1196:"ee9fd62d",1206:"38101a41",1235:"a7456010",1271:"6ab77946",1418:"fe973245",1878:"49061c4f",1903:"acecf23e",2006:"e66df417",2057:"6b3f972c",2150:"36504d1f",2256:"40a793ad",2383:"aaf06d06",2540:"0d7929ce",2711:"9e4087bc",2823:"807f69b6",2929:"978f5458",3031:"f1e2e86c",3249:"ccc49370",3259:"001f4272",3576:"d2399e65",3637:"f4f34a3a",3844:"09b877a9",3889:"54dba71d",4086:"7be04b14",4126:"e00b981d",4134:"393be207",4194:"c051bf5e",4212:"621db11d",4307:"9f9dbde3",4395:"2d4abd1b",4400:"ccdcb152",4423:"66a0c2bb",4579:"6e84c0ae",4583:"1df93b7f",4791:"8450c046",4813:"6875c492",5124:"f5934a05",5155:"fd49cdbd",5164:"7365917a",5201:"7b756da4",5292:"aff0a4ba",5656:"fddc7b52",5742:"aba21aa0",5799:"99450296",6038:"475f08fe",6061:"1f391b9e",6062:"fd9d1f1b",6063:"6f4d7c1e",6494:"c7b2753b",6501:"105f05b3",6574:"133bbacd",6848:"24963d3d",6943:"f05d36a5",7006:"5d34ccd1",7098:"a7bd4aaa",7145:"634b6fb7",7148:"1d6b1623",7233:"104bd8b1",7398:"b2ed70dc",7472:"814f3328",7578:"af2e7acf",7643:"a6aa9e1f",7737:"1e53ba9d",8209:"01a85c17",8401:"17896441",8535:"7a70fae9",8633:"8b4c439c",8647:"af96c809",8780:"b3bb8099",8786:"540e98b2",8912:"bc1b0a4f",8932:"89790afe",8987:"47a92b8a",8998:"2975eaac",9048:"a94703ab",9118:"f1d7aa2e",9121:"0b7b4c13",9325:"59362658",9359:"f8428497",9388:"efd0c195",9647:"5e95c892",9858:"36994c47",9989:"7e921b28"}[e]||e)+"."+{697:"0cd3787d",824:"00443aae",867:"49849b3a",941:"7ab073b3",1045:"dbe33a14",1083:"bc8047c4",1159:"9f6205d7",1196:"a13f182a",1206:"5a25eb89",1235:"2f05987d",1271:"243754cd",1418:"e899bdae",1467:"70871107",1878:"59b14a5a",1903:"2399748f",2006:"b6f5f9ff",2057:"c4a961c1",2150:"8d53a329",2213:"48817a72",2256:"a275bcad",2383:"899b44ba",2540:"bb190655",2711:"5813f263",2823:"6fa767fc",2929:"d9a84e4b",3031:"8056b6e7",3249:"22472b40",3259:"6951373b",3403:"5b912781",3576:"36e951b0",3637:"2321e716",3844:"88eb5d7f",3889:"b8b55320",4086:"54feaa63",4126:"26f9b193",4134:"b503e730",4194:"2052f5fa",4212:"4a8f4a49",4307:"b4d72f69",4395:"ef2cb6db",4400:"d50c7d6c",4423:"a11b44da",4579:"5929fc75",4583:"2183e388",4791:"8b1641ea",4813:"aa52ff70",5124:"ae92a1dc",5155:"a340ab1b",5164:"23a4cd5a",5201:"84f3f826",5292:"b01c9171",5656:"8342e6f7",5742:"88370a23",5799:"bdc0e527",6038:"6baba7d2",6061:"2190483b",6062:"9e5c894c",6063:"2fe42676",6494:"d431db6e",6501:"d0676c3d",6574:"bc24fa3f",6848:"50abb361",6943:"7ac9571b",7006:"d332b3f1",7098:"707bc455",7145:"bd0583a9",7148:"485d1992",7233:"d2c04590",7398:"7088d117",7472:"1ceae169",7578:"486c123f",7643:"6bef32b0",7737:"84dbe0aa",8209:"ad4ab695",8401:"9f365af6",8535:"383d494c",8633:"0b0840a4",8647:"b4e22e4e",8780:"b2f4baab",8786:"d5f7c43b",8912:"1fae8f75",8932:"80b039a3",8987:"43e26c62",8998:"2245497d",9048:"e8b41c14",9118:"3e584a09",9121:"17236630",9325:"7d2d1d45",9359:"560c841d",9388:"70afc781",9647:"17607f73",9858:"56f87c0d",9989:"a48e9eff"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),d={},b="my-website:",r.l=(e,a,f,c)=>{if(d[e])d[e].push(a);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var b=d[e];if(delete d[e],t.parentNode&&t.parentNode.removeChild(t),b&&b.forEach((e=>e(f))),a)return a(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/rts-docs-dev/pr-preview/pr-65/",r.gca=function(e){return e={17896441:"8401",59362658:"9325",99450296:"5799","07dc3999":"697","29e9f0fb":"824","33fc5bb8":"867","709608ee":"941","3bd1cd05":"1045","02759b6a":"1083","78ec2d69":"1159",ee9fd62d:"1196","38101a41":"1206",a7456010:"1235","6ab77946":"1271",fe973245:"1418","49061c4f":"1878",acecf23e:"1903",e66df417:"2006","6b3f972c":"2057","36504d1f":"2150","40a793ad":"2256",aaf06d06:"2383","0d7929ce":"2540","9e4087bc":"2711","807f69b6":"2823","978f5458":"2929",f1e2e86c:"3031",ccc49370:"3249","001f4272":"3259",d2399e65:"3576",f4f34a3a:"3637","09b877a9":"3844","54dba71d":"3889","7be04b14":"4086",e00b981d:"4126","393be207":"4134",c051bf5e:"4194","621db11d":"4212","9f9dbde3":"4307","2d4abd1b":"4395",ccdcb152:"4400","66a0c2bb":"4423","6e84c0ae":"4579","1df93b7f":"4583","8450c046":"4791","6875c492":"4813",f5934a05:"5124",fd49cdbd:"5155","7365917a":"5164","7b756da4":"5201",aff0a4ba:"5292",fddc7b52:"5656",aba21aa0:"5742","475f08fe":"6038","1f391b9e":"6061",fd9d1f1b:"6062","6f4d7c1e":"6063",c7b2753b:"6494","105f05b3":"6501","133bbacd":"6574","24963d3d":"6848",f05d36a5:"6943","5d34ccd1":"7006",a7bd4aaa:"7098","634b6fb7":"7145","1d6b1623":"7148","104bd8b1":"7233",b2ed70dc:"7398","814f3328":"7472",af2e7acf:"7578",a6aa9e1f:"7643","1e53ba9d":"7737","01a85c17":"8209","7a70fae9":"8535","8b4c439c":"8633",af96c809:"8647",b3bb8099:"8780","540e98b2":"8786",bc1b0a4f:"8912","89790afe":"8932","47a92b8a":"8987","2975eaac":"8998",a94703ab:"9048",f1d7aa2e:"9118","0b7b4c13":"9121",f8428497:"9359",efd0c195:"9388","5e95c892":"9647","36994c47":"9858","7e921b28":"9989"}[e]||e,r.p+r.u(e)},(()=>{var e={5354:0,1869:0};r.f.j=(a,f)=>{var d=r.o(e,a)?e[a]:void 0;if(0!==d)if(d)f.push(d[2]);else if(/^(1869|5354)$/.test(a))e[a]=0;else{var b=new Promise(((f,b)=>d=e[a]=[f,b]));f.push(d[2]=b);var c=r.p+r.u(a),t=new Error;r.l(c,(f=>{if(r.o(e,a)&&(0!==(d=e[a])&&(e[a]=void 0),d)){var b=f&&("load"===f.type?"missing":f.type),c=f&&f.target&&f.target.src;t.message="Loading chunk "+a+" failed.\n("+b+": "+c+")",t.name="ChunkLoadError",t.type=b,t.request=c,d[1](t)}}),"chunk-"+a,a)}},r.O.j=a=>0===e[a];var a=(a,f)=>{var d,b,c=f[0],t=f[1],o=f[2],n=0;if(c.some((a=>0!==e[a]))){for(d in t)r.o(t,d)&&(r.m[d]=t[d]);if(o)var i=o(r)}for(a&&a(f);n{"use strict";var e,a,f,d,b,c={},t={};function r(e){var a=t[e];if(void 0!==a)return a.exports;var f=t[e]={exports:{}};return c[e].call(f.exports,f,f.exports,r),f.exports}r.m=c,e=[],r.O=(a,f,d,b)=>{if(!f){var c=1/0;for(i=0;i=b)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,b0&&e[i-1][2]>b;i--)e[i]=e[i-1];e[i]=[f,d,b]},r.n=e=>{var a=e&&e.__esModule?()=>e.default:()=>e;return r.d(a,{a:a}),a},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,d){if(1&d&&(e=this(e)),8&d)return e;if("object"==typeof e&&e){if(4&d&&e.__esModule)return e;if(16&d&&"function"==typeof e.then)return e}var b=Object.create(null);r.r(b);var c={};a=a||[null,f({}),f([]),f(f)];for(var t=2&d&&e;"object"==typeof t&&!~a.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((a=>c[a]=()=>e[a]));return c.default=()=>e,r.d(b,c),b},r.d=(e,a)=>{for(var f in a)r.o(a,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:a[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((a,f)=>(r.f[f](e,a),a)),[])),r.u=e=>"assets/js/"+({697:"07dc3999",824:"29e9f0fb",867:"33fc5bb8",941:"709608ee",1045:"3bd1cd05",1083:"02759b6a",1159:"78ec2d69",1196:"ee9fd62d",1206:"38101a41",1235:"a7456010",1271:"6ab77946",1418:"fe973245",1878:"49061c4f",1903:"acecf23e",2006:"e66df417",2057:"6b3f972c",2150:"36504d1f",2256:"40a793ad",2383:"aaf06d06",2540:"0d7929ce",2711:"9e4087bc",2823:"807f69b6",2929:"978f5458",3031:"f1e2e86c",3249:"ccc49370",3259:"001f4272",3576:"d2399e65",3637:"f4f34a3a",3844:"09b877a9",3889:"54dba71d",4086:"7be04b14",4126:"e00b981d",4134:"393be207",4194:"c051bf5e",4212:"621db11d",4307:"9f9dbde3",4395:"2d4abd1b",4400:"ccdcb152",4423:"66a0c2bb",4579:"6e84c0ae",4583:"1df93b7f",4791:"8450c046",4813:"6875c492",5124:"f5934a05",5155:"fd49cdbd",5164:"7365917a",5201:"7b756da4",5292:"aff0a4ba",5656:"fddc7b52",5742:"aba21aa0",5799:"99450296",6038:"475f08fe",6061:"1f391b9e",6062:"fd9d1f1b",6063:"6f4d7c1e",6494:"c7b2753b",6501:"105f05b3",6574:"133bbacd",6848:"24963d3d",6943:"f05d36a5",7006:"5d34ccd1",7098:"a7bd4aaa",7145:"634b6fb7",7148:"1d6b1623",7233:"104bd8b1",7398:"b2ed70dc",7472:"814f3328",7578:"af2e7acf",7643:"a6aa9e1f",7737:"1e53ba9d",8209:"01a85c17",8401:"17896441",8535:"7a70fae9",8633:"8b4c439c",8647:"af96c809",8780:"b3bb8099",8786:"540e98b2",8912:"bc1b0a4f",8932:"89790afe",8987:"47a92b8a",8998:"2975eaac",9048:"a94703ab",9118:"f1d7aa2e",9121:"0b7b4c13",9325:"59362658",9359:"f8428497",9388:"efd0c195",9647:"5e95c892",9858:"36994c47",9989:"7e921b28"}[e]||e)+"."+{697:"0cd3787d",824:"00443aae",867:"49849b3a",941:"7ab073b3",1045:"dbe33a14",1083:"bc8047c4",1159:"9f6205d7",1196:"a13f182a",1206:"5a25eb89",1235:"2f05987d",1271:"243754cd",1418:"e899bdae",1467:"70871107",1878:"59b14a5a",1903:"2399748f",2006:"b6f5f9ff",2057:"c4a961c1",2150:"8d53a329",2213:"48817a72",2256:"a275bcad",2383:"899b44ba",2540:"5637b530",2711:"5813f263",2823:"6fa767fc",2929:"d9a84e4b",3031:"8056b6e7",3249:"22472b40",3259:"6951373b",3403:"5b912781",3576:"36e951b0",3637:"2321e716",3844:"88eb5d7f",3889:"b8b55320",4086:"54feaa63",4126:"26f9b193",4134:"b503e730",4194:"2052f5fa",4212:"4a8f4a49",4307:"b4d72f69",4395:"ef2cb6db",4400:"d50c7d6c",4423:"a11b44da",4579:"5929fc75",4583:"2183e388",4791:"8b1641ea",4813:"aa52ff70",5124:"ae92a1dc",5155:"a340ab1b",5164:"23a4cd5a",5201:"84f3f826",5292:"b01c9171",5656:"8342e6f7",5742:"88370a23",5799:"bdc0e527",6038:"6baba7d2",6061:"2190483b",6062:"9e5c894c",6063:"2fe42676",6494:"d431db6e",6501:"d0676c3d",6574:"bc24fa3f",6848:"50abb361",6943:"7ac9571b",7006:"d332b3f1",7098:"707bc455",7145:"bd0583a9",7148:"485d1992",7233:"d2c04590",7398:"7088d117",7472:"1ceae169",7578:"486c123f",7643:"6bef32b0",7737:"84dbe0aa",8209:"ad4ab695",8401:"9f365af6",8535:"383d494c",8633:"0b0840a4",8647:"b4e22e4e",8780:"b2f4baab",8786:"d5f7c43b",8912:"1fae8f75",8932:"80b039a3",8987:"43e26c62",8998:"2245497d",9048:"e8b41c14",9118:"3e584a09",9121:"17236630",9325:"7d2d1d45",9359:"bdfa7c22",9388:"70afc781",9647:"17607f73",9858:"56f87c0d",9989:"a48e9eff"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,a)=>Object.prototype.hasOwnProperty.call(e,a),d={},b="my-website:",r.l=(e,a,f,c)=>{if(d[e])d[e].push(a);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var b=d[e];if(delete d[e],t.parentNode&&t.parentNode.removeChild(t),b&&b.forEach((e=>e(f))),a)return a(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/rts-docs-dev/pr-preview/pr-65/",r.gca=function(e){return e={17896441:"8401",59362658:"9325",99450296:"5799","07dc3999":"697","29e9f0fb":"824","33fc5bb8":"867","709608ee":"941","3bd1cd05":"1045","02759b6a":"1083","78ec2d69":"1159",ee9fd62d:"1196","38101a41":"1206",a7456010:"1235","6ab77946":"1271",fe973245:"1418","49061c4f":"1878",acecf23e:"1903",e66df417:"2006","6b3f972c":"2057","36504d1f":"2150","40a793ad":"2256",aaf06d06:"2383","0d7929ce":"2540","9e4087bc":"2711","807f69b6":"2823","978f5458":"2929",f1e2e86c:"3031",ccc49370:"3249","001f4272":"3259",d2399e65:"3576",f4f34a3a:"3637","09b877a9":"3844","54dba71d":"3889","7be04b14":"4086",e00b981d:"4126","393be207":"4134",c051bf5e:"4194","621db11d":"4212","9f9dbde3":"4307","2d4abd1b":"4395",ccdcb152:"4400","66a0c2bb":"4423","6e84c0ae":"4579","1df93b7f":"4583","8450c046":"4791","6875c492":"4813",f5934a05:"5124",fd49cdbd:"5155","7365917a":"5164","7b756da4":"5201",aff0a4ba:"5292",fddc7b52:"5656",aba21aa0:"5742","475f08fe":"6038","1f391b9e":"6061",fd9d1f1b:"6062","6f4d7c1e":"6063",c7b2753b:"6494","105f05b3":"6501","133bbacd":"6574","24963d3d":"6848",f05d36a5:"6943","5d34ccd1":"7006",a7bd4aaa:"7098","634b6fb7":"7145","1d6b1623":"7148","104bd8b1":"7233",b2ed70dc:"7398","814f3328":"7472",af2e7acf:"7578",a6aa9e1f:"7643","1e53ba9d":"7737","01a85c17":"8209","7a70fae9":"8535","8b4c439c":"8633",af96c809:"8647",b3bb8099:"8780","540e98b2":"8786",bc1b0a4f:"8912","89790afe":"8932","47a92b8a":"8987","2975eaac":"8998",a94703ab:"9048",f1d7aa2e:"9118","0b7b4c13":"9121",f8428497:"9359",efd0c195:"9388","5e95c892":"9647","36994c47":"9858","7e921b28":"9989"}[e]||e,r.p+r.u(e)},(()=>{var e={5354:0,1869:0};r.f.j=(a,f)=>{var d=r.o(e,a)?e[a]:void 0;if(0!==d)if(d)f.push(d[2]);else if(/^(1869|5354)$/.test(a))e[a]=0;else{var b=new Promise(((f,b)=>d=e[a]=[f,b]));f.push(d[2]=b);var c=r.p+r.u(a),t=new Error;r.l(c,(f=>{if(r.o(e,a)&&(0!==(d=e[a])&&(e[a]=void 0),d)){var b=f&&("load"===f.type?"missing":f.type),c=f&&f.target&&f.target.src;t.message="Loading chunk "+a+" failed.\n("+b+": "+c+")",t.name="ChunkLoadError",t.type=b,t.request=c,d[1](t)}}),"chunk-"+a,a)}},r.O.j=a=>0===e[a];var a=(a,f)=>{var d,b,c=f[0],t=f[1],o=f[2],n=0;if(c.some((a=>0!==e[a]))){for(d in t)r.o(t,d)&&(r.m[d]=t[d]);if(o)var i=o(r)}for(a&&a(f);n Archive | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/authors/all-sebastien-lorber-articles/index.html b/pr-preview/pr-65/blog/authors/all-sebastien-lorber-articles/index.html index 4f83e468..58e94af0 100644 --- a/pr-preview/pr-65/blog/authors/all-sebastien-lorber-articles/index.html +++ b/pr-preview/pr-65/blog/authors/all-sebastien-lorber-articles/index.html @@ -5,7 +5,7 @@ Sébastien Lorber - One post | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/authors/index.html b/pr-preview/pr-65/blog/authors/index.html index 8fed60e9..fca677c6 100644 --- a/pr-preview/pr-65/blog/authors/index.html +++ b/pr-preview/pr-65/blog/authors/index.html @@ -5,7 +5,7 @@ Authors | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/authors/yangshun/index.html b/pr-preview/pr-65/blog/authors/yangshun/index.html index 75e230d8..5bca2109 100644 --- a/pr-preview/pr-65/blog/authors/yangshun/index.html +++ b/pr-preview/pr-65/blog/authors/yangshun/index.html @@ -5,7 +5,7 @@ Yangshun Tay - 0 posts | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/index.html b/pr-preview/pr-65/blog/index.html index b281885f..425af6e9 100644 --- a/pr-preview/pr-65/blog/index.html +++ b/pr-preview/pr-65/blog/index.html @@ -5,7 +5,7 @@ Blog | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/mdx-blog-post/index.html b/pr-preview/pr-65/blog/mdx-blog-post/index.html index 6a1dc687..130a7dd1 100644 --- a/pr-preview/pr-65/blog/mdx-blog-post/index.html +++ b/pr-preview/pr-65/blog/mdx-blog-post/index.html @@ -5,7 +5,7 @@ MDX Blog Post | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/tags/docusaurus/index.html b/pr-preview/pr-65/blog/tags/docusaurus/index.html index 8bca6878..76c801cc 100644 --- a/pr-preview/pr-65/blog/tags/docusaurus/index.html +++ b/pr-preview/pr-65/blog/tags/docusaurus/index.html @@ -5,7 +5,7 @@ One post tagged with "Docusaurus" | Research Technology Servcies - + diff --git a/pr-preview/pr-65/blog/tags/index.html b/pr-preview/pr-65/blog/tags/index.html index 48d5d780..0a17a966 100644 --- a/pr-preview/pr-65/blog/tags/index.html +++ b/pr-preview/pr-65/blog/tags/index.html @@ -5,7 +5,7 @@ Tags | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/genai/intro/index.html b/pr-preview/pr-65/docs/genai/intro/index.html index 35486408..984ed618 100644 --- a/pr-preview/pr-65/docs/genai/intro/index.html +++ b/pr-preview/pr-65/docs/genai/intro/index.html @@ -5,7 +5,7 @@ Generative AI for Research workflows | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/connecting_to_hpc/index.html b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/connecting_to_hpc/index.html index a6fac44f..7299f8d9 100644 --- a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/connecting_to_hpc/index.html +++ b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/connecting_to_hpc/index.html @@ -5,7 +5,7 @@ Connecting to the HPC Cluster | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ood/index.html b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ood/index.html index 82bfc930..4c90bfb8 100644 --- a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ood/index.html +++ b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ood/index.html @@ -5,7 +5,7 @@ Open OnDemand (Web-based Graphical User Interface) | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ssh_tunneling_and_x11_forwarding/index.html b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ssh_tunneling_and_x11_forwarding/index.html index a6f9e955..a3607671 100644 --- a/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ssh_tunneling_and_x11_forwarding/index.html +++ b/pr-preview/pr-65/docs/hpc/connecting_to_hpc/ssh_tunneling_and_x11_forwarding/index.html @@ -5,7 +5,7 @@ SSH Tunneling and X11 Forwarding | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/containers/containers/index.html b/pr-preview/pr-65/docs/hpc/containers/containers/index.html index 5b94110a..b5f65854 100644 --- a/pr-preview/pr-65/docs/hpc/containers/containers/index.html +++ b/pr-preview/pr-65/docs/hpc/containers/containers/index.html @@ -5,11 +5,11 @@ Using Containers on HPC | Research Technology Servcies - +

Using Containers on HPC

-
+ \ No newline at end of file diff --git a/pr-preview/pr-65/docs/hpc/containers/intro/index.html b/pr-preview/pr-65/docs/hpc/containers/intro/index.html index a5cb391e..018a510f 100644 --- a/pr-preview/pr-65/docs/hpc/containers/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/containers/intro/index.html @@ -5,7 +5,7 @@ Apptainer/Singularity | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda/index.html b/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda/index.html index 237b252b..8acb74d5 100644 --- a/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda/index.html +++ b/pr-preview/pr-65/docs/hpc/containers/singularity_with_conda/index.html @@ -5,7 +5,7 @@ Singularity with Conda | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity/index.html b/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity/index.html index aabf593c..7dd7d8f8 100644 --- a/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity/index.html +++ b/pr-preview/pr-65/docs/hpc/containers/squash_file_system_and_singularity/index.html @@ -5,7 +5,7 @@ Squash File System and Singularity | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/datasets/intro/index.html b/pr-preview/pr-65/docs/hpc/datasets/intro/index.html index 4b574c5c..0758f705 100644 --- a/pr-preview/pr-65/docs/hpc/datasets/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/datasets/intro/index.html @@ -5,7 +5,7 @@ Datasets Available | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/datasets/squash_file_system_and_singularity/index.html b/pr-preview/pr-65/docs/hpc/datasets/squash_file_system_and_singularity/index.html index 337e05ca..c28b82b8 100644 --- a/pr-preview/pr-65/docs/hpc/datasets/squash_file_system_and_singularity/index.html +++ b/pr-preview/pr-65/docs/hpc/datasets/squash_file_system_and_singularity/index.html @@ -5,7 +5,7 @@ Squash File System and Singularity | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/getting_started/getting_and_renewing_an_account/index.html b/pr-preview/pr-65/docs/hpc/getting_started/getting_and_renewing_an_account/index.html index 8e730942..3bd6fdf2 100644 --- a/pr-preview/pr-65/docs/hpc/getting_started/getting_and_renewing_an_account/index.html +++ b/pr-preview/pr-65/docs/hpc/getting_started/getting_and_renewing_an_account/index.html @@ -5,7 +5,7 @@ Getting and Renewing an Account | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/getting_started/hpc_accounts_external_collaborators/index.html b/pr-preview/pr-65/docs/hpc/getting_started/hpc_accounts_external_collaborators/index.html index 88ae049c..150529c5 100644 --- a/pr-preview/pr-65/docs/hpc/getting_started/hpc_accounts_external_collaborators/index.html +++ b/pr-preview/pr-65/docs/hpc/getting_started/hpc_accounts_external_collaborators/index.html @@ -5,7 +5,7 @@ HPC Accounts for Sponsored External Collaborators | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/getting_started/intro/index.html b/pr-preview/pr-65/docs/hpc/getting_started/intro/index.html index 9e9b6762..ea771c92 100644 --- a/pr-preview/pr-65/docs/hpc/getting_started/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/getting_started/intro/index.html @@ -5,7 +5,7 @@ Start here! | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_approve_hpc_account_request/index.html b/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_approve_hpc_account_request/index.html index 1115a8cb..73152797 100644 --- a/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_approve_hpc_account_request/index.html +++ b/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_approve_hpc_account_request/index.html @@ -5,7 +5,7 @@ How to approve an HPC Account Request | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_renew_hpc_account_iiq/index.html b/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_renew_hpc_account_iiq/index.html index abc34ee2..a7af2ece 100644 --- a/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_renew_hpc_account_iiq/index.html +++ b/pr-preview/pr-65/docs/hpc/getting_started/walkthrough_renew_hpc_account_iiq/index.html @@ -5,7 +5,7 @@ Renewing your HPC Account with IIQ | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/intro/index.html b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/intro/index.html index e78373d1..7849a437 100644 --- a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/intro/index.html @@ -5,7 +5,7 @@ Machine Learning on HPC | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_fine_tuning/index.html b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_fine_tuning/index.html index 59f3bc55..1b395701 100644 --- a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_fine_tuning/index.html +++ b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_fine_tuning/index.html @@ -5,7 +5,7 @@ Fine tune LLMs on HPC | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_on_hpc/index.html b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_on_hpc/index.html index f9781170..2b3dd442 100644 --- a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_on_hpc/index.html +++ b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/llm_on_hpc/index.html @@ -5,7 +5,7 @@ LLMs on HPC | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/pytorch_on_hpc/index.html b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/pytorch_on_hpc/index.html index c4d1c2a5..d0ca5a3d 100644 --- a/pr-preview/pr-65/docs/hpc/ml_ai_hpc/pytorch_on_hpc/index.html +++ b/pr-preview/pr-65/docs/hpc/ml_ai_hpc/pytorch_on_hpc/index.html @@ -5,7 +5,7 @@ PyTorch on HPC | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/ood/open_on_demand/index.html b/pr-preview/pr-65/docs/hpc/ood/open_on_demand/index.html index aec49e07..d0f777a7 100644 --- a/pr-preview/pr-65/docs/hpc/ood/open_on_demand/index.html +++ b/pr-preview/pr-65/docs/hpc/ood/open_on_demand/index.html @@ -5,7 +5,7 @@ Open OnDemand (OOD) with Conda/Singularity | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/spec_sheet/index.html b/pr-preview/pr-65/docs/hpc/spec_sheet/index.html index 4df7a4f1..d7f00c38 100644 --- a/pr-preview/pr-65/docs/hpc/spec_sheet/index.html +++ b/pr-preview/pr-65/docs/hpc/spec_sheet/index.html @@ -5,7 +5,7 @@ Greene Spec Sheet | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/available_storage_systems/index.html b/pr-preview/pr-65/docs/hpc/storage/available_storage_systems/index.html index d18d84df..faca52cf 100644 --- a/pr-preview/pr-65/docs/hpc/storage/available_storage_systems/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/available_storage_systems/index.html @@ -5,7 +5,7 @@ Available storage systems | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/best_practices/index.html b/pr-preview/pr-65/docs/hpc/storage/best_practices/index.html index 4a6a6294..0df13735 100644 --- a/pr-preview/pr-65/docs/hpc/storage/best_practices/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/best_practices/index.html @@ -5,7 +5,7 @@ Best Practices on HPC Storage | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/data_management/index.html b/pr-preview/pr-65/docs/hpc/storage/data_management/index.html index 12755fa9..5f81d9e6 100644 --- a/pr-preview/pr-65/docs/hpc/storage/data_management/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/data_management/index.html @@ -5,7 +5,7 @@ Data Management | Research Technology Servcies - + @@ -62,6 +62,6 @@

HPC VastHPC Research Project Space

HPC Work​

The HPC team makes available a number of public sets that are commonly used in analysis jobs. The data sets are available Read-Only under /scratch/work/public.

For some of the datasets users must provide a signed usage agreement before accessing.

Public datasets available on the HPC clusters can be viewed on the Datasets page.

HPC Archive​

Once the Analysis stage of the research data lifecycle has completed, HPC users should tar their data and code into a single tar.gz file and then copy the file to their archive directory (/archive/$USER). The HPC Archive file system is not accessible by running jobs; it is suitable for long-term data storage. Each user has access to a default disk quota of 2TB and 20,000 inode (files) limit. The rather low limit on the number of inodes per user is intentional. The archive file system is available only on login nodes of Greene. The archive file system is backed up daily.

  • Here is an example tar command that combines the data in a directory named my_run_dir under $SCRATCH and outputs the tar file in the user's $ARCHIVE:
    • -
# to archive `$SCRATCH/my_run_dir`
tar cvf $ARCHIVE/simulation_01.tar -C $SCRATCH my_run_dir

NYU (Google) Drive​

Google Drive (NYU Drive) is accessible from the NYU HPC environment and provides an option to users who wish to archive data or share data with external collaborators who do not have access to the NYU HPC environment.

Currently (March 2021) there is no limit on the amount of data a user can store on Google Drive and there is no cost associated with storing data on Google Drive (although we hear rumors that free storage on Google Drive may be ending soon).

However, there are limits to the data transfer rate in moving to/from Google Drive. Thus, moving many small files to Google Drive is not going to be efficient.

Please read the Instructions on how to use cloud storage within the NYU HPC Environment.

HPC Storage Mounts Comparison Table​

Data Management Subpages​

Data Transfers​

Link to Data Transfers Page

Sharing Data on HPC​

Link to Sharing Data on HPC Page

Research Project Space​

Link to Research Project Space Page

Squash File System and Singularity​

Link to Squash File System and Singularity Page

+
# to archive `$SCRATCH/my_run_dir`
tar cvf $ARCHIVE/simulation_01.tar -C $SCRATCH my_run_dir

NYU (Google) Drive​

Google Drive (NYU Drive) is accessible from the NYU HPC environment and provides an option to users who wish to archive data or share data with external collaborators who do not have access to the NYU HPC environment.

Currently (March 2021) there is no limit on the amount of data a user can store on Google Drive and there is no cost associated with storing data on Google Drive (although we hear rumors that free storage on Google Drive may be ending soon).

However, there are limits to the data transfer rate in moving to/from Google Drive. Thus, moving many small files to Google Drive is not going to be efficient.

Please read the Instructions on how to use cloud storage within the NYU HPC Environment.

HPC Storage Mounts Comparison Table​

Data Management Subpages​

Data Transfers​

Link to Data Transfers Page

Sharing Data on HPC​

Link to Sharing Data on HPC Page

Research Project Space​

Link to Research Project Space Page

Squash File System and Singularity​

Link to Squash File System and Singularity Page

\ No newline at end of file diff --git a/pr-preview/pr-65/docs/hpc/storage/data_transfers/index.html b/pr-preview/pr-65/docs/hpc/storage/data_transfers/index.html index 182e7521..4eb8315b 100644 --- a/pr-preview/pr-65/docs/hpc/storage/data_transfers/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/data_transfers/index.html @@ -5,7 +5,7 @@ Data Transfers | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/globus/index.html b/pr-preview/pr-65/docs/hpc/storage/globus/index.html index e0148584..09a690a6 100644 --- a/pr-preview/pr-65/docs/hpc/storage/globus/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/globus/index.html @@ -5,7 +5,7 @@ Globus | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/intro/index.html b/pr-preview/pr-65/docs/hpc/storage/intro/index.html index b20c7ea1..72e0e618 100644 --- a/pr-preview/pr-65/docs/hpc/storage/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/intro/index.html @@ -5,7 +5,7 @@ HPC Storage | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/large_number_of_small_files/index.html b/pr-preview/pr-65/docs/hpc/storage/large_number_of_small_files/index.html index 7f4c68e9..8ec1214c 100644 --- a/pr-preview/pr-65/docs/hpc/storage/large_number_of_small_files/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/large_number_of_small_files/index.html @@ -5,7 +5,7 @@ Large Number of Small Files | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/research_project_space/index.html b/pr-preview/pr-65/docs/hpc/storage/research_project_space/index.html index 340e6a6a..5e83d5c6 100644 --- a/pr-preview/pr-65/docs/hpc/storage/research_project_space/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/research_project_space/index.html @@ -5,7 +5,7 @@ Research Project Space (RPS) | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc/index.html b/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc/index.html index 4e219f41..84863631 100644 --- a/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/sharing_data_on_hpc/index.html @@ -5,7 +5,7 @@ Sharing Data on HPC | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/system_status/index.html b/pr-preview/pr-65/docs/hpc/storage/system_status/index.html index ea9b44bf..7ce7fac8 100644 --- a/pr-preview/pr-65/docs/hpc/storage/system_status/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/system_status/index.html @@ -5,7 +5,7 @@ HPC Storage System Status | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone/index.html b/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone/index.html index d184e7b4..7c1daa9e 100644 --- a/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone/index.html +++ b/pr-preview/pr-65/docs/hpc/storage/transferring_cloud_storage_data_with_rclone/index.html @@ -5,7 +5,7 @@ Transferring Cloud Storage Data with rclone | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_main_commands/index.html b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_main_commands/index.html index d14438f6..5dbb8020 100644 --- a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_main_commands/index.html +++ b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_main_commands/index.html @@ -5,7 +5,7 @@ Slurm: Main Commands | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_submitting_jobs/index.html b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_submitting_jobs/index.html index c886c3c4..5d7f2089 100644 --- a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_submitting_jobs/index.html +++ b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_submitting_jobs/index.html @@ -5,7 +5,7 @@ Slurm - Submitting Jobs | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_tutorial/index.html b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_tutorial/index.html index 764f379d..42d53a28 100644 --- a/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_tutorial/index.html +++ b/pr-preview/pr-65/docs/hpc/submitting_jobs/slurm_tutorial/index.html @@ -5,7 +5,7 @@ Slurm Tutorial | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/system_status/index.html b/pr-preview/pr-65/docs/hpc/system_status/index.html index dc5883a8..bfaa8533 100644 --- a/pr-preview/pr-65/docs/hpc/system_status/index.html +++ b/pr-preview/pr-65/docs/hpc/system_status/index.html @@ -5,7 +5,7 @@ Greene System Status | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/conda_environments/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/conda_environments/index.html index 3598de9a..caea7cfc 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/conda_environments/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/conda_environments/index.html @@ -5,7 +5,7 @@ Conda Environments (Python, R) | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/intro/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/intro/index.html index 03f1a3ea..6a8918a0 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/intro/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/intro/index.html @@ -5,7 +5,7 @@ Tools and Software | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/python_packages_with_virtual_environments/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/python_packages_with_virtual_environments/index.html index 6a087bd1..a25ab9dc 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/python_packages_with_virtual_environments/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/python_packages_with_virtual_environments/index.html @@ -5,7 +5,7 @@ Python Packages with Virtual Environments | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/r_packages_with_renv/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/r_packages_with_renv/index.html index a6b235f8..ead61aae 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/r_packages_with_renv/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/r_packages_with_renv/index.html @@ -5,7 +5,7 @@ R Packages with renv | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/singularity_run_custom_applications_with_containers/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/singularity_run_custom_applications_with_containers/index.html index 78c1aa51..f0eef58e 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/singularity_run_custom_applications_with_containers/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/singularity_run_custom_applications_with_containers/index.html @@ -5,7 +5,7 @@ Singularity: Run Custom Applications with Containers | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tools_and_software/software_on_greene/index.html b/pr-preview/pr-65/docs/hpc/tools_and_software/software_on_greene/index.html index 4327ea6c..4edf2bbd 100644 --- a/pr-preview/pr-65/docs/hpc/tools_and_software/software_on_greene/index.html +++ b/pr-preview/pr-65/docs/hpc/tools_and_software/software_on_greene/index.html @@ -5,7 +5,7 @@ Software on Greene | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/training_and_support/support/index.html b/pr-preview/pr-65/docs/hpc/training_and_support/support/index.html index afab6eee..4cd6721b 100644 --- a/pr-preview/pr-65/docs/hpc/training_and_support/support/index.html +++ b/pr-preview/pr-65/docs/hpc/training_and_support/support/index.html @@ -5,7 +5,7 @@ Support | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/training_and_support/training_at_nyu/index.html b/pr-preview/pr-65/docs/hpc/training_and_support/training_at_nyu/index.html index fe48a226..4c8769b9 100644 --- a/pr-preview/pr-65/docs/hpc/training_and_support/training_at_nyu/index.html +++ b/pr-preview/pr-65/docs/hpc/training_and_support/training_at_nyu/index.html @@ -5,7 +5,7 @@ HPC training at NYU | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/training_and_support/training_outside/index.html b/pr-preview/pr-65/docs/hpc/training_and_support/training_outside/index.html index 34d4d414..45159213 100644 --- a/pr-preview/pr-65/docs/hpc/training_and_support/training_outside/index.html +++ b/pr-preview/pr-65/docs/hpc/training_and_support/training_outside/index.html @@ -5,7 +5,7 @@ HPC training outside NYU | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tutorials/hpc_foundations/index.html b/pr-preview/pr-65/docs/hpc/tutorials/hpc_foundations/index.html index 1c6679b7..4098eb1b 100644 --- a/pr-preview/pr-65/docs/hpc/tutorials/hpc_foundations/index.html +++ b/pr-preview/pr-65/docs/hpc/tutorials/hpc_foundations/index.html @@ -5,7 +5,7 @@ HPC Foundations | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hpc/tutorials/linux_tutorial/index.html b/pr-preview/pr-65/docs/hpc/tutorials/linux_tutorial/index.html index 73650b1c..b4911e5b 100644 --- a/pr-preview/pr-65/docs/hpc/tutorials/linux_tutorial/index.html +++ b/pr-preview/pr-65/docs/hpc/tutorials/linux_tutorial/index.html @@ -5,7 +5,7 @@ Linux Tutorial | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/hsrn/intro/index.html b/pr-preview/pr-65/docs/hsrn/intro/index.html index 18722307..edd979ca 100644 --- a/pr-preview/pr-65/docs/hsrn/intro/index.html +++ b/pr-preview/pr-65/docs/hsrn/intro/index.html @@ -5,7 +5,7 @@ High Speed Research Network | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/rtc/intro/index.html b/pr-preview/pr-65/docs/rtc/intro/index.html index 72118892..da4c8cea 100644 --- a/pr-preview/pr-65/docs/rtc/intro/index.html +++ b/pr-preview/pr-65/docs/rtc/intro/index.html @@ -5,7 +5,7 @@ Research Technology Cloud | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/dlp/dlp/index.html b/pr-preview/pr-65/docs/srde/dlp/dlp/index.html index 573a9227..2f3acadf 100644 --- a/pr-preview/pr-65/docs/srde/dlp/dlp/index.html +++ b/pr-preview/pr-65/docs/srde/dlp/dlp/index.html @@ -5,7 +5,7 @@ DLP Interpretation Guide | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/faq/basics/index.html b/pr-preview/pr-65/docs/srde/faq/basics/index.html index 60892c9c..3f6d6eb3 100644 --- a/pr-preview/pr-65/docs/srde/faq/basics/index.html +++ b/pr-preview/pr-65/docs/srde/faq/basics/index.html @@ -5,7 +5,7 @@ About SRDE, projects and getting started | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/faq/env_roles/index.html b/pr-preview/pr-65/docs/srde/faq/env_roles/index.html index f30223b9..aed492ec 100644 --- a/pr-preview/pr-65/docs/srde/faq/env_roles/index.html +++ b/pr-preview/pr-65/docs/srde/faq/env_roles/index.html @@ -5,7 +5,7 @@ Environment and Roles | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/faq/using_srde/index.html b/pr-preview/pr-65/docs/srde/faq/using_srde/index.html index 18b8fdec..00d75f8b 100644 --- a/pr-preview/pr-65/docs/srde/faq/using_srde/index.html +++ b/pr-preview/pr-65/docs/srde/faq/using_srde/index.html @@ -5,7 +5,7 @@ Using the SRDE | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/getting_started/eligibility_accounts/index.html b/pr-preview/pr-65/docs/srde/getting_started/eligibility_accounts/index.html index 512b99a5..4eb4380c 100644 --- a/pr-preview/pr-65/docs/srde/getting_started/eligibility_accounts/index.html +++ b/pr-preview/pr-65/docs/srde/getting_started/eligibility_accounts/index.html @@ -5,7 +5,7 @@ Secure Research Data Environment (SRDE) | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/getting_started/intro/index.html b/pr-preview/pr-65/docs/srde/getting_started/intro/index.html index e62c2a9f..e0e030a9 100644 --- a/pr-preview/pr-65/docs/srde/getting_started/intro/index.html +++ b/pr-preview/pr-65/docs/srde/getting_started/intro/index.html @@ -5,7 +5,7 @@ Start here! | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/support/support/index.html b/pr-preview/pr-65/docs/srde/support/support/index.html index 4807d600..682854f0 100644 --- a/pr-preview/pr-65/docs/srde/support/support/index.html +++ b/pr-preview/pr-65/docs/srde/support/support/index.html @@ -5,7 +5,7 @@ Support | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/user_guide/best_practices/index.html b/pr-preview/pr-65/docs/srde/user_guide/best_practices/index.html index acff5730..a0c0901c 100644 --- a/pr-preview/pr-65/docs/srde/user_guide/best_practices/index.html +++ b/pr-preview/pr-65/docs/srde/user_guide/best_practices/index.html @@ -5,7 +5,7 @@ Best Practices | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/user_guide/connecting/index.html b/pr-preview/pr-65/docs/srde/user_guide/connecting/index.html index 3c039094..5bd8942d 100644 --- a/pr-preview/pr-65/docs/srde/user_guide/connecting/index.html +++ b/pr-preview/pr-65/docs/srde/user_guide/connecting/index.html @@ -5,7 +5,7 @@ Connecting to SRDE | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/user_guide/data_access/index.html b/pr-preview/pr-65/docs/srde/user_guide/data_access/index.html index 3305f625..5f8f6fc3 100644 --- a/pr-preview/pr-65/docs/srde/user_guide/data_access/index.html +++ b/pr-preview/pr-65/docs/srde/user_guide/data_access/index.html @@ -5,7 +5,7 @@ Data Access | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/user_guide/data_transfers/index.html b/pr-preview/pr-65/docs/srde/user_guide/data_transfers/index.html index 2c828776..9243be17 100644 --- a/pr-preview/pr-65/docs/srde/user_guide/data_transfers/index.html +++ b/pr-preview/pr-65/docs/srde/user_guide/data_transfers/index.html @@ -5,7 +5,7 @@ Managing Data Transfer | Research Technology Servcies - + diff --git a/pr-preview/pr-65/docs/srde/user_guide/troubleshooting/index.html b/pr-preview/pr-65/docs/srde/user_guide/troubleshooting/index.html index a09d7812..cd461e7c 100644 --- a/pr-preview/pr-65/docs/srde/user_guide/troubleshooting/index.html +++ b/pr-preview/pr-65/docs/srde/user_guide/troubleshooting/index.html @@ -5,7 +5,7 @@ Troubleshooting | Research Technology Servcies - + diff --git a/pr-preview/pr-65/index.html b/pr-preview/pr-65/index.html index 9df6599c..91a6f32e 100644 --- a/pr-preview/pr-65/index.html +++ b/pr-preview/pr-65/index.html @@ -5,7 +5,7 @@ Hello from Research Technology Servcies | Research Technology Servcies - + diff --git a/pr-preview/pr-65/markdown-page/index.html b/pr-preview/pr-65/markdown-page/index.html index fd50abcc..9a4f28d3 100644 --- a/pr-preview/pr-65/markdown-page/index.html +++ b/pr-preview/pr-65/markdown-page/index.html @@ -5,7 +5,7 @@ Markdown page example | Research Technology Servcies - +