diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 1806bdb00d1..7e4edc2dc36 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: WARPX_CI_EB: 'TRUE' # default: 60; maximum: 360 - timeoutInMinutes: 180 + timeoutInMinutes: 240 steps: # set up caches: diff --git a/.clang-tidy b/.clang-tidy index a659b8dc5be..21f0343519c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,13 +7,13 @@ Checks: ' -bugprone-unchecked-optional-access, cert-*, -cert-err58-cpp, + clang-diagnostic-*, cppcoreguidelines-*, -cppcoreguidelines-avoid-c-arrays, -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, - -cppcoreguidelines-no-malloc, -cppcoreguidelines-narrowing-conversions, -cppcoreguidelines-non-private-member-variables-in-classes, -cppcoreguidelines-owning-memory, @@ -29,38 +29,19 @@ Checks: ' -modernize-return-braced-init-list, -modernize-use-trailing-return-type, mpi-*, - performance-faster-string-find, - performance-for-range-copy, - performance-implicit-conversion-in-loop, - performance-inefficient-algorithm, - performance-inefficient-string-concatenation, - performance-inefficient-vector-operation, - performance-move-const-arg, - performance-move-constructor-init, - performance-no-automatic-move, - performance-no-int-to-ptr, - readability-avoid-const-params-in-decls, - readability-const-return-type, - readability-container-contains, - readability-container-data-pointer, - readability-container-size-empty, - readability-non-const-parameter, - readability-redundant-control-flow, - readability-redundant-declaration, - readability-redundant-function-ptr-dereference, - readability-redundant-member-init, - readability-redundant-preprocessor, - readability-redundant-smartptr-get, - readability-redundant-string-cstr, - readability-redundant-string-init, - readability-simplify-boolean-expr, - readability-simplify-subscript-expr, - readability-static-accessed-through-instance, - readability-static-definition-in-anonymous-namespace, - readability-string-compare, - readability-suspicious-call-argument, - readability-uniqueptr-delete-release, - readability-use-anyofallof, + performance-*, + -performance-unnecessary-copy-initialization, + -performance-unnecessary-value-param, + portability-*, + readability-*, + -readability-convert-member-functions-to-static, + -readability-else-after-return, + -readability-function-cognitive-complexity, + -readability-identifier-length, + -readability-implicit-bool-conversion, + -readability-isolate-declaration, + -readability-magic-numbers, + -readability-named-parameter, -readability-uppercase-literal-suffix ' diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index adfb827e4d8..6a1172802a8 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -12,43 +12,62 @@ jobs: runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/clang14.sh - - name: build WarpX using clang-tidy + - name: set up cache + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: build WarpX & run clang-tidy run: | - + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=300M + export CCACHE_EXTRAFILES=${{ github.workspace }}/.clang-tidy + export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt + ccache -z export CXX=$(which clang++) export CC=$(which clang) - # The following wrapper ensures that only source files - # in WarpX/Source/* are actually processed by clang-tidy - #_______________________________ - cat > clang_tidy_wrapper << EOF - #!/bin/bash - REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" - if [[ \$4 =~ \$REGEX ]];then - clang-tidy \$@ - fi - EOF - chmod +x clang_tidy_wrapper - #_____________________________________ - cmake -S . -B build_clang_tidy \ - -DCMAKE_CXX_CLANG_TIDY="$PWD/clang_tidy_wrapper;--system-headers=0;--config-file=$PWD/.clang-tidy" \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DWarpX_DIMS="1;2;3;RZ" \ + -DWarpX_DIMS="1;2;RZ;3" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ -DWarpX_PSATD=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ - -DWarpX_PRECISION=SINGLE + -DWarpX_PRECISION=SINGLE \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + + cmake --build build_clang_tidy -j 4 - cmake --build build_clang_tidy -j 2 2> build_clang_tidy/clang-tidy.log + ${{github.workspace}}/.github/workflows/source/makeMakefileForClangTidy.py --input ${{github.workspace}}/ccache.log.txt + make -j4 --keep-going -f clang-tidy-ccache-misses.mak \ + CLANG_TIDY=clang-tidy \ + CLANG_TIDY_ARGS="--config-file=${{github.workspace}}/.clang-tidy --warnings-as-errors=*" - cat build_clang_tidy/clang-tidy.log - if [[ $(wc -m pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml new file mode 100644 index 00000000000..9a2ffb0f61a --- /dev/null +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -0,0 +1,40 @@ +name: CleanUpCachePostPR + +on: + workflow_run: + workflows: [PostPR] + types: + - completed + +jobs: + CleanUpCcacheCachePostPR: + name: Clean Up Ccache Cache Post PR + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1) + # $keys might contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for k in $keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + unset IFS diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml new file mode 100644 index 00000000000..bd1a518acf4 --- /dev/null +++ b/.github/workflows/cleanup-cache.yml @@ -0,0 +1,63 @@ +name: CleanUpCache + +on: + workflow_run: + workflows: [🧹 clang-tidy, 🔍 CodeQL, 🐧 CUDA, 🐧 HIP, 🐧 Intel, 🍏 macOS, 🐧 OpenMP] + types: + - completed + +jobs: + CleanUpCcacheCache: + name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + - name: Clean up ccache + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + + # push or pull_request or schedule or ... + EVENT=${{ github.event.workflow_run.event }} + + # Triggering workflow run name (e.g., LinuxClang) + WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + + if [[ $EVENT == "pull_request" ]]; then + gh run download ${{ github.event.workflow_run.id }} -n pr_number + pr_number=`cat pr_number.txt` + BRANCH=refs/pull/${pr_number}/merge + else + BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} + fi + + # Setting this to not fail the workflow while deleting cache keys. + set +e + + # In our cache keys, substring after `-git-` is git hash, substring + # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`). + # The goal is to keep the last used key of each job and delete all others. + + # something like ccache-LinuxClang- + keyprefix="ccache-${WORKFLOW_NAME}-" + + cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq) + + # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d". + # It might also contain spaces. Thus we set IFS to \n. + IFS=$'\n' + for j in $cached_jobs + do + old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) + for k in $old_keys + do + gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm + done + done + unset IFS diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 55d0e473d27..bc0bee545cc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,42 +29,64 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Packages (C++) if: ${{ matrix.language == 'cpp' }} run: | sudo apt-get update - sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev + sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev ccache python -m pip install --upgrade pip python -m pip install --upgrade wheel python -m pip install --upgrade cmake export CMAKE="$HOME/.local/bin/cmake" && echo "CMAKE=$CMAKE" >> $GITHUB_ENV + - name: Set Up Cache + if: ${{ matrix.language == 'cpp' }} + uses: actions/cache@v4 + with: + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} + restore-keys: | + ccache-${{ github.workflow }}-${{ github.job }}-git- + - name: Configure (C++) if: ${{ matrix.language == 'cpp' }} run: | $CMAKE -S . -B build -DWarpX_OPENPMD=ON - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: config-file: ./.github/codeql/warpx-codeql.yml languages: ${{ matrix.language }} queries: +security-and-quality - name: Build (py) - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 if: ${{ matrix.language == 'python' }} - name: Build (C++) if: ${{ matrix.language == 'cpp' }} run: | - $CMAKE --build build -j 2 + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + + $CMAKE --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache + + # Make sure CodeQL has something to do + touch Source/Utils/WarpXVersion.cpp + export CCACHE_DISABLE=1 + $CMAKE --build build -j 4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" upload: False @@ -85,6 +107,21 @@ jobs: output: sarif-results/${{ matrix.language }}.sarif - name: Upload SARIF - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: sarif-results/${{ matrix.language }}.sarif + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index aab28b74b80..368221efa9c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -19,8 +19,8 @@ jobs: CXXFLAGS: "-Werror" CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 name: Install Python with: python-version: '3.x' @@ -28,19 +28,19 @@ jobs: run: | .github/workflows/dependencies/nvcc11-3.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvcc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: install openPMD-api run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" cmake-easyinstall --prefix=/usr/local \ @@ -53,6 +53,10 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" @@ -69,7 +73,7 @@ jobs: -DWarpX_PSATD=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel @@ -78,6 +82,9 @@ jobs: python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + # make sure legacy build system continues to build, i.e., that we don't forget # to add new .cpp files build_nvcc_gnumake: @@ -85,59 +92,62 @@ jobs: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/nvcc11-8.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-gnumake-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-gnumake- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 9e35dc19489dc5d312e92781cb0471d282cf8370 && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 + cd ../amrex && git checkout --detach 2ecafcff40132f56eb2b494e1a374684ff97117a && cd - + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 + + ccache -s + du -hs ~/.cache/ccache - build_nvhpc21-11-nvcc: - name: NVHPC@21.11 NVCC/NVC++ Release [tests] + build_nvhpc24-1-nvcc: + name: NVHPC@24.1 NVCC/NVC++ Release [tests] runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false #env: # # For NVHPC, Ninja is slower than the default: # CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Dependencies run: .github/workflows/dependencies/nvhpc.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-cuda-nvhpc-${{ hashFiles('.github/workflows/cuda.yml') }}- - ccache-cuda-nvhpc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: Build & Install run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=600M + ccache -z + source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/24.1 which nvcc || echo "nvcc not in PATH!" which nvc++ || echo "nvc++ not in PATH!" which nvc || echo "nvc not in PATH!" @@ -155,13 +165,13 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_COMPUTE=CUDA \ -DWarpX_EB=ON \ - -DWarpX_PYTHON=ON \ + -DWarpX_PYTHON=OFF \ -DAMReX_CUDA_ARCH=8.0 \ -DWarpX_OPENPMD=ON \ -DWarpX_PSATD=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON - cmake --build build -j 2 + cmake --build build -j 4 # work-around for mpi4py 3.1.1 build system issue with using # a GNU-built Python executable with non-GNU Python modules @@ -174,3 +184,21 @@ jobs: #export PYWARPX_LIB_DIR=$PWD/build/lib/site-packages/pywarpx/ #python3 -m pip wheel . #python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/dependencies/ccache.sh b/.github/workflows/dependencies/ccache.sh new file mode 100755 index 00000000000..30a155ff20c --- /dev/null +++ b/.github/workflows/dependencies/ccache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +if [[ $# -eq 2 ]]; then + CVER=$1 +else + CVER=4.8.3 +fi + +wget https://github.com/ccache/ccache/releases/download/v${CVER}/ccache-${CVER}-linux-x86_64.tar.xz +tar xvf ccache-${CVER}-linux-x86_64.tar.xz +sudo cp -f ccache-${CVER}-linux-x86_64/ccache /usr/local/bin/ diff --git a/.github/workflows/dependencies/clang14.sh b/.github/workflows/dependencies/clang14.sh index 18274d5e46c..27597e06c5d 100755 --- a/.github/workflows/dependencies/clang14.sh +++ b/.github/workflows/dependencies/clang14.sh @@ -15,7 +15,6 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries sudo apt-get -qqq update sudo apt-get install -y \ cmake \ - ccache \ clang-14 \ clang-tidy-14 \ libblas-dev \ @@ -29,6 +28,9 @@ sudo apt-get install -y \ libomp-dev \ ninja-build +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh index 9ecc5e4ca19..3b146405b4b 100755 --- a/.github/workflows/dependencies/dpcpp.sh +++ b/.github/workflows/dependencies/dpcpp.sh @@ -34,7 +34,6 @@ for itry in {1..5} do sudo apt-get install -y --no-install-recommends \ build-essential \ - ccache \ cmake \ intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \ g++ gfortran \ @@ -58,3 +57,6 @@ sudo rm -rf /opt/intel/oneapi/mkl/latest/lib/intel64/*.a \ du -sh /opt/intel/oneapi/ du -sh /opt/intel/oneapi/*/* df -h + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc.sh b/.github/workflows/dependencies/gcc.sh index d845463a682..fb2351f8a92 100755 --- a/.github/workflows/dependencies/gcc.sh +++ b/.github/workflows/dependencies/gcc.sh @@ -16,9 +16,11 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12.sh b/.github/workflows/dependencies/gcc12.sh index 01f30f07a31..fb46c6b811b 100755 --- a/.github/workflows/dependencies/gcc12.sh +++ b/.github/workflows/dependencies/gcc12.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -28,3 +27,6 @@ sudo apt-get install -y \ ninja-build \ pkg-config \ wget + +# ccache +$(dirname "$0")/ccache.sh diff --git a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh index 65f029ea67b..9ff3d615a3e 100755 --- a/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh +++ b/.github/workflows/dependencies/gcc12_blaspp_lapackpp.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ g++-12 \ gnupg \ @@ -30,6 +29,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2e7830368ca..2225e670bb0 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -41,7 +41,11 @@ sudo apt-get install -y --no-install-recommends \ rocm-dev \ rocfft-dev \ rocprim-dev \ - rocrand-dev + rocrand-dev \ + hiprand-dev + +# ccache +$(dirname "$0")/ccache.sh # activate # @@ -56,12 +60,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 5242a836a42..74f0db4b46a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -17,12 +17,14 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + # Ref.: https://github.com/rscohn2/oneapi-ci sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB diff --git a/.github/workflows/dependencies/nvcc11-3.sh b/.github/workflows/dependencies/nvcc11-3.sh index ed9fd211128..92e2717e425 100755 --- a/.github/workflows/dependencies/nvcc11-3.sh +++ b/.github/workflows/dependencies/nvcc11-3.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvcc11-8.sh b/.github/workflows/dependencies/nvcc11-8.sh index 33af2c1ade5..6089360392b 100755 --- a/.github/workflows/dependencies/nvcc11-8.sh +++ b/.github/workflows/dependencies/nvcc11-8.sh @@ -26,6 +26,9 @@ sudo apt-get install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb sudo dpkg -i cuda-keyring_1.0-1_all.deb @@ -55,12 +58,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvhpc.sh b/.github/workflows/dependencies/nvhpc.sh index 9c6166d386d..3533d6ca9f6 100755 --- a/.github/workflows/dependencies/nvhpc.sh +++ b/.github/workflows/dependencies/nvhpc.sh @@ -25,20 +25,23 @@ sudo apt install -y \ pkg-config \ wget +# ccache +$(dirname "$0")/ccache.sh + echo 'deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | \ sudo tee /etc/apt/sources.list.d/nvhpc.list sudo apt update -y && \ -sudo apt install -y --no-install-recommends nvhpc-21-11 && \ +sudo apt install -y --no-install-recommends nvhpc-24-1 && \ sudo rm -rf /var/lib/apt/lists/* && \ - sudo rm -rf /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/examples \ - /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/profilers \ - /opt/nvidia/hpc_sdk/Linux_x86_64/21.11/math_libs/11.5/targets/x86_64-linux/lib/lib*_static*.a + sudo rm -rf /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/examples \ + /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/profilers \ + /opt/nvidia/hpc_sdk/Linux_x86_64/24.1/math_libs/11.5/targets/x86_64-linux/lib/lib*_static*.a # things should reside in /opt/nvidia/hpc_sdk now # activation via: # source /etc/profile.d/modules.sh -# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.11 +# module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/24.1 # cmake-easyinstall # @@ -46,12 +49,3 @@ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.c sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" export CEI_TMP="/tmp/cei" - -# ccache 4.2+ -# -CXXFLAGS="" cmake-easyinstall --prefix=/usr/local \ - git+https://github.com/ccache/ccache.git@v4.6 \ - -DCMAKE_BUILD_TYPE=Release \ - -DENABLE_DOCUMENTATION=OFF \ - -DENABLE_TESTING=OFF \ - -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/pyfull.sh b/.github/workflows/dependencies/pyfull.sh index b0a4f607a51..12678763da5 100755 --- a/.github/workflows/dependencies/pyfull.sh +++ b/.github/workflows/dependencies/pyfull.sh @@ -16,7 +16,6 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ clang \ cmake \ gnupg \ @@ -35,6 +34,9 @@ sudo apt-get install -y \ python3-setuptools \ wget +# ccache +$(dirname "$0")/ccache.sh + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 40e5055cd8b..0e311f061ef 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -15,25 +15,25 @@ jobs: CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: .github/workflows/dependencies/hip.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-3dsp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-3dsp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -56,13 +56,16 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_PSATD=ON - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . python3 -m pip install *.whl + ccache -s + du -hs ~/.cache/ccache + build_hip_2d_dp: name: HIP 2D DP runs-on: ubuntu-20.04 @@ -71,25 +74,25 @@ jobs: CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: .github/workflows/dependencies/hip.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-hip-2ddp-${{ hashFiles('.github/workflows/hip.yml') }}- - ccache-hip-2ddp- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + source /etc/profile.d/rocm.sh hipcc --version which clang @@ -113,9 +116,27 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ -DWarpX_PSATD=ON - cmake --build build_2d -j 2 + cmake --build build_2d -j 4 export WARPX_MPI=OFF export PYWARPX_LIB_DIR=$PWD/build_2d/lib/site-packages/pywarpx/ python3 -m pip wheel . python3 -m pip install *.whl + + ccache -s + du -hs ~/.cache/ccache + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml index 57a25ce7629..42923d3df8e 100644 --- a/.github/workflows/insitu.yml +++ b/.github/workflows/insitu.yml @@ -20,7 +20,7 @@ jobs: container: image: senseiinsitu/ci:fedora35-amrex-20220613 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure run: | cmake -S . -B build \ @@ -28,7 +28,7 @@ jobs: -DWarpX_COMPUTE=NOACC - name: Build run: | - cmake --build build -j 2 + cmake --build build -j 4 ascent: name: Ascent @@ -41,7 +41,7 @@ jobs: container: image: alpinedav/ascent:0.9.2 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Configure run: | . /ascent_docker_setup_env.sh @@ -51,7 +51,7 @@ jobs: - name: Build run: | . /ascent_docker_setup_env.sh - cmake --build build -j 2 + cmake --build build -j 4 - name: Test run: | cp Examples/Physics_applications/laser_acceleration/inputs_3d . @@ -61,7 +61,7 @@ jobs: max_step = 40 \ diag1.intervals = 30:40:10 \ diag1.format = ascent - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 with: name: ascent-test-artifacts path: | diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index b4a8869d498..3b1d6b546a4 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -17,24 +17,25 @@ jobs: #env: # CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/icc.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + export CCACHE_DEPEND=1 + ccache -z + set +eu source /opt/intel/oneapi/setvars.sh set -eu @@ -52,7 +53,7 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF - cmake --build build_dp -j 2 + cmake --build build_dp -j 4 cmake -S . -B build_sp \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -63,9 +64,12 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + build_icpx: name: oneAPI ICX SP runs-on: ubuntu-20.04 @@ -78,26 +82,27 @@ jobs: # CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: | .github/workflows/dependencies/dpcpp.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-icpx-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-icpx- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -115,9 +120,12 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 cmake --build build_sp --target pip_install + ccache -s + du -hs ~/.cache/ccache + - name: run pywarpx run: | set +e @@ -138,26 +146,27 @@ jobs: # CMAKE_GENERATOR: Ninja if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies shell: bash run: | .github/workflows/dependencies/dpcpp.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-intel-dpcc-${{ hashFiles('.github/workflows/intel.yml') }}- - ccache-intel-dpcc- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX shell: bash run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + set +e source /opt/intel/oneapi/setvars.sh set -e @@ -175,10 +184,28 @@ jobs: -DWarpX_MPI=OFF \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE - cmake --build build_sp -j 2 + cmake --build build_sp -j 4 + + ccache -s + du -hs ~/.cache/ccache # Skip this as it will copy the binary artifacts and we are tight on disk space # python3 -m pip install --upgrade pip # python3 -m pip install --upgrade build packaging setuptools wheel # PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . # python3 -m pip install *.whl + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 68ea4737107..f34f9f3534d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -17,26 +17,13 @@ jobs: # For macOS, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 - - name: Brew Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key - with: - path: | - /usr/local/bin - /usr/local/lib - /usr/local/share - /Users/runner/Library/Caches/Homebrew - key: brew-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }} - restore-keys: | - brew-macos-appleclang- + - uses: actions/checkout@v4 - name: install dependencies run: | - brew --cache set +e brew unlink gcc brew update + brew upgrade || true brew install --overwrite python brew install ccache brew install fftw @@ -58,17 +45,20 @@ jobs: python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade mpi4py - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: path: /Users/runner/Library/Caches/ccache - key: ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}- - ccache-macos-appleclang- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + export CCACHE_DEPEND=1 + ccache -z + source py-venv/bin/activate cmake -S . -B build_dp \ @@ -89,9 +79,26 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + ccache -s + - name: run pywarpx run: | source py-venv/bin/activate export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml new file mode 100644 index 00000000000..2768ef376cc --- /dev/null +++ b/.github/workflows/post-pr.yml @@ -0,0 +1,20 @@ +name: PostPR +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml index 08050768894..a1c29416b3e 100644 --- a/.github/workflows/source.yml +++ b/.github/workflows/source.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Non-ASCII characters run: .github/workflows/source/hasNonASCII - name: TABs diff --git a/.github/workflows/source/makeMakefileForClangTidy.py b/.github/workflows/source/makeMakefileForClangTidy.py new file mode 100755 index 00000000000..07809187dbd --- /dev/null +++ b/.github/workflows/source/makeMakefileForClangTidy.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +""" +A script processing ccache log file to make Make file for Clang-Tidy + +This generates a makefile for clang-tidying ccache-cache-missing files. This +could be used to speed up clang-tidy in CIs. +""" + +import argparse +import re +import sys + + +def makeMakefileForClangTidy(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--input", + help="Ccache log file", + default="ccache.log.txt") + parser.add_argument("--identifier", + help="Unique identifier for finding compilation line in the log file", + default="WarpX/Source") + # We assume WarpX/Source can be used as an identifier to distinguish + # WarpX code from amrex, openMPD, and cmake's temporary files like + # build/CMakeFiles/CMakeScratch/TryCompile-hw3x4m/test_mpi.cpp + parser.add_argument("--output", + help="Make file for clang-tidy", + default="clang-tidy-ccache-misses.mak") + args = parser.parse_args() + + fin = open(args.input, "r") + fout = open(args.output, "w") + + fout.write("CLANG_TIDY ?= clang-tidy\n") + fout.write("override CLANG_TIDY_ARGS += --extra-arg=-Wno-unknown-warning-option --extra-arg-before=--driver-mode=g++\n") + fout.write("\n") + + fout.write(".SECONDEXPANSION:\n") + fout.write("clang-tidy: $$(all_targets)\n") + fout.write("\t@echo SUCCESS\n\n") + + exe_re = re.compile(r" Executing .*? (-.*{}.*) -c .* -o .* (\S*)".format(args.identifier)) + + count = 0 + for line in fin.readlines(): + ret_exe_re = exe_re.search(line) + if (ret_exe_re): + fout.write("target_{}: {}\n".format(count, ret_exe_re.group(2))) + fout.write("\t$(CLANG_TIDY) $(CLANG_TIDY_ARGS) $< -- {}\n".format + (ret_exe_re.group(1))) + fout.write("\ttouch target_{}\n\n".format(count)) + count = count + 1 + + fout.write("all_targets =") + for i in range(count): + fout.write(" target_{}".format(i)) + fout.write("\n\n") + + fout.write("clean:\n\t$(RM) $(all_targets)\n\n") + + fout.close() + fin.close() + +if __name__ == "__main__": + makeMakefileForClangTidy(sys.argv) diff --git a/.github/workflows/source/test_ci_matrix.sh b/.github/workflows/source/test_ci_matrix.sh index e77d5ce2685..62903b66e45 100755 --- a/.github/workflows/source/test_ci_matrix.sh +++ b/.github/workflows/source/test_ci_matrix.sh @@ -7,26 +7,26 @@ cd Regression/ # Put the name of all CI tests into a text file python prepare_file_ci.py -grep "\[" ci-tests.ini > ci_all_tests.txt +grep "^\[" ci-tests.ini > ci_all_tests.txt export WARPX_CI_PSATD=TRUE # Concatenate the names of all elements in CI matrix into another test file WARPX_CI_REGULAR_CARTESIAN_1D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini > ci_matrix_elements.txt +grep "^\[" ci-tests.ini > ci_matrix_elements.txt WARPX_CI_REGULAR_CARTESIAN_2D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_REGULAR_CARTESIAN_3D=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_SINGLE_PRECISION=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_RZ_OR_NOMPI=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_QED=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt WARPX_CI_EB=TRUE python prepare_file_ci.py -grep "\[" ci-tests.ini >> ci_matrix_elements.txt +grep "^\[" ci-tests.ini >> ci_matrix_elements.txt # Check that the resulting lists are equal { diff --git a/.github/workflows/source/wrongFileNameInExamples b/.github/workflows/source/wrongFileNameInExamples index e116b9df084..0de69d69c9c 100755 --- a/.github/workflows/source/wrongFileNameInExamples +++ b/.github/workflows/source/wrongFileNameInExamples @@ -18,6 +18,7 @@ do [[ ${file:0:12} != PICMI_inputs ]] && [[ ${file:0:8 } != analysis ]] && [[ ${file: -4} != yaml ]] && + [[ ${file:0:4 } != plot ]] && [[ ${file:0:6 } != README ]] then files+=($file) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 8fa83c99bef..cf4b375ce00 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -14,34 +14,37 @@ jobs: env: CXXFLAGS: "-Werror" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-cxxminimal-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-cxxminimal- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ -DWarpX_MPI=OFF \ -DWarpX_QED=OFF - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_1D_2D: name: GCC 1D & 2D w/ MPI runs-on: ubuntu-22.04 @@ -51,24 +54,24 @@ jobs: CXX: "g++-12" CC: "gcc-12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc12.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-1D-2D-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-1D-2D- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -76,10 +79,13 @@ jobs: -DWarpX_EB=OFF \ -DWarpX_PSATD=ON \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_1d ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_2d + ccache -s + du -hs ~/.cache/ccache + build_3D_sp: name: GCC 3D & RZ w/ MPI, single precision runs-on: ubuntu-22.04 @@ -88,24 +94,23 @@ jobs: CXX: "g++-12" CC: "gcc-12" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc12_blaspp_lapackpp.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-3D-RZ-SP-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-3D-RZ-SP- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=200M + ccache -z # we need to define this *after* having installed the dependencies, # because the compilation of blaspp raises warnings. @@ -121,10 +126,13 @@ jobs: -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 + cmake --build build -j 4 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_3d ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_rz + ccache -s + du -hs ~/.cache/ccache + build_gcc_ablastr: name: GCC ABLASTR w/o MPI runs-on: ubuntu-20.04 @@ -133,30 +141,33 @@ jobs: CMAKE_GENERATOR: Ninja CXXFLAGS: "-Werror" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/gcc.sh sudo apt-get install -y libopenmpi-dev openmpi-bin - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-gccablastr-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-gccablastr- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ -DWarpX_LIB=OFF - cmake --build build -j 2 + cmake --build build -j 4 + + ccache -s + du -hs ~/.cache/ccache build_pyfull: name: Clang pywarpx @@ -168,24 +179,24 @@ jobs: # On CI for this test, Ninja is slower than the default: #CMAKE_GENERATOR: Ninja steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install dependencies run: | .github/workflows/dependencies/pyfull.sh - name: CCache Cache - uses: actions/cache@v3 - # - once stored under a key, they become immutable (even if local cache path content changes) - # - for a refresh the key has to change, e.g., hash of a tracked file in the key + uses: actions/cache@v4 with: - path: | - ~/.ccache - ~/.cache/ccache - key: ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + path: ~/.cache/ccache + key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | - ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}- - ccache-openmp-pyfull- + ccache-${{ github.workflow }}-${{ github.job }}-git- - name: build WarpX run: | + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=10 + export CCACHE_MAXSIZE=100M + ccache -z + python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel @@ -197,9 +208,27 @@ jobs: -DWarpX_PSATD=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON - cmake --build build -j 2 --target pip_install + cmake --build build -j 4 --target pip_install + + ccache -s + du -hs ~/.cache/ccache - name: run pywarpx run: | export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py + + save_pr_number: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Save PR number + env: + PR_NUMBER: ${{ github.event.number }} + run: | + echo $PR_NUMBER > pr_number.txt + - uses: actions/upload-artifact@v4 + with: + name: pr_number + path: pr_number.txt + retention-days: 1 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 8e2bb00f1db..2ef74cdb7f9 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -12,12 +12,12 @@ jobs: runs-on: windows-latest if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 # - once stored under a key, they become immutable (even if local cache path content changes) # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: @@ -38,7 +38,7 @@ jobs: -DWarpX_MPI=OFF ` -DWarpX_PYTHON=ON if(!$?) { Exit $LASTEXITCODE } - cmake --build build --config Debug --parallel 2 + cmake --build build --config Debug --parallel 4 if(!$?) { Exit $LASTEXITCODE } python3 -m pip install --upgrade pip @@ -63,13 +63,13 @@ jobs: runs-on: windows-2019 if: github.event.pull_request.draft == false steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - uses: seanmiddleditch/gha-setup-ninja@master - name: CCache Cache - uses: actions/cache@v3 + uses: actions/cache@v4 # - once stored under a key, they become immutable (even if local cache path content changes) # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: @@ -96,7 +96,7 @@ jobs: -DWarpX_MPI=OFF ^ -DWarpX_OPENPMD=ON if errorlevel 1 exit 1 - cmake --build build --config Release --parallel 2 + cmake --build build --config Release --parallel 4 if errorlevel 1 exit 1 cmake --build build --config Release --target install diff --git a/.gitignore b/.gitignore index d614c1ae2d9..4fa65351331 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,7 @@ Docs/warpx-doxygen-web.tag.xml Docs/openpmd-api-doxygen-web.tag.xml Docs/doxyhtml/ Docs/doxyxml/ -Docs/source/_static/ +Docs/source/_static/doxyhtml #################### # Package Managers # diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb298d39b68..f3d14c123c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,7 @@ repos: # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.4 + rev: v1.5.5 hooks: - id: remove-tabs exclude: 'Make.WarpX|Make.package|Makefile|GNUmake' @@ -76,7 +76,7 @@ repos: # Sorts Python imports according to PEP8 # https://www.python.org/dev/peps/pep-0008/#imports - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) diff --git a/CMakeLists.txt b/CMakeLists.txt index 462bfaf17bc..bb8c7b2c59d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 23.11) +project(WarpX VERSION 24.02) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -43,7 +43,15 @@ set_cxx17_superbuild() # this is an optional tool that stores compiled object files; allows fast # re-builds even with "make clean" in between. Mainly used to store AMReX # objects -set_ccache() +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(WarpX_CCACHE_DEFAULT ON) +else() + set(WarpX_CCACHE_DEFAULT OFF) # we are a subproject in a superbuild +endif() +option(WarpX_CCACHE "Enable ccache for faster rebuilds" ${WarpX_CCACHE_DEFAULT}) +if(WarpX_CCACHE) + set_ccache() +endif() # Output Directories ########################################################## @@ -149,6 +157,7 @@ endif() # this defined the variable BUILD_TESTING which is ON by default #include(CTest) + # Dependencies ################################################################ # diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index bb3a25e0daf..a3a6a3046a9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -117,7 +117,7 @@ A typical format is: This is a short, 40-character title - After a newline, you can write arbitray paragraphs. You + After a newline, you can write arbitrary paragraphs. You usually limit the lines to 70 characters, but if you don't, then nothing bad will happen. diff --git a/Docs/requirements.txt b/Docs/requirements.txt index 86235035252..b16bf161144 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -9,12 +9,15 @@ breathe docutils>=0.17.1 +openpmd-viewer # for checksumAPI + # PICMI API docs # note: keep in sync with version in ../requirements.txt picmistandard==0.28.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python +pybtex pygments recommonmark # Sphinx<7.2 because we are waiting for @@ -25,3 +28,4 @@ sphinx-design sphinx_rtd_theme>=1.1.1 sphinxcontrib-bibtex sphinxcontrib-napoleon +yt # for checksumAPI diff --git a/Docs/source/_static/custom.css b/Docs/source/_static/custom.css new file mode 100644 index 00000000000..b31c9188d76 --- /dev/null +++ b/Docs/source/_static/custom.css @@ -0,0 +1,6 @@ +.eqno { + /* As of 2023 Dec, sphinx_rtd_theme has a bug where equation numbers appear above the equations instead of on the right */ + /* The following is a make-shift fix, but comes at the cost of "making the header link invisible," though I'm not sure what that means */ + /* Copied from https://github.com/readthedocs/sphinx_rtd_theme/issues/301#issuecomment-1205755904 */ + float: right; +} diff --git a/Docs/source/conf.py b/Docs/source/conf.py index f6fa22f40f8..b34c437b829 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -30,9 +30,11 @@ import sys import urllib.request +import pybtex.plugin +from pybtex.style.formatting.unsrt import Style as UnsrtStyle import sphinx_rtd_theme -sys.path.insert(0, os.path.join( os.path.abspath(__file__), '../Python') ) +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../Regression/Checksum')) # -- General configuration ------------------------------------------------ @@ -43,7 +45,8 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', @@ -57,8 +60,29 @@ templates_path = ['_templates'] # Relative path to bibliography file, bibliography style -bibtex_bibfiles = ['./refs.bib'] -bibtex_default_style = 'unsrt' +bibtex_bibfiles = ['latex_theory/allbibs.bib', 'refs.bib'] + +# An brief introduction to custom BibTex formatting can be found in the Sphinx documentation: +# https://sphinxcontrib-bibtex.readthedocs.io/en/latest/usage.html#bibtex-custom-formatting +# +# More details can be gleaned from looking at the pybtex dist-package files. +# Some examples include the following: +# BaseStyle class in pybtex/style/formatting/__init__.py +# UnsrtStyle class in pybtex/style/formating/unsrt.py +class WarpXBibStyle(UnsrtStyle): + # This option makes the family name, i.e, "last" name, of an author to appear first. + # default_name_style = 'lastfirst' + + def __init__(self, *args, **kwargs): + # This option makes the given names of an author abbreviated to just initials. + # Example: "Jean-Luc" becomes "J.-L." + # Set 'abbreviate_names' to True before calling the superclass (BaseStyle class) initializer + kwargs['abbreviate_names'] = True + super().__init__(*args, **kwargs) + +pybtex.plugin.register_plugin('pybtex.style.formatting', 'warpxbibstyle', WarpXBibStyle) + +bibtex_default_style = 'warpxbibstyle' # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -79,9 +103,9 @@ # built documents. # # The short X.Y version. -version = u'23.11' +version = u'24.02' # The full version, including alpha/beta/rc tags. -release = u'23.11' +release = u'24.02' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -110,6 +134,11 @@ html_theme = 'sphinx_rtd_theme' numfig = True +math_eqref_format = "{number}" +numfig_format = {'figure': 'Fig. %s', + 'table': 'Table %s', + 'code-block': 'Listing %s', + 'section': 'Section %s'} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -122,6 +151,9 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +html_css_files = [ + 'custom.css', +] # -- Options for HTMLHelp output ------------------------------------------ diff --git a/Docs/source/dataanalysis/ascent.rst b/Docs/source/dataanalysis/ascent.rst index 1fbeeb043e2..6748b111a2f 100644 --- a/Docs/source/dataanalysis/ascent.rst +++ b/Docs/source/dataanalysis/ascent.rst @@ -274,7 +274,7 @@ Example Actions A visualization of the electric field component :math:`E_x` (variable: ``Ex``) with a contour plot and with added particles can be obtained with the following Ascent Action. This action can be used both in replay as well as in situ runs. -.. literalinclude:: examples/Physics_applications/laser_acceleration/ascent_actions.yaml +.. literalinclude:: ../../../Examples/Physics_applications/laser_acceleration/ascent_actions.yaml :language: yaml There are more `Ascent Actions examples available `_ for you to play. diff --git a/Docs/source/dataanalysis/examples b/Docs/source/dataanalysis/examples deleted file mode 120000 index 76b45c18c59..00000000000 --- a/Docs/source/dataanalysis/examples +++ /dev/null @@ -1 +0,0 @@ -../../../Examples \ No newline at end of file diff --git a/Docs/source/dataanalysis/openpmdviewer.rst b/Docs/source/dataanalysis/openpmdviewer.rst index efb5acb4e50..2e6f5c083d7 100644 --- a/Docs/source/dataanalysis/openpmdviewer.rst +++ b/Docs/source/dataanalysis/openpmdviewer.rst @@ -27,13 +27,7 @@ Usage ----- openPMD-viewer can be used either in simple Python scripts or in `Jupyter `__. -For interactive plots in Jupyter notebook, add this `"cell magic" `__ to the first line of your notebook: - -.. code-block:: python - - %matplotlib notebook - -and for Jupyter Lab use this instead: +For interactive plots in Jupyter notebook or Jupyter Lab, add this `"cell magic" `__ to the first line of your notebook: .. code-block:: python diff --git a/Docs/source/dataanalysis/sensei.rst b/Docs/source/dataanalysis/sensei.rst index 16826c3082c..5e4bfff10fd 100644 --- a/Docs/source/dataanalysis/sensei.rst +++ b/Docs/source/dataanalysis/sensei.rst @@ -30,7 +30,7 @@ and bridge code making it easy to use in AMReX based simulation codes. SENSEI provides a *configurable analysis adaptor* which uses an XML file to select and configure one or more back ends at run time. Run time selection of the back end via XML means one user can access Catalyst, another Libsim, yet -another Python with no changes to the code. This is depicted in figure +another Python with no changes to the code. This is depicted in :numref:`sensei_arch`. On the left side of the figure AMReX produces data, the bridge code pushes the data through the configurable analysis adaptor to the back end that was selected at run time. @@ -99,7 +99,7 @@ The back end is selected and configured at run time using the SENSEI XML file. The XML sets parameters specific to SENSEI and to the chosen back end. Many of the back ends have sophisticated configuration mechanisms which SENSEI makes use of. For example the following XML configuration was used on NERSC's Cori -with WarpX to render 10 iso surfaces, shown in figure :numref:`lpa_visit`, using +with WarpX to render 10 iso surfaces, shown in :numref:`lpa_visit`, using VisIt Libsim. .. code-block:: xml @@ -123,7 +123,7 @@ run of the desired simulation. quadrant has been clipped away to reveal innner structure. The same run and visualization was repeated using ParaView Catalyst, shown in -figure :numref:`lpa_pv`, by providing the following XML configuration. +:numref:`lpa_pv`, by providing the following XML configuration. .. code-block:: xml diff --git a/Docs/source/developers/amrex_basics.rst b/Docs/source/developers/amrex_basics.rst index 577a6547bb5..64ad71af06c 100644 --- a/Docs/source/developers/amrex_basics.rst +++ b/Docs/source/developers/amrex_basics.rst @@ -13,7 +13,7 @@ WarpX is built on the Adaptive Mesh Refinement (AMR) library `AMReX --test-name + ./checksumAPI.py --evaluate --output-file --output-format <'openpmd' or 'plotfile'> --test-name See additional options @@ -41,17 +41,17 @@ See additional options * ``--rtol`` relative tolerance for the comparison * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``) -Reset a benchmark with new values that you know are correct ------------------------------------------------------------ +Create/Reset a benchmark with new values that you know are correct +------------------------------------------------------------------ -Reset a benchmark from a plotfile generated locally -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Create/Reset a benchmark from a plotfile generated locally +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This is using ``checksumAPI.py`` as a Python script. .. code-block:: bash - ./checksumAPI.py --reset-benchmark --plotfile --test-name + ./checksumAPI.py --reset-benchmark --output-file --output-format <'openpmd' or 'plotfile'> --test-name See additional options diff --git a/Docs/source/developers/dimensionality.rst b/Docs/source/developers/dimensionality.rst index e775d31a6df..52493e91e38 100644 --- a/Docs/source/developers/dimensionality.rst +++ b/Docs/source/developers/dimensionality.rst @@ -49,12 +49,12 @@ WarpX axis labels ``x, y, z`` ``x, z`` ``z`` ``x, z`` -------------------- ----------- ----------- ----------- ----------- *Particles* ------------------------------------------------------------------------ -AMReX AoS ``.pos()`` ``0, 1, 2`` ``0, 1`` ``0`` ``0, 1`` +AMReX ``.pos()`` ``0, 1, 2`` ``0, 1`` ``0`` ``0, 1`` WarpX position names ``x, y, z`` ``x, z`` ``z`` ``r, z`` extra SoA attribute ``theta`` ==================== =========== =========== =========== =========== -Please see the following sections for particle AoS and SoA details. +Please see the following sections for particle SoA details. Conventions ----------- diff --git a/Docs/source/developers/fields.rst b/Docs/source/developers/fields.rst index 9ed790e4562..d0af160afef 100644 --- a/Docs/source/developers/fields.rst +++ b/Docs/source/developers/fields.rst @@ -112,9 +112,9 @@ Bilinear filter The multi-pass bilinear filter (applied on the current density) is implemented in ``Source/Filter/``, and class ``WarpX`` holds an instance of this class in member variable ``WarpX::bilinear_filter``. For performance reasons (to avoid creating too many guard cells), this filter is directly applied in communication routines, see ``WarpX::AddCurrentFromFineLevelandSumBoundary`` above and -.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim) +.. doxygenfunction:: WarpX::ApplyFilterJ(const amrex::Vector, 3>> ¤t, int lev, int idim) -.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, const int lev, const int idim, const amrex::Periodicity &period) +.. doxygenfunction:: WarpX::SumBoundaryJ(const amrex::Vector, 3>> ¤t, int lev, int idim, const amrex::Periodicity &period) Godfrey's anti-NCI filter for FDTD simulations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Docs/source/developers/local_compile.rst b/Docs/source/developers/local_compile.rst new file mode 100644 index 00000000000..8bfa033a92d --- /dev/null +++ b/Docs/source/developers/local_compile.rst @@ -0,0 +1,138 @@ +.. _developers-local-compile: + +Fast, Local Compilation +======================= + +For simplicity, WarpX :ref:`compilation with CMake ` by default downloads, configures and compiles compatible versions of :ref:`central dependencies ` such as: + +* `AMReX `__ +* `PICSAR `__ +* `openPMD-api `__ +* `pyAMReX `__ +* `pybind11 `__ + +on-the-fly, which is called a *superbuild*. + +In some scenarios, e.g., when compiling without internet, with slow internet access, or when working on WarpX and its dependencies, modifications to the superbuild strategy might be preferable. +In the below workflows, you as the developer need to make sure to use compatible versions of the dependencies you provide. + + +.. _developers-local-compile-src: + +Compiling From Local Sources +---------------------------- + +This workflow is best for developers that make changes to WarpX, AMReX, PICSAR, openPMD-api and/or pyAMReX at the same time. +For instance, use this if you add a feature in AMReX and want to try it in WarpX before it is proposed as a pull request for inclusion in AMReX. + +Instead of downloading the source code of the above dependencies, one can also use an already cloned source copy. +For instance, clone these dependencies to ``$HOME/src``: + +.. code-block:: bash + + cd $HOME/src + + git clone https://github.com/ECP-WarpX/WarpX.git warpx + git clone https://github.com/AMReX-Codes/amrex.git + git clone https://github.com/openPMD/openPMD-api.git + git clone https://github.com/ECP-WarpX/picsar.git + git clone https://github.com/AMReX-Codes/pyamrex.git + git clone https://github.com/pybind/pybind11.git + +Now modify the dependencies as needed in their source locations, update sources if you cloned them earlier, etc. +When building WarpX, :ref:`the following CMake flags ` will use the respective local sources: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_src=$HOME/src/amrex \ + -DWarpX_openpmd_src=$HOME/src/openPMD-api \ + -DWarpX_picsar_src=$HOME/src/picsar \ + -DWarpX_pyamrex_src=$HOME/src/pyamrex \ + -DWarpX_pybind11_src=$HOME/src/pybind11 + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + + +.. _developers-local-compile-findpackage: + +Compiling With Pre-Compiled Dependencies +---------------------------------------- + +This workflow is the best and fastest to compile WarpX, when you just want to change code in WarpX and have the above central dependencies already made available *in the right configurations* (e.g., w/ or w/o MPI or GPU support) from a :ref:`module system ` or :ref:`package manager `. + +Instead of downloading the source code of the above central dependencies, or using a local copy of their source, we can compile and install those dependencies once. +By setting the `CMAKE_PREFIX_PATH `__ environment variable to the respective dependency install location prefixes, we can instruct CMake to `find their install locations and configurations `__. + +WarpX supports this with :ref:`the following CMake flags `: + +.. code-block:: bash + + cd src/warpx + + rm -rf build + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_amrex_internal=OFF \ + -DWarpX_openpmd_internal=OFF \ + -DWarpX_picsar_internal=OFF \ + -DWarpX_pyamrex_internal=OFF \ + -DWarpX_pybind11_internal=OFF + + cmake --build build -j 8 + cmake --build build -j 8 --target pip_install + +As a background, this is also the workflow how WarpX is built in :ref:`package managers such as Spack and Conda-Forge `. + + +.. _developers-local-compile-pylto: + +Faster Python Builds +-------------------- + +The Python bindings of WarpX and AMReX (pyAMReX) use `pybind11 `__. +Since pybind11 relies heavily on `C++ metaprogramming `__, speeding up the generated binding code requires that we perform a `link-time optimization (LTO) `__ step, also known as `interprocedural optimization (IPO) `__. + +For fast local development cycles, one can skip LTO/IPO with the following flags: + +.. code-block:: bash + + cd src/warpx + + cmake -S . -B build \ + -DWarpX_PYTHON=ON \ + -DWarpX_PYTHON_IPO=OFF \ + -DpyAMReX_IPO=OFF + + cmake --build build -j 8 --target pip_install + +.. note:: + + We might transition to `nanobind `__ in the future, which `does not rely on LTO/IPO `__ for optimal binaries. + You can contribute to `this pyAMReX pull request `__ to help exploring this library (and if it works for the HPC/GPU compilers that we need to support). + +For robustness, our ``pip_install`` target performs a regular ``wheel`` build and then installs it with ``pip``. +This step will check every time of WarpX dependencies are properly installed, to avoid broken installations. +When developing without internet or after the first ``pip_install`` succeeded in repeated installations in rapid development cycles, this check of ``pip`` can be skipped by using the ``pip_install_nodeps`` target instead: + +.. code-block:: bash + + cmake --build build -j 8 --target pip_install_nodeps + + +.. _developers-local-compile-ccache: + +CCache +------ + +WarpX builds will automatically search for `CCache `__ to speed up subsequent compilations in development cycles. +Make sure a :ref:`recent CCache version ` is installed to make use of this feature. + +For power developers that switch a lot between fundamentally different WarpX configurations (e.g., 1D to 3D, GPU and CPU builds, many branches with different bases, developing AMReX and WarpX at the same time), also consider increasing the `CCache cache size `__ and changing the `cache directory `__ if needed, e.g., due to storage quota constraints or to choose a fast(er) filesystem for the cache files. diff --git a/Docs/source/developers/particles.rst b/Docs/source/developers/particles.rst index e9ad7754771..53a0090b5c9 100644 --- a/Docs/source/developers/particles.rst +++ b/Docs/source/developers/particles.rst @@ -36,17 +36,14 @@ A typical loop over particles reads: } } -The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. For this reasons, the particle data needs to be converted in plain-old-data structures. The innermost loop in the code snippet above could look like: +The innermost step ``[MY INNER LOOP]`` typically calls ``amrex::ParallelFor`` to perform operations on all particles in a portable way. The innermost loop in the code snippet above could look like: .. code-block:: cpp - // Get Array-Of-Struct particle data, also called data - // (x, y, z, id, cpu) - const auto& particles = pti.GetArrayOfStructs(); // Get Struct-Of-Array particle data, also called attribs - // (ux, uy, uz, w, Exp, Ey, Ez, Bx, By, Bz) + // (x, y, z, ux, uy, uz, w) auto& attribs = pti.GetAttribs(); - auto& Exp = attribs[PIdx::Ex]; + auto& x = attribs[PIdx::x]; // [...] // Number of particles in this box const long np = pti.numParticles(); @@ -66,7 +63,6 @@ On a loop over boxes in a ``MultiFab`` (``MFIter``), it can be useful to access const int tile_id = mfi.LocalTileIndex(); // Get GPU-friendly arrays of particle data auto& ptile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; - ParticleType* pp = particle_tile.GetArrayOfStructs()().data(); // Only need attribs (i.e., SoA data) auto& soa = ptile.GetStructOfArrays(); // As an example, let's get the ux momentum @@ -87,7 +83,7 @@ Main functions .. doxygenfunction:: PhysicalParticleContainer::PushPX -.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, const amrex::Real dt, const amrex::Real relative_time) +.. doxygenfunction:: WarpXParticleContainer::DepositCurrent(amrex::Vector, 3>> &J, amrex::Real dt, amrex::Real relative_time) .. note:: The current deposition is used both by ``PhysicalParticleContainer`` and ``LaserParticleContainer``, so it is in the parent class ``WarpXParticleContainer``. @@ -109,24 +105,41 @@ Particle attributes ------------------- WarpX adds the following particle attributes by default to WarpX particles. -These attributes are either stored in an Array-of-Struct (AoS) or Struct-of-Array (SoA) location of the AMReX particle containers. +These attributes are stored in Struct-of-Array (SoA) locations of the AMReX particle containers: one SoA for ``amrex::ParticleReal`` attributes, one SoA for ``int`` attributes and one SoA for a ``uint64_t`` global particle index per particle. The data structures for those are either pre-described at compile-time (CT) or runtime (RT). -==================== ================ ================================== ===== ==== ===================== +==================== ================ ================================== ===== ==== ====================== Attribute name ``int``/``real`` Description Where When Notes -==================== ================ ================================== ===== ==== ===================== -``position_x/y/z`` ``real`` Particle position. AoS CT -``cpu`` ``int`` CPU index where the particle AoS CT +==================== ================ ================================== ===== ==== ====================== +``position_x/y/z`` ``real`` Particle position. SoA CT +``weight`` ``real`` Particle position. SoA CT +``momentum_x/y/z`` ``real`` Particle position. SoA CT +``id`` ``amrex::Long`` CPU-local particle index SoA CT First 40 bytes of + where the particle was created. idcpu +``cpu`` ``int`` CPU index where the particle SoA CT Last 24 bytes of idcpu was created. -``id`` ``int`` CPU-local particle index AoS CT - where the particle was created. +``stepScraped`` ``int`` PIC iteration of the last step SoA RT Added when there is + before the particle hits the particle-boundary + boundary. interaction. + Saved in the boundary + buffers. +``deltaTimeScraped`` ``real`` Difference of time between the SoA RT Added when there is + ``stepScraped`` and the exact time particle-boundary + when the particle hits the interaction. + boundary. Saved in the boundary + buffers. +``n_x/y/z`` ``real`` Normal components to the boundary SoA RT Added when there is + on the position where the particle particle-boundary + hits the boundary. interaction. + Saved in the boundary + buffers. ``ionizationLevel`` ``int`` Ion ionization level SoA RT Added when ionization physics is used. ``opticalDepthQSR`` ``real`` QED: optical depth of the Quantum- SoA RT Added when PICSAR QED Synchrotron process physics is used. ``opticalDepthBW`` ``real`` QED: optical depth of the Breit- SoA RT Added when PICSAR QED Wheeler process physics is used. -==================== ================ ================================== ===== ==== ===================== +==================== ================ ================================== ===== ==== ====================== WarpX allows extra runtime attributes to be added to particle containers (through ``AddRealComp("attrname")`` or ``AddIntComp("attrname")``). The attribute name can then be used to access the values of that attribute. diff --git a/Docs/source/developers/profiling.rst b/Docs/source/developers/profiling.rst index 8fe68fac817..5acea786920 100644 --- a/Docs/source/developers/profiling.rst +++ b/Docs/source/developers/profiling.rst @@ -121,7 +121,7 @@ Perlmutter Example """""""""""""""""" Example on how to create traces on a multi-GPU system that uses the Slurm scheduler (e.g., NERSC's Perlmutter system). -You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. +You can either run this on an interactive node or use the Slurm batch script header :ref:`documented here `. .. code-block:: bash @@ -247,7 +247,7 @@ node, or without a workload management system. .. note:: To collect full statistics, Nsight-Compute reruns kernels, - temporarilly saving device memory in host memory. This makes it + temporarily saving device memory in host memory. This makes it slower than Nsight-Systems, so the provided script profiles only a single step of a single process. This is generally enough to extract relevant information. diff --git a/Docs/source/developers/python.rst b/Docs/source/developers/python.rst index aa58b0a398a..108f957c2f3 100644 --- a/Docs/source/developers/python.rst +++ b/Docs/source/developers/python.rst @@ -56,10 +56,10 @@ This is part of the standard - each of the PICMI classes call the method ``handl The main purpose is to process application specific keyword arguments (those that start with ``warpx_`` for example). These are then passed into the ``init`` methods. In the WarpX implementation, in the ``init``, each of the WarpX specific arguments are saved as attributes of the implementation -class instancles. +class instances. It is in the second method, ``initialize_inputs``, where the PICMI input parameters are translated into WarpX input parameters. -This method is called later during the intialization. +This method is called later during the initialization. The prefix instances described above are all accessible in the implementation classes (via the ``pywarpx`` module). For each PICMI input quantity, the appropriate WarpX input parameters are set in the prefix classes. As needed, for example in the ``Species`` class, the dynamic prefix instances are created and the attributes set. diff --git a/Docs/source/developers/repo_organization.rst b/Docs/source/developers/repo_organization.rst index 5eaed5fcce5..cb450cc4c95 100644 --- a/Docs/source/developers/repo_organization.rst +++ b/Docs/source/developers/repo_organization.rst @@ -41,7 +41,7 @@ All WarpX header files need to be specified relative to the ``Source/`` director By default, in a ``MyName.cpp`` source file we do not include headers already included in ``MyName.H``. Besides this exception, if a function or a class is used in a source file, the header file containing its declaration must be included, unless the inclusion of a facade header is more appropriate. This is sometimes the case for AMReX headers. For instance ``AMReX_GpuLaunch.H`` is a façade header for ``AMReX_GpuLaunchFunctsC.H`` and ``AMReX_GpuLaunchFunctsG.H``, which -contain respectively the CPU and the GPU implemetation of some methods, and which should not be included directly. +contain respectively the CPU and the GPU implementation of some methods, and which should not be included directly. Whenever possible, forward declarations headers are included instead of the actual headers, in order to save compilation time (see dedicated section below). In WarpX forward declaration headers have the suffix ``*_fwd.H``, while in AMReX they have the suffix ``*Fwd.H``. The include order (see `PR #874 `__ and `PR #1947 `__) and `proper quotation marks `__ are: diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 227c2fd97b0..2b3fe989f2f 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -18,14 +18,14 @@ We slightly modify this file in ``Regression/prepare_file_ci.py``. For example, if you like to change the compiler to compilation to build on Nvidia GPUs, modify this block to add ``-DWarpX_COMPUTE=CUDA``: -.. code-block:: toml +.. code-block:: ini [source] dir = /home/regtester/AMReX_RegTesting/warpx branch = development cmakeSetupOpts = -DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON -DWarpX_COMPUTE=CUDA -We also support changing compilation options via the usual :ref:`build enviroment variables `. +We also support changing compilation options via the usual :ref:`build environment variables `. For instance, compiling with ``clang++ -Werror`` would be: .. code-block:: sh diff --git a/Docs/source/developers/warning_logger.rst b/Docs/source/developers/warning_logger.rst index 057aa21bb8e..d878b042502 100644 --- a/Docs/source/developers/warning_logger.rst +++ b/Docs/source/developers/warning_logger.rst @@ -47,15 +47,15 @@ Each entry of warning list respects the following format: .. code-block:: sh * --> [PRIORITY] [TOPIC] [raised COUNTER] - * MULTILINE MESSAGGE - * MULTILINE MESSAGGE + * MULTILINE MESSAGE + * MULTILINE MESSAGE * @ Raised by: WHICH_RANKS where: * ``[PRIORITY]`` can be ``[! ]`` (low priority), ``[!! ]`` (medium priority) or ``[!!!]`` (high priority). It indicates the importance of the warning. * ``[TOPIC]`` indicates which part of the code is concerned by the warning (e.g., particles, laser, parallelization...) -* ``MULTILINE MESSAGGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. +* ``MULTILINE MESSAGE`` is an arbitrary text message. It can span multiple-lines. Text is wrapped automatically. * ``COUNTER`` indicates the number of times the warning was raised **across all the MPI ranks**. This means that if we run WarpX with 2048 MPI ranks and each rank raises the same warning once, the displayed message will be ``[raised 2048 times]``. Possible values are ``once``, ``twice``, ``XX times`` * ``WHICH_RANKS`` can be either ``ALL`` or a sequence of rank IDs. It is the list of the MPI ranks which have raised the warning message. diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst index d56974b4101..8b357f5d4da 100644 --- a/Docs/source/developers/workflows.rst +++ b/Docs/source/developers/workflows.rst @@ -10,3 +10,4 @@ Workflows testing documentation checksum + local_compile diff --git a/Docs/source/glossary.rst b/Docs/source/glossary.rst index b78d2446de8..000b8354e15 100644 --- a/Docs/source/glossary.rst +++ b/Docs/source/glossary.rst @@ -17,7 +17,7 @@ Abbreviations * **AMR:** adaptive mesh-refinement * **BC:** boundary condition (of a simulation) * **BCK:** `Benkler-Chavannes-Kuster `__ method, a stabilization technique for small cells in the electromagnetic solver -* **BTD:** backtransformed diagnosics, a method to collect data for analysis from a *boosted frame* simulation +* **BTD:** backtransformed diagnostics, a method to collect data for analysis from a *boosted frame* simulation * **CEX:** charge-exchange collisions * **CFL:** the Courant-Friedrichs-Lewy condition, a numerical parameter for the numerical convergence of PDE solvers * **CI:** continuous integration, automated tests that we perform before a proposed code-change is accepted; see PR diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 420bb376d67..98b0d85391e 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -149,7 +149,12 @@ Please see :ref:`this section `. Nuclear Fusion - Magnetically Confined Plasmas ********************************************** -#. Nicks, B. S., Putvinski, S. and Tajima, T. +#. Nicks B. S., Putvinski S. and Tajima T. **Stabilization of the Alfvén-ion cyclotron instability through short plasmas: Fully kinetic simulations in a high-beta regime**. Physics of Plasmas **30**, 102108, 2023. `DOI:10.1063/5.0163889 `__ + +#. Groenewald R. E., Veksler A., Ceccherini F., Necas A., Nicks B. S., Barnes D. C., Tajima T. and Dettrick S. A. + **Accelerated kinetic model for global macro stability studies of high-beta fusion reactors**. + Physics of Plasmas **30**, 122508, 2023. + `DOI:10.1063/5.0178288 `__ diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 7fc2a1c5833..9f2d2ff038e 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -75,11 +75,9 @@ Usage :hidden: usage/how_to_run - usage/domain_decomposition - usage/parameters - usage/python usage/examples - usage/pwfa + usage/python + usage/parameters usage/workflows usage/faq @@ -109,9 +107,9 @@ Theory :hidden: theory/intro - theory/picsar_theory + theory/pic theory/amr - theory/PML + theory/boundary_conditions theory/boosted_frame theory/input_output theory/collisions @@ -126,10 +124,10 @@ Development :hidden: developers/contributing - developers/workflows developers/developers developers/doxygen developers/gnumake + developers/workflows developers/faq .. good to have in the future: .. developers/repostructure diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index a76e8c212a2..0882efd7fe2 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -116,7 +116,7 @@ By default, the most important dependencies of WarpX are automatically downloade CMake Option Default & Values Description ============================= ============================================== =========================================================== ``BUILD_SHARED_LIBS`` ON/**OFF** `Build shared libraries for dependencies `__ -``CCACHE_PROGRAM`` First found ``ccache`` executable. Set to ``-DCCACHE_PROGRAM=NO`` to disable CCache. +``WarpX_CCACHE`` **ON**/OFF Search and use CCache to speed up rebuilds. ``AMReX_CUDA_PTX_VERBOSE`` ON/**OFF** Print CUDA code generation statistics from ``ptxas``. ``WarpX_amrex_src`` *None* Path to AMReX source directory (preferred if set) ``WarpX_amrex_repo`` ``https://github.com/AMReX-Codes/amrex.git`` Repository URI to pull and build AMReX from @@ -130,7 +130,7 @@ CMake Option Default & Values Des ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from ``WarpX_picsar_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_picsar_repo`` ``WarpX_picsar_internal`` **ON**/OFF Needs a pre-installed PICSAR library if set to ``OFF`` -``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) +``WarpX_pyamrex_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_pyamrex_repo`` ``https://github.com/AMReX-Codes/pyamrex.git`` Repository URI to pull and build pyAMReX from ``WarpX_pyamrex_branch`` *we set and maintain a compatible commit* Repository branch for ``WarpX_pyamrex_repo`` ``WarpX_pyamrex_internal`` **ON**/OFF Needs a pre-installed pyAMReX library if set to ``OFF`` @@ -147,10 +147,12 @@ Relative paths are also supported, e.g. ``-DWarpX_amrex_src=../amrex``. Or build against an AMReX feature branch of a colleague. Assuming your colleague pushed AMReX to ``https://github.com/WeiqunZhang/amrex/`` in a branch ``new-feature`` then pass to ``cmake`` the arguments: ``-DWarpX_amrex_repo=https://github.com/WeiqunZhang/amrex.git -DWarpX_amrex_branch=new-feature``. +More details on this :ref:`workflow are described here `. You can speed up the install further if you pre-install these dependencies, e.g. with a package manager. Set ``-DWarpX__internal=OFF`` and add installation prefix of the dependency to the environment variable `CMAKE_PREFIX_PATH `__. Please see the :ref:`introduction to CMake ` if this sounds new to you. +More details on this :ref:`workflow are described here `. If you re-compile often, consider installing the `Ninja `__ build system. Pass ``-G Ninja`` to the CMake configuration call to speed up parallel compiles. @@ -327,6 +329,7 @@ This is the workflow most developers will prefer as it allows rapid re-compiles: # build & install Python only cmake --build build -j 4 --target pip_install +There is also a ``--target pip_install_nodeps`` option that :ref:`skips pip-based dependency checks `. WarpX release managers might also want to generate a self-contained source package that can be distributed to exotic architectures: diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 7bd6c7dc8d5..94b2be78bec 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -12,13 +12,18 @@ Please see installation instructions below. - `AMReX `__: we automatically download and compile a copy of AMReX - `PICSAR `__: we automatically download and compile a copy of PICSAR +and for Python bindings: + +- `pyAMReX `__: we automatically download and compile a copy of pyAMReX +- `pybind11 `__: we automatically download and compile a copy of pybind11 + Optional dependencies include: - `MPI 3.0+ `__: for multi-node and/or multi-GPU execution - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.0+ (11.3+ recommended) `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support - `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 9617f2a7fd6..a7b0f636b56 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -46,6 +46,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lxplus hpc/ookami hpc/perlmutter + hpc/polaris hpc/quartz hpc/spock hpc/summit diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst index 44b07985670..0b984d5e2be 100644 --- a/Docs/source/install/hpc/adastra.rst +++ b/Docs/source/install/hpc/adastra.rst @@ -31,18 +31,26 @@ If you are new to this system, **please see the following resources**: Preparation ----------- +The following instructions will install WarpX in the ``$SHAREDHOMEDIR`` directory, +which is shared among all the members of a given project. Due to the inode +quota enforced for this machine, a shared installation of WarpX is advised. + Use the following commands to download the WarpX source code: .. code-block:: bash - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + # If you have multiple projects, activate the project that you want to use with: + # + # myproject -a YOUR_PROJECT_NAME + # + git clone https://github.com/ECP-WarpX/WarpX.git $SHAREDHOMEDIR/src/warpx -We use system software modules, add environment hints and further dependencies via the file ``$HOME/adastra_warpx.profile``. +We use system software modules, add environment hints and further dependencies via the file ``$SHAREDHOMEDIR/adastra_warpx.profile``. Create it now: .. code-block:: bash - cp $HOME/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $HOME/adastra_warpx.profile + cp $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $SHAREDHOMEDIR/adastra_warpx.profile .. dropdown:: Script Details :color: light @@ -53,8 +61,8 @@ Create it now: :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor -such as ``nano``, ``emacs``, or ``vim`` (all available by default on -Adastra login nodes). +such as ``nano``, ``emacs``, or ``vim`` (all available by default on Adastra login nodes) and +uncomment the 3rd line (which sets ``$proj`` as the active project). .. important:: @@ -62,14 +70,14 @@ Adastra login nodes). .. code-block:: bash - source $HOME/adastra_warpx.profile + source $SHAREDHOMEDIR/adastra_warpx.profile Finally, since Adastra does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - bash $HOME/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh - source $HOME/sw/adastra/gpu/venvs/warpx-adastra/bin/activate + bash $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh + source $SHAREDHOMEDIR/sw/adastra/gpu/venvs/warpx-adastra/bin/activate .. dropdown:: Script Details :color: light @@ -89,13 +97,13 @@ Use the following :ref:`cmake commands ` to compile the applicat .. code-block:: bash - cd $HOME/src/warpx + cd $SHAREDHOMEDIR/src/warpx rm -rf build_adastra cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra -j 16 -The WarpX application executables are now in ``$HOME/src/warpx/build_adastra/bin/``. +The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``. Additionally, the following commands will install WarpX as a Python module: .. code-block:: bash @@ -119,7 +127,7 @@ If you already installed WarpX in the past and want to update it, start by getti .. code-block:: bash - cd $HOME/src/warpx + cd $SHAREDHOMEDIR/src/warpx # read the output of this command - does it look ok? git status diff --git a/Docs/source/install/hpc/fugaku.rst b/Docs/source/install/hpc/fugaku.rst index 43eb414de09..5e5fc57c3b0 100644 --- a/Docs/source/install/hpc/fugaku.rst +++ b/Docs/source/install/hpc/fugaku.rst @@ -14,8 +14,10 @@ If you are new to this system, **please see the following resources**: * `Fugaku user guide `__ -Installation ------------- +.. _building-fugaku-preparation: + +Preparation +----------- Use the following commands to download the WarpX source code and switch to the correct branch: @@ -24,52 +26,49 @@ Use the following commands to download the WarpX source code and switch to the c git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx -Compiling WarpX on Fugaku is more pratical on a compute node. Use the following commands to acquire a compute node for one hour: +Compiling WarpX on Fugaku is more practical on a compute node. Use the following commands to acquire a compute node for one hour: .. code-block:: bash pjsub --interact -L "elapse=02:00:00" -L "node=1" --sparam "wait-time=300" --mpi "max-proc-per-node=48" --all-mount-gfscache -Then, load ``cmake`` and ``ninja`` using ``spack``: +We use system software modules, add environment hints and further dependencies via the file ``$HOME/fugaku_warpx.profile``. +Create it now, modify it if needed, and source it (it will take few minutes): .. code-block:: bash - . /vol0004/apps/oss/spack/share/spack/setup-env.sh - spack load cmake@3.21.4%fj@4.8.1 arch=linux-rhel8-a64fx + cp $HOME/src/warpx/Tools/machines/fugaku-riken/fugaku_warpx.profile.example $HOME/fugaku_warpx.profile + source $HOME/fugaku_warpx.profile - # optional: faster builds - spack load ninja@1.11.1%fj@4.8.1 +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - # avoid harmless warning messages "[WARN] xos LPG [...]" - export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/fugaku_warpx.profile.example + :language: bash - -At this point we need to download and compile the libraries required for OpenPMD support: +Finally, since Fugaku does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - export CC=$(which mpifcc) - export CXX=$(which mpiFCC) - export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" - export CXXFLAGS="${CFLAGS}" + bash $HOME/src/warpx/Tools/machines/fugaku-riken/install_dependencies.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/install_dependencies.sh + :language: bash - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-fg-build - cmake -S src/c-blosc -B src/c-blosc-fg-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install - cmake --build src/c-blosc-fg-build --target install --parallel 24 +.. _building-fugaku-compilation: - # ADIOS2 - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-fg-build - cmake -S src/adios2 -B src/adios2-fg-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install - cmake --build src/adios2-fg-build --target install -j 24 +Compilation +----------- -Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: +Use the following :ref:`cmake commands ` to compile the application executable: .. code-block:: bash @@ -82,16 +81,14 @@ Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following com export CXXFLAGS="-Nclang" cmake -S . -B build -DWarpX_COMPUTE=OMP \ - -DWarpX_DIMS="1;2;3" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS_RELEASE="-Ofast -mllvm -polly -mllvm -polly-parallel" \ - -DAMReX_DIFFERENT_COMPILER=ON \ - -DWarpX_MPI_THREAD_MULTIPLE=OFF + -DWarpX_DIMS="1;2;3" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_FLAGS_RELEASE="-Ofast" \ + -DAMReX_DIFFERENT_COMPILER=ON \ + -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 48 -The general :ref:`cmake compile-time options ` apply as usual. - **That's it!** A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. diff --git a/Docs/source/install/hpc/hpc3.rst b/Docs/source/install/hpc/hpc3.rst index 33296e92b3d..e2f900de273 100644 --- a/Docs/source/install/hpc/hpc3.rst +++ b/Docs/source/install/hpc/hpc3.rst @@ -96,7 +96,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build -j 12 + cmake --build build -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. Additionally, the following commands will install WarpX as a Python module: @@ -106,7 +106,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_py cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_py -j 12 --target pip_install + cmake --build build_py -j 8 --target pip_install Now, you can :ref:`submit HPC3 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit HPC3 jobs (:ref:`example inputs `). diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index 117cc32a0df..fffb917df1c 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -12,83 +12,60 @@ Introduction If you are new to this system, **please see the following resources**: * `IT4I user guide `__ -* Batch system: `PBS `__ +* Batch system: `SLURM `__ * Jupyter service: not provided/documented (yet) * `Filesystems `__: * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota) - * ``/scatch/``: `production directory `__; very fast for parallel jobs (20TB default) + * ``/scratch/``: `production directory `__; very fast for parallel jobs (10TB default) + * ``/mnt/proj/``: per-project work directory, used for long term data storage (20TB default) .. _building-karolina-preparation: -Preparation ------------ - -Use the following commands to download the WarpX source code: +Installation +------------ -.. code-block:: bash +We show how to install from scratch all the dependencies using `Spack `__. - git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx +For size reasons it is not advisable to install WarpX in the ``$HOME`` directory, it should be installed in the "work directory". For this purpose we set an environment variable ``$WORK`` with the path to the "work directory". On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. -.. tab-set:: - - .. tab-item:: A100 GPUs - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash +Profile file +^^^^^^^^^^^^ - export proj="DD-23-83" +One can use the pre-prepared ``karolina_warpx.profile`` script below, +which you can copy to ``${HOME}/karolina_warpx.profile``, edit as required and then ``source``. - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. important:: + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_warpx.profile.example + :language: bash + :caption: Copy the contents of this file to ``${HOME}/karolina_warpx.profile``. - Now, and as the first step on future logins to Karolina, activate these environment settings: +To have the environment activated on every login, add the following line to ``${HOME}/.bashrc``: - .. code-block:: bash - - source $HOME/karolina_gpu_warpx.profile - - Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once: - - .. code-block:: bash +.. code-block:: bash - bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh - source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate + source $HOME/karolina_warpx.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +To install the ``spack`` environment and Python packages: - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh - :language: bash +.. code-block:: bash + bash $WORK/src/warpx/Tools/machines/karolina-it4i/install_dependencies.sh - .. tab-item:: CPU Nodes +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - CPU usage is documentation is TODO. + .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_dependencies.sh + :language: bash .. _building-karolina-compilation: @@ -98,83 +75,28 @@ Compilation Use the following :ref:`cmake commands ` to compile the application executable: -.. tab-set:: - - .. tab-item:: A100 GPUs - - .. code-block:: bash - - cd $HOME/src/warpx - rm -rf build_gpu - - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu -j 12 - - The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: - - .. code-block:: bash - - rm -rf build_gpu_py - - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_gpu_py -j 12 --target pip_install - - .. tab-item:: CPU Nodes - - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu + cd $WORK/src/warpx + rm -rf build_gpu - cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu -j 12 + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu -j 48 - The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/``. - Additionally, the following commands will install WarpX as a Python module: +The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. +Additionally, the following commands will install WarpX as a Python module: - .. code-block:: bash +.. code-block:: bash - cd $HOME/src/warpx - rm -rf build_cpu_py + cd $WORK/src/warpx + rm -rf build_gpu_py - cmake -S . -B build_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" - cmake --build build_cpu_py -j 12 --target pip_install + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scatch/``. - - -.. _building-karolina-update: - -Update WarpX & Dependencies ---------------------------- - -If you already installed WarpX in the past and want to update it, start by getting the latest source code: - -.. code-block:: bash - - cd $HOME/src/warpx - - # read the output of this command - does it look ok? - git status - - # get the latest WarpX source code - git fetch - git pull - - # read the output of these commands - do they look ok? - git status - git log # press q to exit - -And, if needed, - -- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `, -- log out and into the system, activate the now updated environment profile as usual, -- :ref:`execute the dependency install scripts `. - -As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch/``. .. _running-cpp-karolina: @@ -182,33 +104,24 @@ As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and Running ------- -.. tab-set:: +The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#SBATCH --nodes=`` accordingly) on the supercomputer Karolina at IT4I. +This partition has up to `72 nodes `__. +Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. - .. tab-item:: A100 (40GB) GPUs +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. +Note that we run one MPI rank per GPU. - The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I. - This partition as up to `72 nodes `__. - Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors. +.. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.sbatch + :language: bash + :caption: You can copy this file from ``$WORK/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.sbatch``. - Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``. - Note that we run one MPI rank per GPU. - - .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub - :language: bash - :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``. - - To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run - - .. code-block:: bash - - qsub karolina_gpu.qsub - - to submit the job. +To run a simulation, copy the lines above to a file ``karolina_gpu.sbatch`` and run +.. code-block:: bash - .. tab-item:: CPU Nodes + sbatch karolina_gpu.sbatch - CPU usage is documentation is TODO. +to submit the job. .. _post-processing-karolina: diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index ac15844a5ff..d8565f06983 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -178,7 +178,7 @@ Finally, since lassen does not yet provide software modules for some of our depe .. code-block:: bash bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen-toss3/bin/activate + source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate .. dropdown:: Script Details :color: light @@ -263,15 +263,15 @@ The batch script below can be used to run a WarpX simulation on 2 nodes on the s Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. Note that the only option so far is to run with one MPI rank per GPU. -.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen.bsub +.. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100.bsub :language: bash - :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen.bsub``. + :caption: You can copy this file from ``Tools/machines/lassen-llnl/lassen_v100.bsub``. -To run a simulation, copy the lines above to a file ``lassen.bsub`` and run +To run a simulation, copy the lines above to a file ``lassen_v100.bsub`` and run .. code-block:: bash - bsub lassen.bsub + bsub lassen_v100.bsub to submit the job. diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 3de7f4755cb..1f3c1e8fbfa 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -103,9 +103,9 @@ Additionally, the following commands will install WarpX as a Python module: cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install -Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Leonardo jobs (:ref:`example inputs `). -For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$CINECA_SCRATCH``. .. _building-leonardo-update: diff --git a/Docs/source/install/hpc/polaris.rst b/Docs/source/install/hpc/polaris.rst new file mode 100644 index 00000000000..d20ecccee32 --- /dev/null +++ b/Docs/source/install/hpc/polaris.rst @@ -0,0 +1,187 @@ +.. _building-polaris: + +Polaris (ALCF) +============== + +The `Polaris cluster `__ is located at ALCF. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `ALCF user guide `__ +* Batch system: `PBS `__ +* `Filesystems `__ + +.. _building-polaris-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Polaris, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes. + +.. tab-set:: + + .. tab-item:: A100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/polaris_gpu_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example $HOME/polaris_gpu_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``proj_name``, then run ``nano $HOME/polaris_gpu_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="proj_name" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to Polaris, activate these environment settings: + + .. code-block:: bash + + source $HOME/polaris_gpu_warpx.profile + + Finally, since Polaris does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/polaris-alcf/install_gpu_dependencies.sh + source ${CFS}/${proj%_g}/${USER}/sw/polaris/gpu/venvs/warpx/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/install_gpu_dependencies.sh + :language: bash + + + .. tab-item:: CPU Nodes + + *Under construction* + + +.. _building-polaris-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + + .. tab-item:: A100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu + + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_pm_gpu -j 16 + + The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. + Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_pm_gpu_py + + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_pm_gpu_py -j 16 --target pip_install + + .. tab-item:: CPU Nodes + + *Under construction* + +Now, you can :ref:`submit Polaris compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit Polaris jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``$PSCRATCH``. + + +.. _building-polaris-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the polaris_gpu_warpx.profile or polaris_cpu_warpx files `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_pm_*`` and rebuild WarpX. + + +.. _running-cpp-polaris: + +Running +------- + +.. tab-set:: + + .. tab-item:: A100 (40GB) GPUs + + The batch script below can be used to run a WarpX simulation on multiple nodes (change ```` accordingly) on the supercomputer Polaris at ALCF. + + Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + Note that we run one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/polaris-alcf/polaris_gpu.pbs + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/polaris-alcf/polaris_gpu.pbs``. + + To run a simulation, copy the lines above to a file ``polaris_gpu.pbs`` and run + + .. code-block:: bash + + qsub polaris_gpu.pbs + + to submit the job. + + + .. tab-item:: CPU Nodes + + *Under construction* diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index de7c5ace848..03860687971 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -37,14 +37,14 @@ Create it now: .. code-block:: bash - cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example $HOME/quartz_warpx.profile + cp $HOME/src/warpx/Tools/machines/quartz-llnl/quartz_warpx.profile.example $HOME/quartz_warpx.profile .. dropdown:: Script Details :color: light :icon: info :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz/quartz_warpx.profile.example + .. literalinclude:: ../../../../Tools/machines/quartz-llnl/quartz_warpx.profile.example :language: bash Edit the 2nd line of this script, which sets the ``export proj=""`` variable. diff --git a/Docs/source/latex_theory/AMR/AMR.tex b/Docs/source/latex_theory/AMR/AMR.tex index 948921eca5e..3ca94d95436 100644 --- a/Docs/source/latex_theory/AMR/AMR.tex +++ b/Docs/source/latex_theory/AMR/AMR.tex @@ -22,7 +22,7 @@ \section{Mesh refinement} In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss' Law around the refined patch, leading to long range errors \cite{Vaylpb2002,Colellajcp2010}. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. \subsection{Electrostatic} -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. +A cornerstone of the Particle-In-Cell method is that, given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called ``momentum conserving'' gathering scheme; b) on average within one cell if using the ``energy conserving'' gathering scheme \cite{Birdsalllangdon}. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) \cite{Vaylpb2002,Colellajcp2010}. A sketch of the implementation of mesh refinement in WarpX is given in Figure~\ref{fig:ESAMR} (left). Given the solution of the electric potential at a refinement level $L_n$, it is interpolated onto the boundaries of the grid patch(es) at the next refined level $L_{n+1}$. The electric potential is then computed at level $L_{n+1}$ by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. @@ -53,7 +53,7 @@ \subsection{Electrostatic} % \caption{Snapshot from a 3D self-consistent simulation of the injector in the High Current Experiment shows the beam emerging from the source at low energy (blue) and being accelerated (green-yellow-orange) and transported in a four quadrupole front end. The automatic layout of the mesh refinement patches from a 2D axisymmetric simulation of the source area shows 2 levels of refinement, concentrating the finer meshes around the emitter (white curve surface) and the beam edge (dark blue).} % \label{fig:ESHCX} %\end{figure} -%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the souce area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. +%Automatic remeshing has been implemented in WarpX following the procedure described in \cite{Vaynim2005}, refining on criteria based on measures of local charge density magnitude and gradients. AMR WarpX simulations were applied to the modeling of the front end injector of the High Current Experiment (HCX) \cite{Prostprstab2005}, and provided the first numerically converged estimates of phase space beam distorsions, which directly affects beam quality \cite{Vaypop04}. Fig.~\ref{fig:ESHCX} shows snapshots from 2D axisymmetric simulation of the source area illustrating the automatic placement of refined patches, and 3D simulation of the full injector showing the beam generation, acceleration and transport. \subsection{Electromagnetic} The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of \cite{Vayjcp01}, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. diff --git a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex index 471863ac3ee..fe8365d15dd 100644 --- a/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex +++ b/Docs/source/latex_theory/Boosted_frame/Boosted_frame.tex @@ -175,7 +175,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} the simulation, and the artificial ``bump'' is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in \cite{KirchenARXIV2016,LeheARXIV2016}, which +A new scheme was recently proposed, in \cite{KirchenPOP2016,LehePRE2016}, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity -- with no arbitrary correction -- by simply integrating the PIC equations in \emph{Galilean coordinates} (also known as @@ -217,7 +217,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} \emph{themselves} are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current $\vec{J}$ within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme \cite{LeheARXIV2016}. +window and in a Galilean PSATD scheme \cite{LehePRE2016}. In the Galilean coordinates $\vec{x}'$, the equations of particle motion and the Maxwell equations take the form @@ -235,7 +235,7 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Integrating these equations from $t=n\Delta t$ to $t=(n+1)\Delta t$ results in the following update equations (see -\cite{LeheARXIV2016} for the details of the derivation): +\cite{LehePRE2016} for the details of the derivation): % \begin{subequations} \begin{align} @@ -271,5 +271,5 @@ \subsection{Numerical Stability and alternate formulation in a Galilean frame} Note that, in the limit $\vgal=\vec{0}$, (\ref{eq:disc-maxwell1}) and (\ref{eq:disc-maxwell2}) reduce to the standard PSATD equations \cite{Habericnsp73}, as expected. -As shown in \cite{KirchenARXIV2016,LeheARXIV2016}, +As shown in \cite{KirchenPOP2016,LehePRE2016}, the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. diff --git a/Docs/source/latex_theory/Makefile b/Docs/source/latex_theory/Makefile index 916bb2399d7..9b2f598f51d 100644 --- a/Docs/source/latex_theory/Makefile +++ b/Docs/source/latex_theory/Makefile @@ -7,14 +7,14 @@ all: $(SRC_FILES) clean pandoc Boosted_frame/Boosted_frame.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o boosted_frame.rst pandoc input_output/input_output.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o input_output.rst mv *.rst ../theory - cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o picsar_theory.rst - mv ../../../../picsar/Doxygen/pages/latex_theory/picsar_theory.rst ../theory + cd ../../../../picsar/Doxygen/pages/latex_theory/; pandoc theory.tex --mathjax --wrap=preserve --bibliography allbibs.bib -o pic.rst + mv ../../../../picsar/Doxygen/pages/latex_theory/pic.rst ../theory cp ../../../../picsar/Doxygen/images/PIC.png ../theory cp ../../../../picsar/Doxygen/images/Yee_grid.png ../theory clean: rm -f ../theory/intro.rst rm -f ../theory/warpx_theory.rst - rm -f ../theory/picsar_theory.rst + rm -f ../theory/pic.rst rm -f ../theory/PIC.png rm -f ../theory/Yee_grid.png diff --git a/Docs/source/latex_theory/allbibs.bib b/Docs/source/latex_theory/allbibs.bib index 81a02e75494..b3475c5a81b 100644 --- a/Docs/source/latex_theory/allbibs.bib +++ b/Docs/source/latex_theory/allbibs.bib @@ -4,11 +4,11 @@ BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop @article{QuickpicParallel, -author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W.B. and Muggli, P. and Katsouleas, T.}, +author = {Feng, B. and Huang, C. and Decyk, V. and Mori, W. B. and Muggli, P. and Katsouleas, T.}, doi = {10.1016/j.jcp.2009.04.019}, issn = {00219991}, journal = {Journal of Computational Physics}, -month = {aug}, +month = {Aug}, number = {15}, pages = {5340--5348}, title = {{Enhancing parallel quasi-static particle-in-cell simulations with a pipelining algorithm}}, @@ -20,16 +20,17 @@ @book{HockneyEastwoodBook author = {Hockney, R W and Eastwood, J W}, isbn = {0-85274-392-0}, pages = {xxi+540 pp}, +publisher = {Routledge}, title = {{Computer simulation using particles}}, type = {Book}, year = {1988} } @article{Parkerjcp1991, -author = {Parker, Se and Birdsall, Ck}, +author = {Parker, S E and Birdsall, C K}, doi = {10.1016/0021-9991(91)90040-R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {91--102}, title = {{Numerical Error In Electron Orbits With Large Omega-Ce Delta-T}}, @@ -44,7 +45,7 @@ @inproceedings{Fawleyipac10 year = {2010} } @article{Godfreyjcp74, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {4}, @@ -54,11 +55,11 @@ @article{Godfreyjcp74 year = {1974} } @article{Quickpic, -author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and {Antonsen Jr.}, T M and Katsouleas, T}, +author = {Huang, C and Decyk, V K and Ren, C and Zhou, M and Lu, W and Mori, W B and Cooley, J H and Antonsen, Jr, T M and Katsouleas, T}, doi = {10.1016/J.Jcp.2006.01.039}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {658--679}, title = {{Quickpic: A Highly Efficient Particle-In-Cell Code For Modeling Wakefield Acceleration In Plasmas}}, @@ -70,7 +71,7 @@ @article{Lundprstab2009 doi = {10.1103/Physrevstab.12.114801}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {nov}, +month = {Nov}, number = {11}, title = {{Generation Of Initial Kinetic Distributions For Simulation Of Long-Pulse Charged Particle Beams With High Space-Charge Intensity}}, volume = {12}, @@ -81,7 +82,7 @@ @article{CowanPRSTAB13 doi = {10.1103/PhysRevSTAB.16.041303}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations}}, volume = {16}, @@ -102,7 +103,7 @@ @article{YuPRL2014 author = {Yu, L.-L. and Esarey, E and Schroeder, C B and Vay, J.-L. and Benedetti, C and Geddes, C G R and Chen, M and Leemans, W P}, doi = {10.1103/PhysRevLett.112.125001}, journal = {Phys. Rev. Lett.}, -month = {mar}, +month = {Mar}, number = {12}, pages = {125001}, publisher = {American Physical Society}, @@ -118,7 +119,7 @@ @article{Vaynim2007 institution = {Univ Paris Sud Xi}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {65--69}, title = {{Self-Consistent Simulations Of Heavy-Ion Beams Interacting With Electron-Clouds}}, @@ -138,10 +139,10 @@ @article{Vay2000 year = {2000} } @article{Vayjcp2011, -author = {Vay, J L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J-L and Geddes, C G R and Cormier-Michel, E and Grote, D P}, doi = {10.1016/J.Jcp.2011.04.003}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {15}, pages = {5908--5929}, title = {{Numerical Methods For Instability Mitigation In The Modeling Of Laser Wakefield Accelerators In A Lorentz-Boosted Frame}}, @@ -152,21 +153,15 @@ @article{Krallpre1993 author = {Krall, J and Ting, A and Esarey, E and Sprangle, P}, doi = {10.1103/Physreve.48.2157}, journal = {Physical Review E}, -month = {sep}, +month = {Sep}, number = {3}, pages = {2157--2161}, title = {{Enhanced Acceleration In A Self-Modulated-Laser Wake-Field Accelerator}}, volume = {48}, year = {1993} } -@article{Vayarxiv10_2, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1011.0917V2}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Dennisw1997585, -author = {W., Dennis and Hewett}, +author = {Dennis W. Hewett}, doi = {10.1006/Jcph.1997.5835}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -178,12 +173,11 @@ @article{Dennisw1997585 year = {1997} } @article{Habib2016, - archivePrefix = {arXiv}, arxivId = {1603.09303}, -author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C. -K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J. -L. and Viren, B. and Wurthwein, F. and Xiao, L.}, +author = {Habib, Salman and Roser, Robert and Gerber, Richard and Antypas, Katie and Riley, Katherine and Williams, Tim and Wells, Jack and Straatsma, Tjerk and Almgren, A. and Amundson, J. and Bailey, S. and Bard, D. and Bloom, K. and Bockelman, B. and Borgland, A. and Borrill, J. and Boughezal, R. and Brower, R. and Cowan, B. and Finkel, H. and Frontiere, N. and Fuess, S. and Ge, L. and Gnedin, N. and Gottlieb, S. and Gutsche, O. and Han, T. and Heitmann, K. and Hoeche, S. and Ko, K. and Kononenko, O. and LeCompte, T. and Li, Z. and Lukic, Z. and Mori, W. and Nugent, P. and Ng, C.-K. and Oleynik, G. and O'Shea, B. and Padmanabhan, N. and Petravick, D. and Petriello, F. J. and Power, J. and Qiang, J. and Reina, L. and Rizzo, T. J. and Ryne, R. and Schram, M. and Spentzouris, P. and Toussaint, D. and Vay, J.-L. and Viren, B. and Wurthwein, F. and Xiao, L.}, eprint = {1603.09303}, -month = {mar}, +month = {Mar}, title = {{ASCR/HEP Exascale Requirements Review Report}}, url = {http://arxiv.org/abs/1603.09303}, year = {2016} @@ -192,8 +186,8 @@ @article{Shadwickpop09 author = {Shadwick, B A and Schroeder, C B and Esarey, E}, doi = {10.1063/1.3124185}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56704}, title = {{Nonlinear Laser Energy Depletion In Laser-Plasma Accelerators}}, @@ -206,7 +200,7 @@ @article{Leemansaac2010 journal = {Aip Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, pages = {3--11}, -title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 Gev Laser Plasma Accelerator}}, +title = {{The Berkeley Lab Laser Accelerator (Bella): A 10 GeV Laser Plasma Accelerator}}, volume = {1299}, year = {2010} } @@ -214,7 +208,7 @@ @article{Marderjcp87 author = {Marder, B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jan}, +month = {Jan}, number = {1}, pages = {48--55}, title = {{A Method For Incorporating Gauss Law Into Electromagnetic Pic Codes}}, @@ -231,7 +225,7 @@ @article{Ohmurapiers2010 year = {2010} } @article{Adamjcp1982, -author = {Adam, Jc and Serveniere, Ag and Langdon, Ab}, +author = {Adam, J C and Serveniere, Ag and Langdon, A B}, doi = {10.1016/0021-9991(82)90076-6}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -242,7 +236,7 @@ @article{Adamjcp1982 year = {1982} } @article{Tajimaprl79, -author = {Tajima, T and Dawson, Jm}, +author = {Tajima, T and Dawson, J M}, issn = {0031-9007}, journal = {Physical Review Letters}, number = {4}, @@ -251,13 +245,19 @@ @article{Tajimaprl79 volume = {43}, year = {1979} } -@article{Benedetti08, -author = {{Benedetti et al.}, C}, -journal = {Nuclear Inst. And Methods In Physics Research A}, -pages = {94--98}, -title = {{No Title}}, +@article{Benedetti09, +author = {C. Benedetti and P. Londrillo and V. Petrillo and L. Serafini and A. Sgattoni and P. Tomassini and G. Turchetti}, +doi = {https://doi.org/10.1016/j.nima.2009.05.064}, +issn = {0168-9002}, +journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, +keywords = {PIC simulation, Laser–plasma interaction, FEL}, +note = {Compton sources for X/γ rays: Physics and applications}, +number = {1, Supplement }, +pages = {S94-S98}, +title = {PIC simulations of the production of high-quality electron beams via laser–plasma interaction}, +url = {https://www.sciencedirect.com/science/article/pii/S0168900209009784}, volume = {608}, -year = {2009} +year = {2009}, } @inproceedings{Godfreyicnsp80, author = {Godfrey, B B}, @@ -270,7 +270,7 @@ @article{Blumenfeld2007 author = {Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and Lu, Wei and Marsh, Kenneth A and Mori, Warren B and Muggli, Patric and Oz, Erdem and Siemann, Robert H and Walz, Dieter and Zhou, Miaomiao}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7129}, pages = {741--744}, title = {{Energy doubling of 42[thinsp]GeV electrons in a metre-scale plasma wakefield accelerator}}, @@ -280,7 +280,7 @@ @article{Blumenfeld2007 } @inproceedings{Vayicap2002, annote = {7Th International Conference On Computational Accelerator Physics, Michigan State Univ, E Lansing, Mi, Oct 15-18, 2002}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, booktitle = {Computational Accelerator Physics 2002}, editor = {Berz, M and Makino, K}, isbn = {0-7503-0939-3}, @@ -308,16 +308,15 @@ @article{VayFRACAD2014 } @article{AndriyashPoP2016, - author = "Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin", - title = "Laser-plasma interactions with a Fourier-Bessel particle-in-cell method", - journal = "Physics of Plasmas", - year = "2016", - volume = "23", - number = "3", - eid = 033110, - pages = "", - url = "http://scitation.aip.org/content/aip/journal/pop/23/3/10.1063/1.4943281", - doi = "http://dx.doi.org/10.1063/1.4943281" +author = {Andriyash, Igor A. and Lehe, Remi and Lifschitz, Agustin}, +doi = {10.1063/1.4943281}, +eid = {033110}, +journal = {Physics of Plasmas}, +number = {3}, +pages = {}, +title = {{Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}}, +volume = {23}, +year = {2016} } @article{DawsonRMP83, @@ -340,7 +339,7 @@ @inproceedings{Qiang @inproceedings{Geddespac09, address = {Vancouver, Canada}, annote = {We6Rfp075}, -author = {{Geddes et al.}, C G R}, +author = {C. G. R. Geddes and E. Cormier-Michel and E. Esarey and C. B. Schroeder and W. P. Leemans}, booktitle = {Proc. Particle Accelerator Conference}, title = {{Scaled Simulation Design Of High Quality Laser Wakefield Accelerator Stages}}, year = {2009} @@ -348,10 +347,9 @@ @inproceedings{Geddespac09 @article{LotovPRSTAB2003, author = {Lotov, K. V.}, doi = {10.1103/PhysRevSTAB.6.061301}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lotov - 2003 - Fine wakefield structure in the blowout regime of plasma wakefield accelerators.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {jun}, +month = {Jun}, number = {6}, pages = {061301}, publisher = {American Physical Society}, @@ -361,7 +359,7 @@ @article{LotovPRSTAB2003 year = {2003} } @article{Godfreyjcp75, -author = {Godfrey, Bb}, +author = {Godfrey, B B}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -374,7 +372,7 @@ @article{Kishekprl2012 author = {Kishek, R A}, doi = {10.1103/Physrevlett.108.035003}, journal = {Phys. Rev. Lett.}, -month = {jan}, +month = {Jan}, number = {3}, pages = {35003}, publisher = {American Physical Society}, @@ -392,12 +390,12 @@ @article{Zhang2016 } @article{Cohennim2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -405,10 +403,11 @@ @article{Cohennim2009 year = {2009} } @article{Osiris, -author = {{Fonseca et al.}, R A}, -journal = {Lec. Notes In Comp. Sci.}, -pages = {342}, -title = {{No Title}}, +author = {Fonseca, R. A. and Silva, L. O. and Tsung, F. S. and Decyk, V. K. and Lu, W. and Ren, C. and Mori, W. B. and Deng, S. and Lee, S. and Katsouleas, T. and Adam, J. C.}, +booktitle = {Computational Science --- ICCS 2002}, +pages = {342--351}, +publisher = {Springer Berlin Heidelberg}, +title = {{OSIRIS: A Three-Dimensional, Fully Relativistic Particle in Cell Code for Modeling Plasma Based Accelerators}}, volume = {2329}, year = {2002} } @@ -439,7 +438,7 @@ @article{GeddesPRL2008 author = {Geddes, C G R and Nakamura, K and Plateau, G R and Toth, Cs. and Cormier-Michel, E and Esarey, E and Schroeder, C B and Cary, J R and Leemans, W P}, doi = {10.1103/PhysRevLett.100.215004}, journal = {Phys. Rev. Lett.}, -month = {may}, +month = {May}, number = {21}, pages = {215004}, publisher = {American Physical Society}, @@ -465,12 +464,6 @@ @misc{Huebl2015 url = {http://dx.doi.org/10.5281/zenodo.591699}, year = {2015} } -@article{Vayarxiv10_1, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:1009.2727V2}, -title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, -year = {2010} -} @article{Huangscidac09, author = {Huang, C and An, W and Decyk, V K and Lu, W and Mori, W B and Tsung, F S and Tzoufras, M and Morshed, S and Antonsen, T and Feng, B and Katsouleas, T and Fonseca, R A and Martins, S F and Vieira, J and Silva, L O and Esarey, E and Geddes, C G R and Leemans, W P and Cormier-Michel, E and Vay, J.-L. and Bruhwiler, D L and Cowan, B and Cary, J R and Paul, K}, issn = {1742-6596}, @@ -494,7 +487,7 @@ @article{Cormierprstab10 title = {{Propagation Of Higher Order Modes In Plasma Channels And Shaping Of The Transverse Field In Laser Plasma Accelerators}} } @article{Yee, -author = {Yee, Ks}, +author = {Yee, K S}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, number = {3}, @@ -521,7 +514,7 @@ @inproceedings{Cormieraac08 booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, pages = {297--302}, -title = {{Scaled Simulations Of A 10 Gev Accelerator}}, +title = {{Scaled Simulations Of A 10 GeV Accelerator}}, volume = {1086}, year = {2009} } @@ -536,11 +529,11 @@ @inproceedings{Bruhwileraac08 } @article{Vaynim2005, annote = {15Th International Symposium On Heavy Ion Inertial Fusion, Princeton, Nj, Jun 07-11, 2004}, -author = {Vay, Jl and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Friedman, A and Grote, D P}, doi = {10.1016/J.Nima.2005.01.232}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {may}, +month = {May}, number = {1-2}, pages = {347--352}, title = {{Application Of Adaptive Mesh Refinement To Pic Simulations In Heavy Ion Fusion}}, @@ -549,7 +542,7 @@ @article{Vaynim2005 } @article{BulanovSV2014, -author = {{Bulanov S V and Wilkens J J and Esirkepov T Zh and Korn G and Kraft G and Kraft S D and Molls M and Khoroshkov V S}}, +author = {Bulanov, S V and Wilkens, J J and Esirkepov, T Zh and Korn, G and Kraft, G and Kraft, S D and Molls, M and Khoroshkov, V S}, issn = {1063-7869}, journal = {Physics-Uspekhi}, number = {12}, @@ -560,11 +553,11 @@ @article{BulanovSV2014 year = {2014} } @article{Furmanprstab2002, -author = {Furman, Ma and Pivi, Mtf}, +author = {Furman, M A and Pivi, M T F}, doi = {10.1103/Physrevstab.5.124404}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {dec}, +month = {Dec}, number = {12}, title = {{Probabilistic Model For The Simulation Of Secondary Electron Emission}}, volume = {5}, @@ -573,10 +566,9 @@ @article{Furmanprstab2002 @article{Qiang2014, author = {Qiang, J. and Corlett, J. and Mitchell, C. E. and Papadopoulos, C. F. and Penn, G. and Placidi, M. and Reinsch, M. and Ryne, R. D. and Sannibale, F. and Sun, C. and Venturini, M. and Emma, P. and Reiche, S.}, doi = {10.1103/PhysRevSTAB.17.030701}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Qiang et al. - 2014 - Start-to-end simulation of x-ray radiation of a next generation light source using the real number of electrons.pdf:pdf}, issn = {1098-4402}, journal = {Physical Review Special Topics - Accelerators and Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {030701}, publisher = {American Physical Society}, @@ -601,13 +593,12 @@ @misc{Bruhwilerpc08 year = {2008} } @article{Yu2014, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and An, Weiming and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2014.02.016}, issn = {00219991}, journal = {Journal of Computational Physics}, keywords = {Boosted frame simulation,IN-CELL CODE,INSTABILITIES,Laser wakefield accelerator,Numerical Cerenkov instability,PARTICLE SIMULATION,PLASMAS,Particle-in-cell,Plasma simulation,Spectral solver}, -month = {jun}, +month = {Jun}, pages = {124--138}, publisher = {ACADEMIC PRESS INC ELSEVIER SCIENCE, 525 B ST, STE 1900, SAN DIEGO, CA 92101-4495 USA}, title = {{Modeling of laser wakefield acceleration in Lorentz boosted frame using EM-PIC code with spectral solver}}, @@ -616,13 +607,13 @@ @article{Yu2014 year = {2014} } @article{Vaypop2011, -author = {Vay, J -L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C G R and Esarey, E and Schroeder, C B and Leemans, W P and Cormier-Michel, E and Grote, D P}, doi = {10.1063/1.3663841}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {dec}, +journal = {Physics of Plasmas}, +month = {Dec}, number = {12}, -title = {{Modeling Of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, +title = {{Modeling Of 10 GeV-1 TeV Laser-Plasma Accelerators Using Lorentz Boosted Simulations}}, volume = {18}, year = {2011} } @@ -631,7 +622,7 @@ @article{LehePRSTAB13 doi = {10.1103/PhysRevSTAB.16.021301}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{Numerical growth of emittance in simulations of laser-wakefield acceleration}}, volume = {16}, @@ -641,7 +632,7 @@ @article{Langdoncpc92 author = {Langdon, A B}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {jul}, +month = {Jul}, number = {3}, pages = {447--450}, title = {{On Enforcing Gauss Law In Electromagnetic Particle-In-Cell Codes}}, @@ -653,31 +644,29 @@ @article{Colellajcp2010 doi = {10.1016/J.Jcp.2009.07.004}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {4}, pages = {947--957}, title = {{Controlling Self-Force Errors At Refinement Boundaries For Amr-Pic}}, volume = {229}, year = {2010} } - @article{Coleieee1997, author = {Cole, J. B.}, issn = {0018-9480}, journal = {Ieee Transactions On Microwave Theory And Techniques}, -month = {jun}, +month = {Jun}, number = {6}, pages = {991--996}, title = {{A High-Accuracy Realization Of The Yee Algorithm Using Non-Standard Finite Differences}}, volume = {45}, year = {1997} } - @article{Coleieee2002, author = {Cole, J. B.}, issn = {0018-926X}, journal = {Ieee Transactions On Antennas And Propagation}, -month = {sep}, +month = {Sep}, number = {9}, pages = {1185--1191}, title = {{High-Accuracy Yee Algorithm Based On Nonstandard Finite Differences: New Developments And Verifications}}, @@ -685,7 +674,6 @@ @article{Coleieee2002 year = {2002}, doi = {10.1109/Tap.2002.801268}, } - @article{HajimaNIM09, annote = {Workshop on Compton Sources for X-gamma Rays, Porto Conte, ITALY, SEP 07-12, 2008}, author = {Hajima, R and Kikuzawa, N and Nishimori, N and Hayakawa, T and Shizuma, T and Kawase, K and Kando, M and Minehara, E and Toyokawa, H and Ohgaki, H}, @@ -693,7 +681,7 @@ @article{HajimaNIM09 institution = {Ist Nazl Fis Nucl; ICFA}, issn = {0168-9002}, journal = {NUCLEAR INSTRUMENTS {\&} METHODS IN PHYSICS RESEARCH SECTION A-ACCELERATORS SPECTROMETERS DETECTORS AND ASSOCIATED EQUIPMENT}, -month = {sep}, +month = {Sep}, number = {1}, pages = {S57--S61}, title = {{Detection of radioactive isotopes by using laser Compton scattered gamma-ray beams}}, @@ -705,7 +693,7 @@ @article{MatlisJOSA11 doi = {10.1364/JOSAB.28.000023}, issn = {0740-3224}, journal = {JOURNAL OF THE OPTICAL SOCIETY OF AMERICA B-OPTICAL PHYSICS}, -month = {jan}, +month = {Jan}, number = {1}, pages = {23--27}, title = {{Single-shot spatiotemporal measurements of ultrashort THz waveforms using temporal electric-field cross correlation}}, @@ -713,7 +701,6 @@ @article{MatlisJOSA11 year = {2011} } @article{Quickpic2, - author = {An, Weiming and Decyk, Viktor K. and Mori, Warren B. and Antonsen, Thomas M.}, doi = {10.1016/j.jcp.2013.05.020}, issn = {00219991}, @@ -730,29 +717,32 @@ @misc{Cowanpriv2010 year = {2010} } @inproceedings{Geddesscidac09, -author = {{Geddes et al.}, Cgr}, +author = {Geddes, Cameron G.R. and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and Hamill, Paul and Messmer, Peter and Mullowney, Paul and Nieter, Chet and Paul, Kevin and Shasharina, Svetlana and Veitzer, Seth and Weber, Gunther and Rubel, Oliver and Ushizima, Daniela and Bethel, Wes and Wu, John}, booktitle = {Scidac Review 13}, -pages = {13}, +journal = {SciDAC Review}, +number = {13}, +pages = {13--21}, title = {{Laser Plasma Particle Accelerators: Large Fields For Smaller Facility Sources}}, +url = {https://www.osti.gov/biblio/971264}, year = {2009} } @article{Gomberoffpop2007, author = {Gomberoff, K and Fajans, J and Friedman, A and Grote, D and Vay, J.-L. and Wurtele, J S}, doi = {10.1063/1.2778420}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {oct}, +journal = {Physics of Plasmas}, +month = {Oct}, number = {10}, title = {{Simulations Of Plasma Confinement In An Antihydrogen Trap}}, volume = {14}, year = {2007} } @article{Morapop1997, -author = {Mora, P and Antonsen, Tm}, +author = {Mora, P and Antonsen, T M}, doi = {10.1063/1.872134}, issn = {1070-664X}, journal = {Phys. Plasmas}, -month = {jan}, +month = {Jan}, number = {1}, pages = {217--229}, title = {{Kinetic Modeling Of Intense, Short Laser Pulses Propagating In Tenuous Plasmas}}, @@ -770,12 +760,12 @@ @inproceedings{Cowanaac08 } @article{Cohenprstab2009, annote = {17Th International Symposium On Heavy Ion Inertial Fusion, Tokyo, Japan, Aug 04-08, 2008}, -author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J -L.}, +author = {Cohen, R H and Friedman, A and Grote, D P and Vay, J-L}, doi = {10.1016/J.Nima.2009.03.083}, institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {53--55}, title = {{An Implicit ``Drift-Lorentz{\{}''{\}} Mover For Plasma And Beam Simulations}}, @@ -783,8 +773,7 @@ @article{Cohenprstab2009 year = {2009} } @article{Kaganovich2012, - -author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean Luc and Friedman, Alex}, +author = {Kaganovich, Igor D. and Massidda, Scott and Startsev, Edward A. and Davidson, Ronald C. and Vay, Jean-Luc and Friedman, Alex}, journal = {Nuclear Instruments and Methods in Physics Research, Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, keywords = {Beam dynamics,Longitudinal compression,Voltage errors}, pages = {48--63}, @@ -792,20 +781,22 @@ @article{Kaganovich2012 volume = {678}, year = {2012} } -@article{Vincenti2016, -author = {Vincenti, H. and Lehe, R. and Sasanka, R. and Vay, J-L.}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Vincenti et al. - 2016 - An efficient and portable SIMD algorithm for chargecurrent deposition in Particle-In-Cell codes.pdf:pdf}, -journal = {Computer Programs in Physics}, -pages = {To appear}, -title = {{An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}}, -url = {https://arxiv.org/abs/1601.02056}, -year = {2016} +@article{VincentiCPC2017, +author = {H. Vincenti and M. Lobet and R. Lehe and R. Sasanka and J.-L. Vay}, +doi = {https://doi.org/10.1016/j.cpc.2016.08.023}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +pages = {145-154}, +title = {An efficient and portable SIMD algorithm for charge/current deposition in Particle-In-Cell codes}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465516302764}, +volume = {210}, +year = {2017}, } @article{Vaypop2008, -author = {Vay, J L}, +author = {Vay, J-L}, doi = {10.1063/1.2837054}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56701}, title = {{Simulation Of Beams Or Plasmas Crossing At Relativistic Velocity}}, @@ -817,7 +808,7 @@ @article{Wieland2016 doi = {10.3847/0004-637X/820/1/62}, issn = {1538-4357}, journal = {The Astrophysical Journal}, -month = {mar}, +month = {Mar}, number = {1}, pages = {62}, title = {{NONRELATIVISTIC PERPENDICULAR SHOCKS MODELING YOUNG SUPERNOVA REMNANTS: NONSTATIONARY DYNAMICS AND PARTICLE ACCELERATION AT FORWARD AND REVERSE SHOCKS}}, @@ -831,11 +822,11 @@ @misc{Vay url = {http://www.nersc.gov/assets/Uploads/DOEExascaleReviewVay.pdf} } @article{Faurenature04, -author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, Jp and Burgy, F and Malka, V}, +author = {Faure, J and Glinec, Y and Pukhov, A and Kiselev, S and Gordienko, S and Lefebvre, E and Rousseau, J P and Burgy, F and Malka, V}, doi = {10.1038/Nature02963}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {541--544}, title = {{A Laser-Plasma Accelerator Producing Monoenergetic Electron Beams}}, @@ -843,11 +834,11 @@ @article{Faurenature04 year = {2004} } @article{Greenwoodjcp04, -author = {Greenwood, Ad and Cartwright, Kl and Luginsland, Jw and Baca, Ea}, +author = {Greenwood, A D and Cartwright, K L and Luginsland, J W and Baca, E A}, doi = {10.1016/J.Jcp.2004.06.021}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {665--684}, title = {{On The Elimination Of Numerical Cerenkov Radiation In Pic Simulations}}, @@ -855,8 +846,7 @@ @article{Greenwoodjcp04 year = {2004} } @article{GodfreyIEEE2014, - -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {IEEE Transactions on Plasma Science}, keywords = {Accelerators,numerical stability,particle beams,particle-in-cell (PIC),relativistic effects,simulation,spectral methods}, number = {5}, @@ -874,23 +864,28 @@ @inproceedings{Cowanicap09 year = {2009} } @article{Wuprstab2011, -author = {Wu, H -C. and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, +author = {Wu, H-C and Meyer-Ter-Vehn, J and Hegelich, B M and Fernandez, J C}, doi = {10.1103/Physrevstab.14.070702}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {jul}, +month = {Jul}, number = {7}, title = {{Nonlinear Coherent Thomson Scattering From Relativistic Electron Sheets As A Means To Produce Isolated Ultrabright Attosecond X-Ray Pulses}}, volume = {14}, year = {2011} } @article{VayAAC2010, -author = {Vay, J -. L and Geddes, C G R and Benedetti, C and Bruhwiler, D L and Cormier-Michel, E and Cowan, B M and Cary, J R and Grote, D P}, +author = {Vay, J.‐L. and Geddes, C. G. R. and Benedetti, C. and Bruhwiler, D. L. and Cormier‐Michel, E. and Cowan, B. M. and Cary, J. R. and Grote, D. P.}, doi = {10.1063/1.3520322}, -journal = {Aip Conference Proceedings}, +eprint = {https://pubs.aip.org/aip/acp/article-pdf/1299/1/244/11928106/244\_1\_online.pdf}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, keywords = {[978-0-7354-0853-1/10/{\$}30.00]}, +month = {Nov}, +number = {1}, pages = {244--249}, title = {{Modeling Laser Wakefield Accelerators In A Lorentz Boosted Frame}}, +url = {https://doi.org/10.1063/1.3520322}, volume = {1299}, year = {2010} } @@ -905,21 +900,23 @@ @article{Vayprl07 year = {2007} } @article{VayJCP2013, - -author = {Vay, Jean Luc and Haber, Irving and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B.}, +doi = {10.1016/j.jcp.2013.03.010}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, keywords = {Domain decomposition,Electromagnetic,FFT,Fast fourier transform,Parallel,Particle-In-Cell,Spectral}, +month = {Jun}, pages = {260--268}, title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, volume = {243}, year = {2013} } @article{Kaganovichpop2004, -author = {Kaganovich, Id and Startsev, Ea and Davidson, Rc}, +author = {Kaganovich, I D and Startsev, E A and Davidson, R C}, doi = {10.1063/1.1758945}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {jul}, +journal = {Physics of Plasmas}, +month = {Jul}, number = {7}, pages = {3546--3552}, title = {{Nonlinear Plasma Waves Excitation By Intense Ion Beams In Background Plasma}}, @@ -928,12 +925,12 @@ @article{Kaganovichpop2004 } @article{Cohenpop2005, annote = {46Th Annual Meeting Of The Division Of Plasma Physics Of The American-Physical-Society, Savannah, Ga, Nov 15-19, 2004}, -author = {Cohen, Rh and Friedman, A and Covo, Mk and Lund, Sm and Molvik, Aw and Bieniosek, Fm and Seidl, Pa and Vay, Jl and Stoltz, P and Veitzer, S}, +author = {Cohen, R H and Friedman, A and Covo, M K and Lund, S M and Molvik, A W and Bieniosek, F M and Seidl, P A and Vay, J-L and Stoltz, P and Veitzer, S}, doi = {10.1063/1.1882292}, institution = {Amer Phys Soc}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Simulating Electron Clouds In Heavy-Ion Accelerators}}, volume = {12}, @@ -941,11 +938,11 @@ @article{Cohenpop2005 } @article{Vayfed1996, annote = {7Th International Symposium On Heavy Ion Inertial Fusion, Princeton Plasma Phys Lab, Princeton, Nj, Sep 06-09, 1995}, -author = {Vay, Jl and Deutsch, C}, +author = {Vay, J-L and Deutsch, C}, doi = {10.1016/S0920-3796(96)00502-9}, issn = {0920-3796}, journal = {Fusion Engineering And Design}, -month = {nov}, +month = {Nov}, pages = {467--476}, title = {{A Three-Dimensional Electromagnetic Particle-In-Cell Code To Simulate Heavy Ion Beam Propagation In The Reaction Chamber}}, volume = {32-33}, @@ -962,10 +959,8 @@ @article{Fengjcp09 year = {2009} } @article{BESAC2013, - author = {BESAC}, doi = {10.1037/a0034573}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/582d1799e353fccb9458ff0ccb827db4b01d7308.pdf:pdf}, issn = {1935-990X}, journal = {The American psychologist}, number = {7}, @@ -977,11 +972,11 @@ @article{BESAC2013 year = {2013} } @article{Prostprstab2005, -author = {Prost, Lr and Seidl, Pa and Bieniosek, Fm and Celata, Cm and Faltens, A and Baca, D and Henestroza, E and Kwan, Jw and Leitner, M and Waldron, Wl and Cohen, R and Friedman, A and Grote, D and Lund, Sm and Molvik, Aw and Morse, E}, +author = {Prost, L R and Seidl, P A and Bieniosek, F M and Celata, C M and Faltens, A and Baca, D and Henestroza, E and Kwan, J W and Leitner, M and Waldron, W L and Cohen, R and Friedman, A and Grote, D and Lund, S M and Molvik, A W and Morse, E}, doi = {10.1103/Physrevstab.8.020101}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {feb}, +month = {Feb}, number = {2}, title = {{High Current Transport Experiment For Heavy Ion Inertial Fusion}}, volume = {8}, @@ -1000,13 +995,12 @@ @inproceedings{Godfrey2013PPPS title = {{Numerical Stability of the Pseudo-Spectral EM PIC Algorithm}}, year = {2013} } - @article{Vaypop04, author = {Vay, J.-L. and Colella, P and Kwan, J W and Mccorquodale, P and Serafini, D B and Friedman, A and Grote, D P and Westenskow, G and Adam, J.-C. and Heron, A and Haber, I}, doi = {10.1063/1.1689669}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {2928--2934}, title = {{Application Of Adaptive Mesh Refinement To Particle-In-Cell Simulations Of Plasmas And Beams}}, @@ -1017,7 +1011,7 @@ @article{Friedmanjcp90 author = {Friedman, A}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {292--312}, title = {{A 2Nd-Order Implicit Particle Mover With Adjustable Damping}}, @@ -1027,8 +1021,8 @@ @article{Friedmanjcp90 @article{Rittershoferpop2010, author = {Rittershofer, W and Schroeder, C B and Esarey, E and Gruner, F J and Leemans, W P}, doi = {10.1063/1.3430638}, -journal = {Physics Of Plasmas}, -month = {jun}, +journal = {Physics of Plasmas}, +month = {Jun}, number = {6}, pages = {63104}, title = {{Tapered Plasma Channels To Phase-Lock Accelerating And Focusing Forces In Laser-Plasma Accelerators}}, @@ -1043,16 +1037,17 @@ @article{Vorpal volume = {196}, year = {2004} } -@article{LiARXIV2016, - -archivePrefix = {arXiv}, -arxivId = {1605.01496}, -author = {Li, Fei and Yu, Peicheng and Xu, Xinlu and Fiuza, Frederico and Decyk, Viktor K. and Dalichaouch, Thamine and Davidson, Asher and Tableman, Adam and An, Weiming and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Mori, Warren B.}, -eprint = {1605.01496}, -month = {may}, -title = {{Controlling the Numerical Cerenkov Instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, -url = {http://arxiv.org/abs/1605.01496}, -year = {2016} +@article{LiCPC2017, +author = {Fei Li and Peicheng Yu and Xinlu Xu and Frederico Fiuza and Viktor K. Decyk and Thamine Dalichaouch and Asher Davidson and Adam Tableman and Weiming An and Frank S. Tsung and Ricardo A. Fonseca and Wei Lu and Warren B. Mori}, +doi = {https://doi.org/10.1016/j.cpc.2017.01.001}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {PIC simulation, Hybrid Maxwell solver, Relativistic plasma drift, Numerical Cerenkov instability, Lorentz boosted frame}, +pages = {6-17}, +title = {{Controlling the numerical Cerenkov instability in PIC simulations using a customized finite difference Maxwell solver and a local FFT based current correction}}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465517300012}, +volume = {214}, +year = {2017}, } @article{XuJCP2013, author = {Xu, Xinlu and Yu, Peicheng and Martins, Samual F and Tsung, Frank S and Decyk, Viktor K and Vieira, Jorge and Fonseca, Ricardo A and Lu, Wei and Silva, Luis O and Mori, Warren B}, @@ -1072,17 +1067,17 @@ @article{Cormierpre08 doi = {10.1103/Physreve.78.016404}, issn = {1539-3755}, journal = {Physical Review E}, -month = {jul}, +month = {Jul}, number = {1, Part 2}, title = {{Unphysical Kinetic Effects In Particle-In-Cell Modeling Of Laser Wakefield Accelerators}}, volume = {78}, year = {2008} } @article{Berengerjcp96, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {2}, pages = {363--379}, title = {{Three-Dimensional Perfectly Matched Layer For The Absorption Of Electromagnetic Waves}}, @@ -1090,16 +1085,21 @@ @article{Berengerjcp96 year = {1996} } @article{Kalmykovprl09, -author = {{Kalmykov et al.}, S}, +author = {Kalmykov, S. and Yi, S. A. and Khudik, V. and Shvets, G.}, +doi = {10.1103/PhysRevLett.103.135004}, +issue = {13}, journal = {Phys. Rev. Lett.}, +month = {Sep}, +numpages = {4}, pages = {135004}, -title = {{No Title}}, +publisher = {American Physical Society}, +title = {{Electron Self-Injection and Trapping into an Evolving Plasma Bubble}}, +url = {https://link.aps.org/doi/10.1103/PhysRevLett.103.135004}, volume = {103}, year = {2009} } @article{Geddes2015, - -author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, +author = {Geddes, Cameron G R and Rykovanov, Sergey and Matlis, Nicholas H. and Steinke, Sven and Vay, Jean-Luc and Esarey, Eric H. and Ludewigt, Bernhard and Nakamura, Kei and Quiter, Brian J. and Schroeder, Carl B. and Toth, Csaba and Leemans, Wim P.}, journal = {Nuclear Instruments and Methods in Physics Research, Section B: Beam Interactions with Materials and Atoms}, keywords = {Active interrogation,Homeland security,Laser plasma accelerator,Monoenergetic photon source,Nonproliferation}, pages = {116--121}, @@ -1112,7 +1112,7 @@ @article{Antonsenprl1992 author = {Antonsen, T M and Mora, P}, doi = {10.1103/Physrevlett.69.2204}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {15}, pages = {2204--2207}, title = {{Self-Focusing And Raman-Scattering Of Laser-Pulses In Tenuous Plasmas}}, @@ -1120,8 +1120,7 @@ @article{Antonsenprl1992 year = {1992} } @article{GodfreyCPC2015, - -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Computer Physics Communications}, keywords = {Numerical stability,Particle-in-cell,Pseudo-Spectral Time-Domain,Relativistic beam}, pages = {221--225}, @@ -1142,10 +1141,10 @@ @article{VayCSD12 year = {2012} } @article{Esirkepovcpc01, -author = {Esirkepov, Tz}, +author = {Esirkepov, T Z}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {apr}, +month = {Apr}, number = {2}, pages = {144--153}, title = {{Exact Charge Conservation Scheme For Particle-In-Cell Simulation With An Arbitrary Form-Factor}}, @@ -1156,8 +1155,8 @@ @article{Martinspop10 author = {Martins, S F and Fonseca, R A and Vieira, J and Silva, L O and Lu, W and Mori, W B}, doi = {10.1063/1.3358139}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56705}, title = {{Modeling Laser Wakefield Accelerator Experiments With Ultrafast Particle-In-Cell Simulations In Boosted Frames}}, @@ -1166,7 +1165,7 @@ @article{Martinspop10 } @inproceedings{Habericnsp73, address = {Berkeley, Ca}, -author = {Haber, I and Lee, R and Klein, Hh and Boris, Jp}, +author = {Haber, I and Lee, R and Klein, H H and Boris, J P}, booktitle = {Proc. Sixth Conf. Num. Sim. Plasmas}, pages = {46--48}, title = {{Advances In Electromagnetic Simulation Techniques}}, @@ -1177,7 +1176,7 @@ @article{Martinscpc10 doi = {10.1016/J.Cpc.2009.12.023}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {may}, +month = {May}, number = {5}, pages = {869--875}, title = {{Numerical Simulations Of Laser Wakefield Accelerators In Optimal Lorentz Frames}}, @@ -1189,7 +1188,7 @@ @article{Vaycpc04 doi = {10.1016/J.Cpc.2004.06.026}, issn = {0010-4655}, journal = {Computer Physics Communications}, -month = {dec}, +month = {Dec}, number = {1-3}, pages = {171--177}, title = {{Asymmetric Pml For The Absorption Of Waves. Application To Mesh Refinement In Electromagnetic Particle-In-Cell Plasma Simulations}}, @@ -1197,8 +1196,7 @@ @article{Vaycpc04 year = {2004} } @article{GodfreyJCP2014_PSATD, - -author = {Godfrey, Brendan B. and Vay, Jean Luc and Haber, Irving}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc and Haber, Irving}, journal = {Journal of Computational Physics}, keywords = {Numerical stability,Particle-in-cell,Pseudo-spectral,Relativistic beam}, pages = {689--704}, @@ -1211,20 +1209,19 @@ @article{PruetJAP06 doi = {10.1063/1.2202005}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {jun}, +month = {Jun}, number = {12}, title = {{Detecting clandestine material with nuclear resonance fluorescence}}, volume = {99}, year = {2006} } @article{YuCPC2015-Circ, - author = {Yu, Peicheng and Xu, Xinlu and Tableman, Adam and Decyk, Viktor K. and Tsung, Frank S. and Fiuza, Frederico and Davidson, Asher and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.08.026}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,CODES,Hybrid Maxwell solver,IN-CELL SIMULATION,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,OSIRIS,PARTICLE SIMULATION,PIC SIMULATIONS,PIC simulation,PLASMAS,Quasi-3D algorithm,Relativistic plasma drift,STABILITY}, -month = {dec}, +month = {Dec}, pages = {144--152}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit}}, @@ -1233,10 +1230,10 @@ @article{YuCPC2015-Circ year = {2015} } @article{Berengerjcp94, -author = {Berenger, Jp}, +author = {Berenger, J P}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {oct}, +month = {Oct}, number = {2}, pages = {185--200}, title = {{A Perfectly Matched Layer For The Absorption Of Electromagnetic-Waves}}, @@ -1253,11 +1250,11 @@ @article{Rubel2016 year = {2016} } @article{Geddesnature04, -author = {Geddes, Cgr and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, Cb and Bruhwiler, D and Nieter, C and Cary, J and Leemans, Wp}, +author = {Geddes, C G R and Toth, C and {Van Tilborg}, J and Esarey, E and Schroeder, C B and Bruhwiler, D and Nieter, C and Cary, J and Leemans, W P}, doi = {10.1038/Nature02900}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {538--541}, title = {{High-Quality Electron Beams From A Laser Wakefield Accelerator Using Plasma-Channel Guiding}}, @@ -1265,11 +1262,11 @@ @article{Geddesnature04 year = {2004} } @article{Munzjcp2000, -author = {Munz, Cd and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, +author = {Munz, C D and Omnes, P and Schneider, R and Sonnendrucker, E and Voss, U}, doi = {10.1006/Jcph.2000.6507}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {jul}, +month = {Jul}, number = {2}, pages = {484--511}, title = {{Divergence Correction Techniques For Maxwell Solvers Based On A Hyperbolic Model}}, @@ -1277,10 +1274,9 @@ @article{Munzjcp2000 year = {2000} } @article{DavidsonJCP2015, - -author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F.S. and Lu, W. and Vieira, J. and Fonseca, R.A. and Silva, L.O. and Mori, W.B.}, +author = {Davidson, A. and Tableman, A. and An, W. and Tsung, F. S. and Lu, W. and Vieira, J. and Fonseca, R. A. and Silva, L. O. and Mori, W. B.}, doi = {10.1016/j.jcp.2014.10.064}, -issn = {00219991}, +issn = {0021-9991}, journal = {Journal of Computational Physics}, pages = {1063--1077}, title = {{Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in \Phi into OSIRIS}}, @@ -1301,8 +1297,7 @@ @article{GodfreyJCP2014 year = {2014} } @article{Godfrey2013, - -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Esirkepov algorithm,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {33--46}, @@ -1316,8 +1311,8 @@ @article{Gilsonpop2010 doi = {10.1063/1.3354109}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Studies Of Emittance Growth And Halo Particle Production In Intense Charged Particle Beams Using The Paul Trap Simulator Experiment}}, volume = {17}, @@ -1348,7 +1343,7 @@ @article{Steinke2016 author = {Steinke, S and van Tilborg, J and Benedetti, C and Geddes, C G R and Schroeder, C B and Daniels, J and Swanson, K K and Gonsalves, A J and Nakamura, K and Matlis, N H and Shaw, B H and Esarey, E and Leemans, W P}, issn = {0028-0836}, journal = {Nature}, -month = {feb}, +month = {Feb}, number = {7589}, pages = {190--193}, publisher = {Nature Publishing Group, a division of Macmillan Publishers Limited. All Rights Reserved.}, @@ -1376,7 +1371,7 @@ @article{Genonioppj2010 year = {2010} } @article{LewisJCP1972, -author = {Lewis, H.Ralph}, +author = {Lewis, H. Ralph}, doi = {http://dx.doi.org/10.1016/0021-9991(72)90044-7}, issn = {0021-9991}, journal = {Journal of Computational Physics}, @@ -1400,11 +1395,11 @@ @article{Vay2002 year = {2002} } @article{Manglesnature04, -author = {Mangles, Spd and Murphy, Cd and Najmudin, Z and Thomas, Agr and Collier, Jl and Dangor, Ae and Divall, Ej and Foster, Ps and Gallacher, Jg and Hooker, Cj and Jaroszynski, Da and Langley, Aj and Mori, Wb and Norreys, Pa and Tsung, Fs and Viskup, R and Walton, Br and Krushelnick, K}, +author = {Mangles, S P D and Murphy, C D and Najmudin, Z and Thomas, A G R and Collier, J L and Dangor, A E and Divall, E J and Foster, P S and Gallacher, J G and Hooker, C J and Jaroszynski, D A and Langley, A J and Mori, W B and Norreys, P A and Tsung, F S and Viskup, R and Walton, B R and Krushelnick, K}, doi = {10.1038/Nature02939}, issn = {0028-0836}, journal = {Nature}, -month = {sep}, +month = {Sep}, number = {7008}, pages = {535--538}, title = {{Monoenergetic Beams Of Relativistic Electrons From Intense Laser-Plasma Interactions}}, @@ -1423,8 +1418,8 @@ @inproceedings{Schroederaac08 @article{Vaypop98, author = {Vay, J.-L. and Deutsch, C}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {apr}, +journal = {Physics of Plasmas}, +month = {Apr}, number = {4}, pages = {1190--1197}, title = {{Charge Compensated Ion Beam Propagation In A Reactor Sized Chamber}}, @@ -1432,11 +1427,11 @@ @article{Vaypop98 year = {1998} } @article{Friedmanjcp1991, -author = {Friedman, A and Parker, Se and Ray, Sl and Birdsall, Ck}, +author = {Friedman, A and Parker, S E and Ray, S L and Birdsall, C K}, doi = {10.1016/0021-9991(91)90265-M}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {sep}, +month = {Sep}, number = {1}, pages = {54--70}, title = {{Multiscale Particle-In-Cell Plasma Simulation}}, @@ -1444,11 +1439,11 @@ @article{Friedmanjcp1991 year = {1991} } @article{Liumotl1997, -author = {Liu, Qh}, +author = {Liu, Q H}, doi = {10.1002/(Sici)1098-2760(19970620)15:3<158::Aid-Mop11>3.3.Co;2-T}, issn = {0895-2477}, journal = {Microwave And Optical Technology Letters}, -month = {jun}, +month = {Jun}, number = {3}, pages = {158--165}, title = {{The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength}}, @@ -1475,45 +1470,19 @@ @article{Leemansphysicstoday10 author = {Leemans, Wim and Esarey, Eric}, issn = {0031-9228}, journal = {Physics Today}, -month = {mar}, +month = {Mar}, number = {3}, pages = {44--49}, title = {{Laser-Driven Plasma-Wave Electron Accelerators}}, volume = {62}, year = {2009} } -@article{Lehe2015, - -archivePrefix = {arXiv}, -arxivId = {1507.04790}, -author = {Lehe, Remi and Kirchen, Manuel and Andriyash, Igor a. and Godfrey, Brendan B. and Vay, Jean-Luc}, -eprint = {1507.04790}, -isbn = {5104866785}, -journal = {arXiv.org}, -keywords = {cylindrical geometry,hankel transform,particle-in-cell,pseudo-spectral}, -pages = {1507.04790v1}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -url = {http://arxiv.org/abs/1507.04790}, -volume = {physics.pl}, -year = {2015} -} -@article{VayJCP13, -author = {Vay, Jean-Luc and Haber, Irving and Godfrey, Brendan B}, -doi = {10.1016/j.jcp.2013.03.010}, -issn = {0021-9991}, -journal = {Journal of Computational Physics}, -month = {jun}, -pages = {260--268}, -title = {{A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas}}, -volume = {243}, -year = {2013} -} @inproceedings{Martinspac09, address = {Vancouver, Canada}, annote = {Th4Gbc05}, -author = {{Martins et al.}, S F}, +author = {Martins, S F and Fonseca, R A and Silva, L O and Mori, W B}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Boosted Frame Pic Simulations Of Lwfa: Towards The Energy Frontier}}, +title = {{Boosted Frame PIC Simulations of LWFA: Towards the Energy Frontier}}, year = {2009} } @article{Vayscidac09, @@ -1535,32 +1504,32 @@ @article{Sprangleprl90 author = {Sprangle, P and Esarey, E and Ting, A}, issn = {0031-9007}, journal = {Physical Review Letters}, -month = {apr}, +month = {Apr}, number = {17}, pages = {2011--2014}, -title = {{Nonlinear-Theory Of Intense Laser-Plasma Interactions}}, +title = {{Nonlinear theory of intense laser-plasma interactions}}, volume = {64}, year = {1990} } @article{Molvikpop2007, annote = {48Th Annual Meeting Of The Division Of Plasma Physics Of The Aps, Philadelphia, Pa, Jan 30-Nov 03, 2006}, -author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L. and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, +author = {Molvik, A W and Covo, M Kireeff and Cohen, R and Friedman, A and Lund, S M and Sharp, W and Vay, J-L and Baca, D and Bieniosek, F and Leister, C and Seidl, P}, doi = {10.1063/1.2436850}, institution = {Aps, Div Plasma Phys}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, title = {{Quantitative Experiments With Electrons In A Positively Charged Beam}}, volume = {14}, year = {2007} } @article{QuiterJAP08, -author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M -A.}, +author = {Quiter, B J and Prussin, S G and Pohl, B and Hall, J and Trebes, J and Stone, G and Descalle, M-A}, doi = {10.1063/1.2876028}, issn = {0021-8979}, journal = {JOURNAL OF APPLIED PHYSICS}, -month = {mar}, +month = {Mar}, number = {6}, title = {{A method for high-resolution x-ray imaging of intermodal cargo containers for fissionable materials}}, volume = {103}, @@ -1568,7 +1537,7 @@ @article{QuiterJAP08 } @book{Taflove2000, address = {Norwood}, -author = {Taflove and Hagness}, +author = {Allen Taflove and Susan C. Hagness}, edition = {2Nd}, publisher = {Ma: Artech House}, title = {{Computational Electrodynamics: The Finite-Difference Time-Domain Method}}, @@ -1578,7 +1547,7 @@ @article{Schroederprl2011 author = {Schroeder, C B and Benedetti, C and Esarey, E and Leemans, W P}, doi = {10.1103/Physrevlett.106.135002}, journal = {Physical Review Letters}, -month = {mar}, +month = {Mar}, number = {13}, pages = {135002}, title = {{Nonlinear Pulse Propagation And Phase Velocity Of Laser-Driven Plasma Waves}}, @@ -1600,10 +1569,8 @@ @article{Logannim2007 year = {2007} } @article{Yu2016, - author = {Yu, Peicheng and Xu, Xinlu and Davidson, Asher and Tableman, Adam and Dalichaouch, Thamine and Li, Fei and Meyers, Michael D. and An, Weiming and Tsung, Frank S. and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.jcp.2016.04.014}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Yu et al. - 2016 - Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry.pdf:pdf}, issn = {00219991}, journal = {Journal of Computational Physics}, title = {{Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}}, @@ -1611,7 +1578,7 @@ @article{Yu2016 } @article{Borisjcp73, address = {525 B St, Ste 1900, San Diego, Ca 92101-4495}, -author = {Boris, Jp and Lee, R}, +author = {Boris, J P and Lee, R}, issn = {0021-9991}, journal = {Journal of Computational Physics}, number = {1}, @@ -1624,7 +1591,7 @@ @article{Borisjcp73 } @inproceedings{BorisICNSP70, address = {Naval Res. Lab., Wash., D. C.}, -author = {Boris, Jp}, +author = {Boris, J P}, booktitle = {Proc. Fourth Conf. Num. Sim. Plasmas}, pages = {3--67}, title = {{Relativistic Plasma Simulation-Optimization of a Hybrid Code}}, @@ -1638,7 +1605,7 @@ @article{Vayjcp01 author = {Vay, J.-L.}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {feb}, +month = {Feb}, number = {1}, pages = {72--98}, title = {{An Extended Fdtd Scheme For The Wave Equation: Application To Multiscale Electromagnetic Simulation}}, @@ -1663,10 +1630,10 @@ @article{Leemansnature06 doi = {10.1038/Nphys418}, issn = {1745-2473}, journal = {Nature Physics}, -month = {oct}, +month = {Oct}, number = {10}, pages = {696--699}, -title = {{Gev Electron Beams From A Centimetre-Scale Accelerator}}, +title = {{GeV electron beams from a centimetre-scale accelerator}}, volume = {2}, year = {2006} } @@ -1675,7 +1642,7 @@ @article{ChenPRSTAB13 doi = {10.1103/PhysRevSTAB.16.030701}, issn = {1098-4402}, journal = {PHYSICAL REVIEW SPECIAL TOPICS-ACCELERATORS AND BEAMS}, -month = {mar}, +month = {Mar}, number = {3}, title = {{Modeling classical and quantum radiation from laser-plasma accelerators}}, volume = {16}, @@ -1686,7 +1653,7 @@ @article{Hipace doi = {10.1088/0741-3335/56/8/084012}, issn = {0741-3335}, journal = {Plasma Physics and Controlled Fusion}, -month = {aug}, +month = {Aug}, number = {8}, pages = {084012}, publisher = {IOP Publishing}, @@ -1696,7 +1663,7 @@ @article{Hipace year = {2014} } @article{Morsenielson1971, -author = {Morse, Rl and Nielson, Cw}, +author = {Morse, R L and Nielson, C W}, doi = {10.1063/1.1693518}, issn = {1070-6631}, journal = {Phys. Fluids}, @@ -1706,15 +1673,18 @@ @article{Morsenielson1971 volume = {14}, year = {1971} } -@article{Vaydpf09, -author = {{Vay et al.}, J.-L.}, -journal = {Arxiv:0909.5603}, -title = {{Speeding Up Simulations Of Relativistic Systems Using An Optimal Boosted Frame}}, +@inproceedings{Vaydpf09, +archivePrefix = {arXiv}, +author = {Vay, J.-L. and Fawley, W. M. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, +booktitle = {{Meeting of the Division of Particles and Fields of the American Physical Society (DPF 2009)}}, +eprint = {0909.5603}, +month = {Sep}, +primaryClass = {physics.acc-ph}, +title = {{Speeding up simulations of relativistic systems using an optimal boosted frame}}, year = {2009} } @misc{Vay2014, - -author = {Vay, Jean Luc and Godfrey, Brendan B.}, +author = {Vay, Jean-Luc and Godfrey, Brendan B.}, booktitle = {Comptes Rendus - Mecanique}, keywords = {Numerical instability,Particle-In-Cell,Plasma simulation,Special relativity}, number = {10-11}, @@ -1724,15 +1694,8 @@ @misc{Vay2014 volume = {342}, year = {2014} } -@article{Lehearxiv2015, -author = {Lehe, R and Kirchen, M and Andriyash, I.{\~{}}A. and Godfrey, B.{\~{}}B. and Vay, J.-L.}, -journal = {arXiv:1507.04790}, -title = {{A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}}, -year = {2015} -} @article{Friedman2014, - -author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean Luc and Haber, Irving and Kishek, Rami A.}, +author = {Friedman, Alex and Cohen, Ronald H. and Grote, David P. and Lund, Steven M. and Sharp, William M. and Vay, Jean-Luc and Haber, Irving and Kishek, Rami A.}, journal = {IEEE Transactions on Plasma Science}, keywords = {Algorithms,Maxwell,Ned Birdsall,computer,laser,numerical simulation,particle beam,particle-in-cell,plasma}, number = {5}, @@ -1743,7 +1706,6 @@ @article{Friedman2014 year = {2014} } @article{Lcode, - author = {Lotov, K. V.}, doi = {10.1063/1.872765}, issn = {1070664X}, @@ -1770,9 +1732,7 @@ @article{GodfreyJCP2014_2 year = {2014} } @article{Turbowave, - author = {Gordon, Daniel F and Mori, W B and Antonsen, Thomas M}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Gordon, Mori, Antonsen - 2000 - A Ponderomotive Guiding Center Particle-in-Cell Code for Efficient Modeling of Laser–Plasma Interactio.pdf:pdf}, journal = {IEEE TRANSACTIONS ON PLASMA SCIENCE}, keywords = {Index Terms—Particle code,plasma simulation}, number = {4}, @@ -1787,7 +1747,7 @@ @article{Habernim2009 institution = {Tokyo Inst Technol, Res Lab Nucl Reactors; Japan Soc Plasma Sci {\&} Nucl Fus Res; Particle Accelerator Soc Japan}, issn = {0168-9002}, journal = {Nuclear Instruments {\&} Methods In Physics Research Section A-Accelerators Spectrometers Detectors And Associated Equipment}, -month = {jul}, +month = {Jul}, number = {1-2}, pages = {64--68}, title = {{Scaled Electron Studies At The University Of Maryland}}, @@ -1805,13 +1765,12 @@ @article{Folegatijpcs2011 year = {2011} } @article{YuCPC2015, - author = {Yu, Peicheng and Xu, Xinlu and Decyk, Viktor K. and Fiuza, Frederico and Vieira, Jorge and Tsung, Frank S. and Fonseca, Ricardo A. and Lu, Wei and Silva, Luis O. and Mori, Warren B.}, doi = {10.1016/j.cpc.2015.02.018}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {ALGORITHM,LASER WAKEFIELD ACCELERATORS,LORENTZ-BOOSTED FRAME,Numerical Cerenkov instability,Numerical dispersion relation,PARTICLE SIMULATION,PLASMA,Particle-in-cell,Plasma simulation,Relativistic drifting plasma,SHOCKS,STABILITY,Spectral solver,WAVES}, -month = {jul}, +month = {Jul}, pages = {32--47}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Elimination of the numerical Cerenkov instability for spectral EM-PIC codes}}, @@ -1824,19 +1783,13 @@ @article{PukhovJPP99 doi = {10.1017/S0022377899007515}, issn = {0022-3778}, journal = {Journal of Plasma Physics}, -month = {apr}, +month = {Apr}, number = {3}, pages = {425--433}, title = {{Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab)}}, volume = {61}, year = {1999} } -@article{Vincentiarxiv2015, -author = {Vincenti, H and Vay, J.-L.}, -journal = {arXiv:1507.05572}, -title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, -year = {2015} -} @inproceedings{Vayipac10, address = {Tokyo, Japan}, annote = {Weobra02}, @@ -1849,7 +1802,7 @@ @article{Cormierprstab2011 author = {Cormier-Michel, E and Esarey, E and Geddes, C G R and Schroeder, C B and Paul, K and Mullowney, P J and Cary, J R and Leemans, W P}, doi = {10.1103/Physrevstab.14.031303}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {mar}, +month = {Mar}, number = {3}, pages = {31303}, title = {{Control Of Focusing Fields In Laser-Plasma Accelerators Using Higher-Order Modes}}, @@ -1857,10 +1810,8 @@ @article{Cormierprstab2011 year = {2011} } @article{Lehe2016, - author = {Lehe, R{\'{e}}mi and Kirchen, Manuel and Andriyash, Igor A. and Godfrey, Brendan B. and Vay, Jean-Luc}, doi = {10.1016/j.cpc.2016.02.007}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Lehe et al. - 2016 - A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.pdf:pdf}, issn = {00104655}, journal = {Computer Physics Communications}, pages = {66--82}, @@ -1877,8 +1828,7 @@ @book{Birdsalllangdon year = {1991} } @inproceedings{Grote2005, - -author = {Grote, David P. and Friedman, Alex and Vay, Jean Luc and Haber, Irving}, +author = {Grote, David P. and Friedman, Alex and Vay, Jean-Luc and Haber, Irving}, booktitle = {AIP Conference Proceedings}, pages = {55--58}, title = {{The WARP code: Modeling high intensity ion beams}}, @@ -1905,7 +1855,7 @@ @article{LeemansPRL2014 author = {Leemans, W P and Gonsalves, A J and Mao, H.-S. and Nakamura, K and Benedetti, C and Schroeder, C B and T{\'{o}}th, Cs. and Daniels, J and Mittelberger, D E and Bulanov, S S and Vay, J.-L. and Geddes, C G R and Esarey, E}, doi = {10.1103/PhysRevLett.113.245002}, journal = {Phys. Rev. Lett.}, -month = {dec}, +month = {Dec}, number = {24}, pages = {245002}, publisher = {American Physical Society}, @@ -1918,7 +1868,7 @@ @article{Abejcp86 author = {Abe, H and Sakairi, N and Itatani, R and Okuda, H}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {apr}, +month = {Apr}, number = {2}, pages = {247--267}, title = {{High-Order Spline Interpolations In The Particle Simulation}}, @@ -1938,11 +1888,11 @@ @article{LeeCPC2015 year = {2015} } @article{Vaylpb2002, -author = {Vay, Jl and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, Dp}, +author = {Vay, J-L and Colella, P and Mccorquodale, P and {Van Straalen}, B and Friedman, A and Grote, D P}, doi = {10.1017/S0263034602204139}, issn = {0263-0346}, journal = {Laser And Particle Beams}, -month = {dec}, +month = {Dec}, number = {4}, pages = {569--575}, title = {{Mesh Refinement For Particle-In-Cell Plasma Simulations: Applications To And Benefits For Heavy Ion Fusion}}, @@ -1962,7 +1912,7 @@ @article{Tzoufrasprl2008 author = {Tzoufras, M and Lu, W and Tsung, F S and Huang, C and Mori, W B and Katsouleas, T and Vieira, J and Fonseca, R A and Silva, L O}, doi = {10.1103/Physrevlett.101.145002}, journal = {Physical Review Letters}, -month = {oct}, +month = {Oct}, number = {14}, pages = {145002}, title = {{Beam Loading In The Nonlinear Regime Of Plasma-Based Acceleration Rid C-6436-2009 Rid B-7680-2009 Rid C-3169-2009}}, @@ -1973,7 +1923,7 @@ @article{Bulanovphysfluid1992 author = {Bulanov, S V and Inovenkov, I N and Kirsanov, V I and Naumova, N M and Sakharov, A S}, doi = {10.1063/1.860046}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7}, pages = {1935--1942}, title = {{Nonlinear Depletion Of Ultrashort And Relativistically Strong Laser-Pulses In An Underdense Plasma}}, @@ -1985,7 +1935,7 @@ @article{Martinsnaturephysics10 doi = {10.1038/Nphys1538}, issn = {1745-2473}, journal = {Nature Physics}, -month = {apr}, +month = {Apr}, number = {4}, pages = {311--316}, title = {{Exploring Laser-Wakefield-Accelerator Regimes For Near-Term Lasers Using Particle-In-Cell Simulation In Lorentz-Boosted Frames}}, @@ -1995,8 +1945,8 @@ @article{Martinsnaturephysics10 @article{Friedmanpop10, author = {Friedman, A and Barnard, J J and Cohen, R H and Grote, D P and Lund, S M and Sharp, W M and Faltens, A and Henestroza, E and Jung, J.-Y. and Kwan, J W and Lee, E P and Leitner, M A and Logan, B G and Vay, J.-L. and Waldron, W L and Davidson, R C and Dorf, M and Gilson, E P and Kaganovich, I D}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {056704 (9 Pp.)}, title = {{Beam Dynamics Of The Neutralized Drift Compression Experiment-Ii, A Novel Pulse-Compressing Ion Accelerator}}, @@ -2006,8 +1956,8 @@ @article{Friedmanpop10 @article{Schroederpop2006, author = {Schroeder, C B and Esarey, E and Shadwick, B A and Leemans, W P}, doi = {10.1063/1.2173960}, -journal = {Physics Of Plasmas}, -month = {mar}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, pages = {33103}, title = {{Trapping, Dark Current, And Wave Breaking In Nonlinear Plasma Waves}}, @@ -2016,12 +1966,12 @@ @article{Schroederpop2006 } @article{Friedmanpfb92, annote = {33Rd Annual Meeting Of The Division Of Plasma Physics Of The American Physical Soc, Tampa, Fl, Nov 04-08, 1991}, -author = {Friedman, A and Grote, Dp and Haber, I}, +author = {Friedman, A and Grote, D P and Haber, I}, doi = {10.1063/1.860024}, institution = {Amer Phys Soc, Div Plasma Phys}, issn = {0899-8221}, journal = {Physics Of Fluids B-Plasma Physics}, -month = {jul}, +month = {Jul}, number = {7, Part 2}, pages = {2203--2210}, title = {{3-Dimensional Particle Simulation Of Heavy-Ion Fusion Beams}}, @@ -2039,7 +1989,7 @@ @article{Winklehnerji2010 doi = {10.1088/1748-0221/5/12/P12001}, issn = {1748-0221}, journal = {Journal of Instrumentation}, -month = {dec}, +month = {Dec}, title = {{Comparison Of Extraction And Beam Transport Simulations With Emittance Measurements From The Ecr Ion Source Venus}}, volume = {5}, year = {2010} @@ -2047,19 +1997,18 @@ @article{Winklehnerji2010 @inproceedings{Vaypac09, address = {Vancouver, Canada}, annote = {Tu1Pbi04}, -author = {{Vay et al.}, J.-L.}, +author = {Vay, J-L and Fawley, W M and Geddes, C G R and Cormier-Michel, E and Grote, D P}, booktitle = {Proc. Particle Accelerator Conference}, -title = {{Application Of The Reduction Of Scale Range In A Lorentz Boosted Frame To The Numerical Simulation Of Particle Acceleration Devices}}, +title = {{Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices}}, year = {2009} } -@article{Vincenti2016a, - +@article{VincentiCPC2017a, author = {Vincenti, H. and Vay, J.-L.}, doi = {10.1016/j.cpc.2015.11.009}, issn = {00104655}, journal = {Computer Physics Communications}, keywords = {3D electromagnetic simulations,ABSORPTION,ALGORITHM,APPROXIMATIONS,CLOSED-FORM EXPRESSIONS,Domain decomposition technique,Effects of stencil truncation errors,PERFECTLY MATCHED LAYER,Perfectly Matched Layers,Pseudo-spectral Maxwell solver,SIMULATIONS,TAYLOR-SERIES,Very high-order Maxwell solver,WAVES}, -month = {mar}, +month = {Mar}, pages = {147--167}, publisher = {ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS}, title = {{Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver}}, @@ -2072,7 +2021,7 @@ @article{Vayjcp02 doi = {10.1006/Jcph.2002.7175}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {dec}, +month = {Dec}, number = {2}, pages = {367--399}, title = {{Asymmetric Perfectly Matched Layer For The Absorption Of Waves}}, @@ -2093,14 +2042,14 @@ @book{Geddesdissertation05 year = {2005} } @article{Tsungpop06, -author = {Tsung, Fs and Lu, W and Tzoufras, M and Mori, Wb and Joshi, C and Vieira, Jm and Silva, Lo and Fonseca, Ra}, +author = {Tsung, F. S. and Lu, W. and Tzoufras, M. and Mori, W. B. and Joshi, C. and Vieira, J. M. and Silva, L. O. and Fonseca, R. A.}, doi = {10.1063/1.2198535}, issn = {1070-664X}, -journal = {Physics Of Plasmas}, -month = {may}, +journal = {Physics of Plasmas}, +month = {May}, number = {5}, pages = {56708}, -title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 Tw Lasers}}, +title = {{Simulation Of Monoenergetic Electron Generation Via Laser Wakefield Accelerators For 5-25 TW Lasers}}, volume = {13}, year = {2006} } @@ -2108,7 +2057,6 @@ @inproceedings{INFERNO address = {Rostock-Warnemünde, Germany}, author = {Benedetti, Carlo and Schroeder, Carl B. and Esarey, Eric and Leemans, Wim P.}, booktitle = {ICAP}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Benedetti et al. - 2012 - Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}ampRNO.pdf:pdf}, pages = {THAAI2}, publisher = {Jacow}, title = {{Efficient Modeling of Laser-plasma Accelerators Using the Ponderomotive-based Code INF{\&}RNO}}, @@ -2120,7 +2068,7 @@ @article{GonsalvesNP2011 doi = {10.1038/NPHYS2071}, issn = {1745-2473}, journal = {NATURE PHYSICS}, -month = {nov}, +month = {Nov}, number = {11}, pages = {862--866}, title = {{Tunable laser plasma accelerator based on longitudinal density tailoring}}, @@ -2132,14 +2080,14 @@ @article{Ohtsuboprstab2010 doi = {10.1103/Physrevstab.13.044201}, issn = {1098-4402}, journal = {Physical Review Special Topics-Accelerators And Beams}, -month = {apr}, +month = {Apr}, number = {4}, title = {{Experimental Study Of Coherent Betatron Resonances With A Paul Trap}}, volume = {13}, year = {2010} } @inproceedings{Warp, -author = {Grote, D P and Friedman, A and Vay, J.-L. and Haber, I}, +author = {Grote, D P and Friedman, A and Vay, J-L and Haber, I}, booktitle = {Aip Conference Proceedings}, issn = {0094-243X}, number = {749}, @@ -2148,11 +2096,11 @@ @inproceedings{Warp year = {2005} } @article{Mccorquodalejcp2004, -author = {Mccorquodale, P and Colella, P and Grote, Dp and Vay, Jl}, +author = {Mccorquodale, P and Colella, P and Grote, D P and Vay, J-L}, doi = {10.1016/J.Jcp.2004.04.022}, issn = {0021-9991}, journal = {Journal of Computational Physics}, -month = {nov}, +month = {Nov}, number = {1}, pages = {34--60}, title = {{A Node-Centered Local Refinement Algorithm For Poisson's Equation In Complex Geometries}}, @@ -2162,7 +2110,6 @@ @article{Mccorquodalejcp2004 @article{Londrillo2010, author = {Londrillo, P. and Benedetti, C. and Sgattoni, A.}, doi = {10.1016/j.nima.2010.01.055}, -file = {:Users/jlvay/Library/Application Support/Mendeley Desktop/Downloaded/Londrillo, Benedetti, Sgattoni - 2010 - Charge preserving high order PIC schemes.pdf:pdf}, issn = {01689002}, journal = {Nuclear Instruments and Methods in Physics Research Section A: Accelerators, Spectrometers, Detectors and Associated Equipment}, number = {1}, @@ -2172,7 +2119,7 @@ @article{Londrillo2010 year = {2010} } @article{GodfreyJCP2014_FDTD, -author = {Godfrey, Brendan B. and Vay, Jean Luc}, +author = {Godfrey, Brendan B. and Vay, Jean-Luc}, journal = {Journal of Computational Physics}, keywords = {Finite difference time-domain,Numerical stability,Particle-in-cell,Relativistic beam}, pages = {1--6}, @@ -2181,11 +2128,11 @@ @article{GodfreyJCP2014_FDTD year = {2014} } @article{Shortley-Weller, -author = {Shortley, Gh and Weller, R}, +author = {Shortley, G H and Weller, R}, doi = {10.1063/1.1710426}, issn = {0021-8979}, journal = {Journal of Applied Physics}, -month = {may}, +month = {May}, number = {5}, pages = {334--348}, title = {{The Numerical Solution Of Laplace's Equation}}, @@ -2193,35 +2140,50 @@ @article{Shortley-Weller year = {1938} } @article{VayPOPL2011, -author = {Vay, Jl and Geddes, C G R and Cormier-Michel, E and Grote, D P}, +author = {Vay, J.-L. and Geddes, C. G. R. and Cormier-Michel, E. and Grote, D. P.}, doi = {10.1063/1.3559483}, -journal = {Physics Of Plasmas}, -month = {mar}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.3559483/16019930/030701\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Mar}, number = {3}, -pages = {30701}, -title = {{Effects Of Hyperbolic Rotation In Minkowski Space On The Modeling Of Plasma Accelerators In A Lorentz Boosted Frame}}, +pages = {030701}, +title = {{Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}}, +url = {https://doi.org/10.1063/1.3559483}, volume = {18}, year = {2011} } - -@article{KirchenARXIV2016, -author = {Kirchen, M. and Lehe, R. and Godfrey, B.~B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A.~R.}, -journal = {arXiv:1608.00215}, +@article{KirchenPOP2016, +author = {Kirchen, M. and Lehe, R. and Godfrey, B. B. and Dornmair, I. and Jalas, S. and Peters, K. and Vay, J.-L. and Maier, A. R.}, +doi = {10.1063/1.4964770}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4964770/14024121/100704\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Oct}, +number = {10}, +pages = {100704}, title = {{Stable discrete representation of relativistically drifting plasmas}}, +url = {https://doi.org/10.1063/1.4964770}, +volume = {23}, year = {2016} } - -@article{LeheARXIV2016, -author = {Lehe, R. and Kirchen, M. and Godfrey, B.~B. and Maier, A.~R. and Vay, J.-L.}, -journal = {arXiv:1608.00227}, -title = {{Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates}}, +@article{LehePRE2016, +author = {Lehe, Remi and Kirchen, Manuel and Godfrey, Brendan B. and Maier, Andreas R. and Vay, Jean-Luc}, +doi = {10.1103/PhysRevE.94.053305}, +issue = {5}, +journal = {Phys. Rev. E}, +month = {Nov}, +numpages = {16}, +pages = {053305}, +publisher = {American Physical Society}, +title = {{Elimination of numerical Cherenkov instability in flowing-plasma particle-in-cell simulations by using Galilean coordinates}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.94.053305}, +volume = {94}, year = {2016} } - @book{godfrey1985iprop, - title={The IPROP Three-Dimensional Beam Propagation Code}, - author={Godfrey, B.B.}, - url={https://books.google.com/books?id=hos\_OAAACAAJ}, - year={1985}, - publisher={Defense Technical Information Center} +author = {Godfrey, B. B.}, +publisher = {Defense Technical Information Center}, +title = {{The IPROP Three-Dimensional Beam Propagation Code}}, +year = {1985} } diff --git a/Docs/source/latex_theory/input_output/input_output.tex b/Docs/source/latex_theory/input_output/input_output.tex index e013e236445..446f3f2b5d6 100644 --- a/Docs/source/latex_theory/input_output/input_output.tex +++ b/Docs/source/latex_theory/input_output/input_output.tex @@ -30,7 +30,7 @@ \subsection{Inputs and outputs in a boosted frame simulation} \subsubsection{Input in a boosted frame simulation} \paragraph{Particles - } -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse ``electron cloud'' instability \cite{Vayprl07}. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity $\{x,y,z,v_x,v_y,v_z\}$ for each beam macroparticle at time $t=0$ for a beam moving at the average velocity $v_b=\beta_b c$ (where $c$ is the speed of light) in the laboratory, and using the standard synchronization ($z=z'=0$ at $t=t'=0$) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity $\beta c$ in the laboratory is as follows (the superscript $'$ relates to quantities known in the boosted frame while the superscript $^*$ relates to quantities that are know at a given longitudinal position $z^*$ but different times of arrival): \begin{enumerate} \item project positions at $z^*=0$ assuming ballistic propagation diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index fdbbe57cf72..895a6c1392b 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -1,498 +1,393 @@ -@article{Tsung2006, -author = {Tsung,F. S. and Lu,W. and Tzoufras,M. and Mori,W. B. and Joshi,C. and Vieira,J. M. and Silva,L. O. and Fonseca,R. A. }, -title = {Simulation of monoenergetic electron generation via laser wakefield accelerators for 5–25TW lasers}, -journal = {Physics of Plasmas}, -volume = {13}, -number = {5}, -pages = {056708}, -year = {2006}, -doi = {10.1063/1.2198535} +@article{TajimaDawson1982, +abstract = "{Parallel intense laser beam ω0, k0 and ω1, k1 shone on a plasma with frequency separation equal to the plasma frequency ωp is capable of creating a coherent large electrostatic field and accelerating particles to high energies in large flux. The photon beam excites through the forward Raman scattering large amplitude plasmons whose phase velocity is equal to (ω−ω1)/(k0−k1), close to c in an underdense plasma. The plasmon traps electrons with electrostatic field EL=γ1/2⊥ mcωp/c, of the order of a few GeV/cm for plasma density to 1018 cm−3. Because of the phase velocity of the field close to c this field carries trapped electrons to high energies: W=2mc2(ω0/ωp)2. Preaccelerated particles (ions, for examples) coherent with the plasmon fields can also be accelerated. The (multiple) forward Raman instability saturates only when a sizable electron population is trapped and most of the electromagnetic energy is cascaded down to the frequency close to the cut‐off (ωp).}", +author = {Tajima, T. and Dawson, J. M.}, +doi = {10.1063/1.33805}, +issn = {0094-243X}, +journal = {AIP Conference Proceedings}, +month = {Sep}, +number = {1}, +pages = {69--93}, +title = {{Laser accelerator by plasma waves}}, +url = {https://doi.org/10.1063/1.33805}, +volume = {91}, +year = {1982} } -@article{Geddes2008, -doi = {10.1088/1742-6596/125/1/012002}, -year = {2008}, -volume = {125}, -number = {1}, -pages = {012002}, -author = {C G R Geddes and D L Bruhwiler and J R Cary and W B Mori and J-L Vay and S F Martins and T Katsouleas and E Cormier-Michel and W M Fawley and C Huang and X Wang and B Cowan and V K Decyk and E Esarey and R A Fonseca and W Lu and P Messmer and P Mullowney and K Nakamura and K Paul and G R Plateau and C B Schroeder and L O Silva and C Toth and F S Tsung and M Tzoufras and T Antonsen and J Vieira and W P Leemans}, -title = {Computational studies and optimization of wakefield accelerators}, -journal = {Journal of Physics: Conference Series} +@article{Esarey1996, +author = {Esarey, E. and Sprangle, P. and Krall, J. and Ting, A.}, +doi = {10.1109/27.509991}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {252--288}, +title = {{Overview of plasma-based accelerator concepts}}, +volume = {24}, +year = {1996} } -@techreport{Geddes2009, -title={Laser plasma particle accelerators: Large fields for smaller facility sources}, -author={Geddes, Cameron GR and Cormier-Michel, Estelle and Esarey, Eric H and Schroeder, Carl B and Vay, Jean-Luc and Leemans, Wim P and Bruhwiler, David L and Cary, John R and Cowan, Ben and Durant, Marc and others}, -year={2009}, -institution={Lawrence Berkeley National Laboratory (LBNL), Berkeley, CA (United States)} +@ARTICLE{Birdsall1991, +author = {Birdsall, C. K.}, +doi = {10.1109/27.106800}, +journal = {IEEE Transactions on Plasma Science}, +number = {2}, +pages = {65--85}, +title = {{Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}}, +volume = {19}, +year = {1991} } -@article{Geddes2010, -title={Scaled simulation design of high quality laser wakefield accelerator stages}, -author={Geddes, CGR}, -journal={}, -year={2010} +@misc{Lim2007, +author = {Lim, Chul-Hyun}, +issn = {0419-4217}, +language = {eng}, +number = {3}, +title = {{The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}}, +url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087}, +volume = {69}, +year = {2007} } -@article{Huang2009, -doi = {10.1088/1742-6596/180/1/012005}, -year = {2009}, -volume = {180}, +@article{Turner2013, +author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, +doi = {10.1063/1.4775084}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Jan}, +note = {013507}, number = {1}, -pages = {012005}, -author = {C Huang and W An and V K Decyk and W Lu and W B Mori and F S Tsung and M Tzoufras and S Morshed and T Antonsen and B Feng and T Katsouleas and R A Fonseca and S F Martins and J Vieira and L O Silva and E Esarey and C G R Geddes and W P Leemans and E Cormier-Michel and J-L Vay and D L Bruhwiler and B Cowan and J R Cary and K Paul}, -title = {Recent results and future challenges for large scale particle-in-cell simulations of plasma-based accelerator concepts}, -journal = {Journal of Physics: Conference Series} +title = {{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}}, +url = {https://doi.org/10.1063/1.4775084}, +volume = {20}, +year = {2013} } -@article{Leemans2014, -title = {Multi-GeV Electron Beams from Capillary-Discharge-Guided Subpetawatt Laser Pulses in the Self-Trapping Regime}, -author = {Leemans, W. P. and Gonsalves, A. J. and Mao, H.-S. and Nakamura, K. and Benedetti, C. and Schroeder, C. B. and T\'oth, Cs. and Daniels, J. and Mittelberger, D. E. and Bulanov, S. S. and Vay, J.-L. and Geddes, C. G. R. and Esarey, E.}, -journal = {Phys. Rev. Lett.}, -volume = {113}, -issue = {24}, -pages = {245002}, -numpages = {5}, -year = {2014}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.113.245002} +@incollection{Nielson1976, +author = {Clair W. Nielson and H. Ralph Lewis}, +booktitle = {Controlled Fusion}, +doi = {10.1016/B978-0-12-460816-0.50015-4}, +editor = {John Killeen}, +issn = {0076-6860}, +pages = {367--388}, +publisher = {Elsevier}, +series = {Methods in Computational Physics: Advances in Research and Applications}, +title = {{Particle-Code Models in the Nonradiative Limit}}, +volume = {16}, +year = {1976} } -@article{Blumenfeld2007, -title={Energy doubling of 42 GeV electrons in a metre-scale plasma wakefield accelerator}, -author={Blumenfeld, Ian and Clayton, Christopher E and Decker, Franz-Josef and Hogan, Mark J and Huang, Chengkun and Ischebeck, Rasmus and Iverson, Richard and Joshi, Chandrashekhar and Katsouleas, Thomas and Kirby, Neil and others}, -journal={Nature}, -volume={445}, -number={7129}, -pages={741--744}, -year={2007}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature05538} +@article{MUNOZ2018, +author = {P. A. Muñoz and N. Jain and P. Kilian and J. Büchner}, +doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, +issn = {0010-4655}, +journal = {Computer Physics Communications}, +keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method}, +pages = {245-264}, +title = {{A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}}, +url = {https://www.sciencedirect.com/science/article/pii/S0010465517303521}, +volume = {224}, +year = {2018} } -@article{Bulanov2014, -doi = {10.3367/UFNe.0184.201412a.1265}, -year = {2014}, -publisher = {Turpion Ltd and the Russian Academy of Sciences}, -volume = {57}, -number = {12}, -pages = {1149}, -author = {S V Bulanov and J J Wilkens and T Zh Esirkepov and G Korn and G Kraft and S D Kraft and M Molls and V S Khoroshkov}, -title = {Laser ion acceleration for hadron therapy}, -journal = {Physics-Uspekhi} +@article{Le2016, +abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", +author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, +doi = {10.1063/1.4943893}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Mar}, +note = {032114}, +number = {3}, +title = {{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}}, +url = {https://doi.org/10.1063/1.4943893}, +volume = {23}, +year = {2016} } -@article{Steinke2016, -title={Multistage coupling of independent laser-plasma accelerators}, -author={Steinke, S and Van Tilborg, J and Benedetti, C and Geddes, CGR and Schroeder, CB and Daniels, J and Swanson, KK and Gonsalves, AJ and Nakamura, K and Matlis, NH and others}, -journal={Nature}, -volume={530}, -number={7589}, -pages={190--193}, -year={2016}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nature16525} +@article{Stanier2020, +author = {A. Stanier and L. Chacón and A. Le}, +doi = {https://doi.org/10.1016/j.jcp.2020.109705}, +issn = {0021-9991}, +journal = {Journal of Computational Physics}, +keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, +pages = {109705}, +title = {{A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}}, +url = {https://www.sciencedirect.com/science/article/pii/S0021999120304794}, +volume = {420}, +year = {2020} } -@article{Sprangle1990, -title = {Nonlinear theory of intense laser-plasma interactions}, -author = {Sprangle, P. and Esarey, E. and Ting, A.}, -journal = {Phys. Rev. Lett.}, -volume = {64}, -issue = {17}, -pages = {2011--2014}, -year = {1990}, +@book{Stix1992, +author = {Stix, T. H.}, +bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +date-added = {2023-06-29 13:51:16 -0700}, +date-modified = {2023-06-29 13:51:16 -0700}, +isbn = {978-0-88318-859-0}, +lccn = {lc91033341}, +publisher = {American Inst. of Physics}, +title = {{Waves in Plasmas}}, +url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, +year = {1992} +} + +@article{Macchi2013, +author = {Macchi, Andrea and Borghesi, Marco and Passoni, Matteo}, +doi = {10.1103/RevModPhys.85.751}, +issue = {2}, +journal = {Rev. Mod. Phys.}, +month = {May}, +numpages = {0}, +pages = {751--793}, publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.64.2011} +title = {{Ion acceleration by superintense laser-plasma interaction}}, +url = {https://link.aps.org/doi/10.1103/RevModPhys.85.751}, +volume = {85}, +year = {2013} } -@article{Antonsen1992, -title = {Self-focusing and Raman scattering of laser pulses in tenuous plasmas}, -author = {Antonsen, T. M. and Mora, P.}, -journal = {Phys. Rev. Lett.}, -volume = {69}, -issue = {15}, -pages = {2204--2207}, -year = {1992}, -publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.69.2204} +@article{Wilks2001, +abstract = "{An explanation for the energetic ions observed in the PetaWatt experiments is presented. In solid target experiments with focused intensities exceeding 1020 W/cm2, high-energy electron generation, hard bremsstrahlung, and energetic protons have been observed on the backside of the target. In this report, an attempt is made to explain the physical process present that will explain the presence of these energetic protons, as well as explain the number, energy, and angular spread of the protons observed in experiment. In particular, we hypothesize that hot electrons produced on the front of the target are sent through to the back off the target, where they ionize the hydrogen layer there. These ions are then accelerated by the hot electron cloud, to tens of MeV energies in distances of order tens of μm, whereupon they end up being detected in the radiographic and spectrographic detectors.}", +author = {Wilks, S. C. and Langdon, A. B. and Cowan, T. E. and Roth, M. and Singh, M. and Hatchett, S. and Key, M. H. and Pennington, D. and MacKinnon, A. and Snavely, R. A.}, +doi = {10.1063/1.1333697}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/8/2/542/12669088/542\_1\_online.pdf}, +issn = {1070-664X}, +journal = {Physics of Plasmas}, +month = {Feb}, +number = {2}, +pages = {542-549}, +title = {{Energetic proton generation in ultra-intense laser–solid interactions}}, +url = {https://doi.org/10.1063/1.1333697}, +volume = {8}, +year = {2001} } -@article{Krall1993, -title = {Enhanced acceleration in a self-modulated-laser wake-field accelerator}, -author = {Krall, J. and Ting, A. and Esarey, E. and Sprangle, P.}, +@article{Bulanov2008, +author = {Bulanov, S. S. and Brantov, A. and Bychenkov, V. Yu. and Chvykov, V. and Kalinchenko, G. and Matsuoka, T. and Rousseau, P. and Reed, S. and Yanovsky, V. and Litzenberg, D. W. and Krushelnick, K. and Maksimchuk, A.}, +doi = {10.1103/PhysRevE.78.026412}, +issue = {2}, journal = {Phys. Rev. E}, -volume = {48}, -issue = {3}, -pages = {2157--2161}, -year = {1993}, +month = {Aug}, +numpages = {6}, +pages = {026412}, publisher = {American Physical Society}, -doi = {10.1103/PhysRevE.48.2157} -} - -@article{Mora1997, -author = {Mora,Patrick and Antonsen, Jr.,Thomas M. }, -title = {Kinetic modeling of intense, short laser pulses propagating in tenuous plasmas}, +title = {{Accelerating monoenergetic protons from ultrathin foils by flat-top laser pulses in the directed-Coulomb-explosion regime}}, +url = {https://link.aps.org/doi/10.1103/PhysRevE.78.026412}, +volume = {78}, +year = {2008} +} + +@article{Dromey2004, +abstract = "{Plasma mirrors are devices capable of switching very high laser powers on subpicosecond time scales with a dynamic range of 20–30 dB. A detailed study of their performance in the near-field of the laser beam is presented, a setup relevant to improving the pulse contrast of modern ultrahigh power lasers (TW–PW). The conditions under which high reflectivity can be achieved and focusability of the reflected beam retained are identified. At higher intensities a region of high specular reflectivity with rapidly decreasing focusability was observed, suggesting that specular reflectivity alone is not an adequate guide to the ideal range of plasma mirror operation. It was found that to achieve high reflectivity with negligible phasefront distortion of the reflected beam the inequality csΔt\\<λLaser must be met (cs: sound speed, Δt: time from plasma formation to the peak of the pulse). The achievable contrast enhancement is given by the ratio of plasma mirror reflectivity to cold reflectivity.}", +author = {Dromey, B. and Kar, S. and Zepf, M. and Foster, P.}, +doi = {10.1063/1.1646737}, +eprint = {https://pubs.aip.org/aip/rsi/article-pdf/75/3/645/8814694/645\_1\_online.pdf}, +issn = {0034-6748}, +journal = {Review of Scientific Instruments}, +month = {Feb}, +number = {3}, +pages = {645-649}, +title = {{The plasma mirror—A subpicosecond optical switch for ultrahigh power lasers}}, +url = {https://doi.org/10.1063/1.1646737}, +volume = {75}, +year = {2004} +} + +@article{Roedel2010, +author = {R\"{o}del,  C. and Heyer,  M. and Behmke,  M. and K\"{u}bel,  M. and J\"{a}ckel,  O. and Ziegler,  W. and Ehrt,  D. and Kaluza,  M. C. and Paulus,  G. G.}, +DOI = {10.1007/s00340-010-4329-7}, +ISSN = {1432-0649}, +journal = {Applied Physics B}, +month = {Nov}, +number = {2}, +pages = {295–302}, +publisher = {Springer Science and Business Media LLC}, +title = {{High repetition rate plasma mirror for temporal contrast enhancement of terawatt femtosecond laser pulses by three orders of magnitude}}, +url = {http://dx.doi.org/10.1007/s00340-010-4329-7}, +volume = {103}, +year = {2010} +} + +@misc{SandbergPASC24, +address = {Zuerich, Switzerland}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. of PASC24}, +note = {submitted}, +series = {PASC'24 - Platform for Advanced Scientific Computing}, +title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +venue = {Zuerich, Switzerland}, +year = {2024} +} + +@inproceedings{SandbergIPAC23, +address = {Venice, Italy}, +author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Ji Qiang and Jean-Luc Vay and Axel Huebl}, +booktitle = {Proc. 14th International Particle Accelerator Conference}, +doi = {10.18429/JACoW-IPAC2023-WEPA101}, +isbn = {978-3-95450-231-8}, +issn = {2673-5490}, +language = {English}, +month = {May}, +number = {14}, +pages = {2885-2888}, +paper = {WEPA101}, +publisher = {JACoW Publishing, Geneva, Switzerland}, +series = {IPAC'23 - 14th International Particle Accelerator Conference}, +title = {{Hybrid beamline element ML-training for surrogates in the ImpactX beam-dynamics code}}, +url = {https://indico.jacow.org/event/41/contributions/2276}, +venue = {Venice, Italy}, +year = {2023} +} + +@article{HigueraPOP2017, +author = {Higuera, A. V. and Cary, J. R.}, +doi = {10.1063/1.4979989}, +eprint = {https://pubs.aip.org/aip/pop/article-pdf/doi/10.1063/1.4979989/15988441/052104\_1\_online.pdf}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {4}, -number = {1}, -pages = {217-229}, -year = {1997}, -doi = {10.1063/1.872134} +month = {04}, +number = {5}, +pages = {052104}, +title = {{Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields}}, +url = {https://doi.org/10.1063/1.4979989}, +volume = {24}, +year = {2017} } -@article{Huang2006, -title = {QUICKPIC: A highly efficient particle-in-cell code for modeling wakefield acceleration in plasmas}, -journal = {Journal of Computational Physics}, -volume = {217}, -number = {2}, -pages = {658-679}, -year = {2006}, +@article{ShuJCP1988, +author = {Chi-Wang Shu and Stanley Osher}, +doi = {https://doi.org/10.1016/0021-9991(88)90177-5}, issn = {0021-9991}, -doi = {10.1016/j.jcp.2006.01.039}, -author = {C. Huang and V.K. Decyk and C. Ren and M. Zhou and W. Lu and W.B. Mori and J.H. Cooley and T.M. Antonsen and T. Katsouleas} -} - -@article{Benedetti2010, -author = {Benedetti,C. and Schroeder,C. B. and Esarey,E. and Geddes,C. G. R. and Leemans,W. P. }, -title = {Efficient Modeling of Laser‐Plasma Accelerators with INF\&RNO}, -journal = {AIP Conference Proceedings}, -volume = {1299}, -number = {1}, -pages = {250-255}, -year = {2010}, -doi = {10.1063/1.3520323} -} - -@article{Cowan2011, -title = {Characteristics of an envelope model for laser–plasma accelerator simulation}, journal = {Journal of Computational Physics}, -volume = {230}, -number = {1}, -pages = {61-86}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2010.09.009}, -author = {Benjamin M. Cowan and David L. Bruhwiler and Estelle Cormier-Michel and Eric Esarey and Cameron G.R. Geddes and Peter Messmer and Kevin M. Paul} -} - -@article{Vay2007, -title = {Noninvariance of Space- and Time-Scale Ranges under a Lorentz Transformation and the Implications for the Study of Relativistic Interactions}, -author = {Vay, J.-L.}, +number = {2}, +pages = {439-471}, +title = {Efficient implementation of essentially non-oscillatory shock-capturing schemes}, +url = {https://www.sciencedirect.com/science/article/pii/0021999188901775}, +volume = {77}, +year = {1988} +} + +@Inbook{VanLeerBookChapter1997, +author = {Van Leer, Bram}, +bookTitle = {Upwind and High-Resolution Schemes}, +doi = {10.1007/978-3-642-60543-7_3}, +editor = {Hussaini, M. Yousuff and {van Leer}, Bram and {Van Rosendale}, John}, +isbn = {978-3-642-60543-7}, +pages = {33--52}, +publisher = {Springer Berlin Heidelberg}, +title = {On The Relation Between The Upwind-Differencing Schemes Of Godunov, Engquist---Osher and Roe}, +url = {https://doi.org/10.1007/978-3-642-60543-7_3}, +year = {1997} +} + +@article{Yakimenko2019, +author = {Yakimenko, V. and Meuren, S. and Del Gaudio, F. and Baumann, C. and Fedotov, A. and Fiuza, F. and Grismayer, T. and Hogan, M. J. and Pukhov, A. and Silva, L. O. and White, G.}, +doi = {10.1103/PhysRevLett.122.190404}, +issue = {19}, journal = {Phys. Rev. Lett.}, -volume = {98}, -issue = {13}, -pages = {130405}, -year = {2007}, +month = {May}, +numpages = {7}, +pages = {190404}, publisher = {American Physical Society}, -doi = {10.1103/PhysRevLett.98.130405} +title = {Prospect of Studying Nonperturbative QED with Beam-Beam Collisions}, +volume = {122}, +year = {2019}, } -@article{Vay2009a, -doi = {10.1088/1742-6596/180/1/012006}, -year = {2009}, -volume = {180}, -number = {1}, -pages = {012006}, -author = {J-L Vay and D L Bruhwiler and C G R Geddes and W M Fawley and S F Martins and J R Cary and E Cormier-Michel and B Cowan and R A Fonseca and M A Furman and W Lu and W B Mori and L O Silva}, -title = {Simulating relativistic beam and plasma systems using an optimal boosted frame}, -journal = {Journal of Physics: Conference Series} -} - -@article{Vay2009b, -title = {Application of the reduction of scale range in a Lorentz boosted frame to the numerical simulation of particle acceleration devices.}, -author = {Vay, J and Fawley, W M and Geddes, C G and Cormier-Michel, E and Grote, D P}, -url = {https://www.osti.gov/biblio/952754}, -journal = {}, -place = {United States}, -year = {2009}, -} - -@article{Vay2010, -author = {Vay,J.‐L. and Geddes,C. G. R. and Benedetti,C. and Bruhwiler,D. L. and Cormier‐Michel,E. and Cowan,B. M. and Cary,J. R. and Grote,D. P. }, -title = {Modeling Laser Wakefield Accelerators in a Lorentz Boosted Frame}, -journal = {AIP Conference Proceedings}, -volume = {1299}, -number = {1}, -pages = {244-249}, -year = {2010}, -doi = {10.1063/1.3520322} -} - -@article{Vay2011a, -title = {Numerical methods for instability mitigation in the modeling of laser wakefield accelerators in a Lorentz-boosted frame}, -journal = {Journal of Computational Physics}, -volume = {230}, -number = {15}, -pages = {5908-5929}, -year = {2011}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2011.04.003}, -author = {J.-L. Vay and C.G.R. Geddes and E. Cormier-Michel and D.P. Grote} -} - -@article{Vay2011b, -author = {Vay,J.-L. and Geddes,C. G. R. and Cormier-Michel,E. and Grote,D. P. }, -title = {Effects of hyperbolic rotation in Minkowski space on the modeling of plasma accelerators in a Lorentz boosted frame}, -journal = {Physics of Plasmas}, -volume = {18}, -number = {3}, -pages = {030701}, -year = {2011}, -doi = {10.1063/1.3559483} -} - -@article{Vay2011c, -author = {Vay,J.-L. and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Leemans,W. P. and Cormier-Michel,E. and Grote,D. P. }, -title = {Modeling of 10 GeV-1 TeV laser-plasma accelerators using Lorentz boosted simulations}, +@article{Groenewald2023, +author = {Groenewald, R. E. and Veksler, A. and Ceccherini, F. and Necas, A. and Nicks, B. S. and Barnes, D. C. and Tajima, T. and Dettrick, S. A.}, +doi = {10.1063/5.0178288}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {18}, +month = {Dec}, number = {12}, -pages = {123103}, -year = {2011}, -doi = {10.1063/1.3663841} +pages = {122508}, +title = {{Accelerated kinetic model for global macro stability studies of high-beta fusion reactors}}, +volume = {30}, +year = {2023}, } -@article{Martins2010a, -title = {Numerical simulations of laser wakefield accelerators in optimal Lorentz frames}, -journal = {Computer Physics Communications}, -volume = {181}, -number = {5}, -pages = {869-875}, -year = {2010}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2009.12.023}, -author = {Samuel F. Martins and Ricardo A. Fonseca and Luís O. Silva and Wei Lu and Warren B. Mori} -} - -@article{Martins2010b, -title={Exploring laser-wakefield-accelerator regimes for near-term lasers using particle-in-cell simulation in Lorentz-boosted frames}, -author={Martins, Samuel F and Fonseca, RA and Lu, Wei and Mori, Warren B and Silva, LO}, -journal={Nature Physics}, -volume={6}, -number={4}, -pages={311--316}, -year={2010}, -publisher={Nature Publishing Group UK London}, -doi={10.1038/nphys1538} -} - -@article{Martins2010c, -author = {Martins,S. F. and Fonseca,R. A. and Vieira,J. and Silva,L. O. and Lu,W. and Mori,W. B. }, -title = {Modeling laser wakefield accelerator experiments with ultrafast particle-in-cell simulations in boosted frames}, +@article{PerezPOP2012, +author = {Pérez, F. and Gremillet, L. and Decoster, A. and Drouin, M. and Lefebvre, E.}, +doi = {10.1063/1.4742167}, +issn = {1070-664X}, journal = {Physics of Plasmas}, -volume = {17}, -number = {5}, -pages = {056705}, -year = {2010}, -doi = {10.1063/1.3358139} -} - -@article{Bruhwiler2009, -author = {Bruhwiler,David L. and Cary,John R. and Cowan,Benjamin M. and Paul,Kevin and Geddes,Cameron G. R. and Mullowney,Paul J. and Messmer,Peter and Esarey,Eric and Cormier‐Michel,Estelle and Leemans,Wim and Vay,Jean‐Luc }, -title = {New Developments in the Simulation of Advanced Accelerator Concepts}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {29-37}, -year = {2009}, -doi = {10.1063/1.3080922} +month = {Aug}, +number = {8}, +pages = {083104}, +title = {{Improved modeling of relativistic collisions and collisional ionization in particle-in-cell codes}}, +volume = {19}, +year = {2012} } -@article{Yu2016, -title = {Enabling Lorentz boosted frame particle-in-cell simulations of laser wakefield acceleration in quasi-3D geometry}, -journal = {Journal of Computational Physics}, -volume = {316}, -pages = {747-759}, -year = {2016}, +@article{HigginsonJCP2019, +author = {Drew Pitney Higginson and Anthony Link and Andrea Schmidt}, +doi = {10.1016/j.jcp.2019.03.020}, issn = {0021-9991}, -doi = {10.1016/j.jcp.2016.04.014}, -author = {Peicheng Yu and Xinlu Xu and Asher Davidson and Adam Tableman and Thamine Dalichaouch and Fei Li and Michael D. Meyers and Weiming An and Frank S. Tsung and Viktor K. Decyk and Frederico Fiuza and Jorge Vieira and Ricardo A. Fonseca and Wei Lu and Luis O. Silva and Warren B. Mori} -} - -@inproceedings{Godfrey1985, -title={The IPROP Three-Dimensional Beam Propagation Code}, -booktitle={}, -author={B. B. Godfrey}, -year={1985} -} - -@article{Lifschitz2009, -title = {Particle-in-Cell modelling of laser–plasma interaction using Fourier decomposition}, journal = {Journal of Computational Physics}, -volume = {228}, -number = {5}, -pages = {1803-1814}, -year = {2009}, -issn = {0021-9991}, -doi = {10.1016/j.jcp.2008.11.017}, -author = {A.F. Lifschitz and X. Davoine and E. Lefebvre and J. Faure and C. Rechatin and V. Malka} +keywords = {Particle-in-cell, Nuclear Fusion, Monte Carlo methods, Inertial confinement fusion, Thermonuclear Fusion}, +month = {Jul}, +pages = {439--453}, +title = {{A pairwise nuclear fusion algorithm for weighted particle-in-cell plasma simulations}}, +volume = {388}, +year = {2019} } -@article{Davidson2015, -title = {Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in ϕ into OSIRIS}, -journal = {Journal of Computational Physics}, -volume = {281}, -pages = {1063-1077}, -year = {2015}, +@article{VerboncoeurJCP2001, +author = {J.P. Verboncoeur}, +doi = {10.1006/jcph.2001.6923}, issn = {0021-9991}, -doi = {10.1016/j.jcp.2014.10.064}, -author = {A. Davidson and A. Tableman and W. An and F.S. Tsung and W. Lu and J. Vieira and R.A. Fonseca and L.O. Silva and W.B. Mori} -} - -@article{Lehe2016, -title = {A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm}, -journal = {Computer Physics Communications}, -volume = {203}, -pages = {66-82}, -year = {2016}, -issn = {0010-4655}, -doi = {10.1016/j.cpc.2016.02.007}, -author = {Rémi Lehe and Manuel Kirchen and Igor A. Andriyash and Brendan B. Godfrey and Jean-Luc Vay} -} - -@article{Andriyash2016, -author = {Andriyash,Igor A. and Lehe,Remi and Lifschitz,Agustin }, -title = {Laser-plasma interactions with a Fourier-Bessel particle-in-cell method}, -journal = {Physics of Plasmas}, -volume = {23}, -number = {3}, -pages = {033110}, -year = {2016}, -doi = {10.1063/1.4943281} -} - -@article{Shadwick2009, -author = {Shadwick,B. A. and Schroeder,C. B. and Esarey,E. }, -title = {Nonlinear laser energy depletion in laser-plasma accelerators}, -journal = {Physics of Plasmas}, -volume = {16}, -number = {5}, -pages = {056704}, -year = {2009}, -doi = {10.1063/1.3124185} -} - -@article{CormierMichel2009, -author = {Cormier‐Michel,Estelle and Geddes,C. G. R. and Esarey,E. and Schroeder,C. B. and Bruhwiler,D. L. and Paul,K. and Cowan,B. and Leemans,W. P. }, -title = {Scaled simulations of a 10 GeV accelerator}, -journal = {AIP Conference Proceedings}, -volume = {1086}, -number = {1}, -pages = {297-302}, -year = {2009}, -doi = {10.1063/1.3080921} -} - -@ARTICLE{Birdsall1991, - author = {Birdsall, C.K.}, - journal = {IEEE Transactions on Plasma Science}, - title = {Particle-in-cell charged-particle simulations, plus Monte Carlo collisions with neutral atoms, PIC-MCC}, - year = {1991}, - volume = {19}, - number = {2}, - pages = {65-85}, - doi = {10.1109/27.106800} -} - -@misc{Lim2007, -author = {Lim, Chul-Hyun}, -issn = {0419-4217}, -language = {eng}, -number = {3}, -title = {The interaction of energetic charged particles with gas and boundaries in the particle simulation of plasmas}, -volume = {69}, -year = {2007}, -url = {https://search.library.berkeley.edu/permalink/01UCS_BER/s4lks2/cdi_proquest_miscellaneous_35689087} -} - -@article{Turner2013, -author = {Turner, M. M. and Derzsi, A. and Donkó, Z. and Eremin, D. and Kelly, S. J. and Lafleur, T. and Mussenbrock, T.}, -title = "{Simulation benchmarks for low-pressure plasmas: Capacitive discharges}", -journal = {Physics of Plasmas}, -volume = {20}, +journal = {Journal of Computational Physics}, number = {1}, -year = {2013}, -month = {01}, -issn = {1070-664X}, -doi = {10.1063/1.4775084}, -url = {https://doi.org/10.1063/1.4775084}, -note = {013507} +pages = {421-427}, +title = {{Symmetric Spline Weighting for Charge and Current Density in Particle Simulation}}, +volume = {174}, +year = {2001} } -@misc{winske2022hybrid, -title={Hybrid codes (massless electron fluid)}, -author={D. Winske and Homa Karimabadi and Ari Le and N. Omidi and Vadim Roytershteyn and Adam Stanier}, -year={2022}, -eprint={2204.01676}, -archivePrefix={arXiv}, -primaryClass={physics.plasm-ph} -} - -@incollection{NIELSON1976, -title = {Particle-Code Models in the Nonradiative Limit}, -editor = {JOHN KILLEEN}, -series = {Methods in Computational Physics: Advances in Research and Applications}, -publisher = {Elsevier}, -volume = {16}, -pages = {367-388}, -year = {1976}, -booktitle = {Controlled Fusion}, -issn = {0076-6860}, -doi = {https://doi.org/10.1016/B978-0-12-460816-0.50015-4}, -author = {CLAIR W. NIELSON and H. RALPH LEWIS} -} - -@article{MUNOZ2018, -title = {A new hybrid code (CHIEF) implementing the inertial electron fluid equation without approximation}, -journal = {Computer Physics Communications}, -volume = {224}, -pages = {245-264}, -year = {2018}, +@article{MuravievCPC2021, +author = {A. Muraviev and A. Bashinov and E. Efimenko and V. Volokitin and I. Meyerov and A. Gonoskov}, +doi = {10.1016/j.cpc.2021.107826}, issn = {0010-4655}, -doi = {https://doi.org/10.1016/j.cpc.2017.10.012}, -url = {https://www.sciencedirect.com/science/article/pii/S0010465517303521}, -author = {P.A. Muñoz and N. Jain and P. Kilian and J. Büchner}, -keywords = {Plasma simulation, Hybrid methods, Particle-in-cell method} -} - -@article{Le2016, -author = {Le, A. and Daughton, W. and Karimabadi, H. and Egedal, J.}, -title = "{Hybrid simulations of magnetic reconnection with kinetic ions and fluid electron pressure anisotropy}", -journal = {Physics of Plasmas}, +journal = {Computer Physics Communications}, +keywords = {Particle-in-cell, Resampling, Merging, Thinning, QED cascades}, +pages = {107826}, +title = {{Strategies for particle resampling in PIC simulations}}, +volume = {262}, +year = {2021} +} + +@article{AkturkOE2004, +author = {Selcuk Akturk and Xun Gu and Erik Zeek and Rick Trebino}, +doi = {10.1364/OPEX.12.004399}, +journal = {Opt. Express}, +keywords = {Pulses; Ultrafast measurements; CCD cameras; Dispersion; Distortion; Fourier transforms; Gaussian beams; Ultrashort pulses}, +month = {Sep}, +number = {19}, +pages = {4399--4410}, +publisher = {Optica Publishing Group}, +title = {{Pulse-front tilt caused by spatial and temporal chirp}}, +volume = {12}, +year = {2004} +} + +@inproceedings{XiaoIEEE2005, +author = {Tian Xiao and Qing Huo Liu}, +booktitle = {2005 IEEE Antennas and Propagation Society International Symposium}, +doi = {10.1109/APS.2005.1551259}, +number = {}, +pages = {122-125 Vol. 1A}, +title = {{An enlarged cell technique for the conformal FDTD method to model perfectly conducting objects}}, +volume = {1A}, +year = {2005} +} + +@article{GrismayerNJP2021, +author = {T Grismayer and R Torres and P Carneiro and F Cruz and R A Fonseca and L O Silva}, +doi = {10.1088/1367-2630/ac2004}, +journal = {New Journal of Physics}, +month = {Sep}, +number = {9}, +pages = {095005}, +publisher = {IOP Publishing}, +title = {{Quantum Electrodynamics vacuum polarization solver}}, volume = {23}, -number = {3}, -year = {2016}, -month = {03}, -abstract = "{We present the first hybrid simulations with kinetic ions and recently developed equations of state for the electron fluid appropriate for reconnection with a guide field. The equations of state account for the main anisotropy of the electron pressure tensor. Magnetic reconnection is studied in two systems, an initially force-free current sheet and a Harris sheet. The hybrid model with the equations of state is compared to two other models, hybrid simulations with isothermal electrons and fully kinetic simulations. Including the anisotropic equations of state in the hybrid model provides a better match to the fully kinetic model. In agreement with fully kinetic results, the main feature captured is the formation of an electron current sheet that extends several ion inertial lengths. This electron current sheet modifies the Hall magnetic field structure near the X-line, and it is not observed in the standard hybrid model with isotropic electrons. The saturated reconnection rate in this regime nevertheless remains similar in all three models. Implications for global modeling are discussed.}", -issn = {1070-664X}, -doi = {10.1063/1.4943893}, -url = {https://doi.org/10.1063/1.4943893}, -note = {032114} -} - -@article{Stanier2020, -title = {A cancellation problem in hybrid particle-in-cell schemes due to finite particle size}, -journal = {Journal of Computational Physics}, -volume = {420}, -pages = {109705}, -year = {2020}, -issn = {0021-9991}, -doi = {https://doi.org/10.1016/j.jcp.2020.109705}, -url = {https://www.sciencedirect.com/science/article/pii/S0021999120304794}, -author = {A. Stanier and L. Chacón and A. Le}, -keywords = {Hybrid, Particle-in-cell, Plasma, Asymptotic-preserving, Cancellation problem, Space weather}, -} - -@book{Stix1992, - author = {Stix, T.H.}, - date-added = {2023-06-29 13:51:16 -0700}, - date-modified = {2023-06-29 13:51:16 -0700}, - isbn = {978-0-88318-859-0}, - lccn = {lc91033341}, - publisher = {American Inst. of Physics}, - title = {Waves in {Plasmas}}, - url = {https://books.google.com/books?id=OsOWJ8iHpmMC}, - year = {1992}, - bdsk-url-1 = {https://books.google.com/books?id=OsOWJ8iHpmMC} +year = {2021} } diff --git a/Docs/source/theory/PML.rst b/Docs/source/theory/PML.rst deleted file mode 100644 index d60be931c5f..00000000000 --- a/Docs/source/theory/PML.rst +++ /dev/null @@ -1,242 +0,0 @@ -.. _theory-bc: - -Boundary conditions -=================== - -Open boundary condition for electromagnetic waves -------------------------------------------------- - -For the TE case, the original Berenger’s Perfectly Matched Layer (PML) writes - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{\partial H_{z}}{\partial y}\label{PML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{\partial H_{z}}{\partial x}\label{PML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{\partial E_{y}}{\partial x}\label{PML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{\partial E_{x}}{\partial y}\label{PML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{PML_def_5}\end{aligned} - -This can be generalized to - -.. math:: - - \begin{aligned} - \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = & \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z}\label{APML_def_1} \\ - \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = & -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z}\label{APML_def_2} \\ - \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = & -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y}\label{APML_def_3} \\ - \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = & \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x}\label{APML_def_4} \\ - H_{z} = & H_{zx}+H_{zy}\label{APML_def_5}\end{aligned} - -For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, -this system reduces to the Berenger PML medium, while adding the additional -constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` -leads to the system of Maxwell equations in vacuum. - -.. _theory-bc-propa-plane-wave: - -Propagation of a Plane Wave in an APML Medium -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) -and pulsation :math:`\omega` propagating in the APML medium with an -angle :math:`\varphi` relative to the x axis - -.. math:: - - \begin{aligned} - E_{x} = & -E_{0}\sin \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_1} \\ - E_{y} = & E_{0}\cos \varphi e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_2} \\ - H_{zx} = & H_{zx0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_AMPL_def_3} \\ - H_{zy} = & H_{zy0}e^{i\omega \left( t-\alpha x-\beta y\right) }\label{Plane_wave_APML_def_4}\end{aligned} - -where :math:`\alpha` and\ :math:`\beta` are two complex constants to -be determined. - -Introducing (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_def_2] <#Plane_wave_APML_def_2>`__), -(`[Plane_wave_AMPL_def_3] <#Plane_wave_AMPL_def_3>`__) and (`[Plane_wave_APML_def_4] <#Plane_wave_APML_def_4>`__) -into (`[APML_def_1] <#APML_def_1>`__), (`[APML_def_2] <#APML_def_2>`__), (`[APML_def_3] <#APML_def_3>`__) -and (`[APML_def_4] <#APML_def_4>`__) gives - -.. math:: - - \begin{aligned} - \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = & \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_1} \\ - \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = & \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) \label{Plane_wave_APML_1_2} \\ - \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = & \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi \label{Plane_wave_APML_1_3} \\ - \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = & \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi \label{Plane_wave_APML_1_4}\end{aligned} - -Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using (`[Plane_wave_APML_1_1] <#Plane_wave_APML_1_1>`__) -and (`[Plane_wave_APML_1_2] <#Plane_wave_APML_1_2>`__), we get - -.. math:: - - \begin{aligned} - \beta = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}}\label{Plane_wave_APML_beta_of_g} \\ - \alpha = & \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}}\label{Plane_wave_APML_alpha_of_g}\end{aligned} - -Adding :math:`H_{zx0}` and :math:`H_{zy0}` from (`[Plane_wave_APML_1_3] <#Plane_wave_APML_1_3>`__) -and (`[Plane_wave_APML_1_4] <#Plane_wave_APML_1_4>`__) and substituting the expressions -for :math:`\alpha` and :math:`\beta` from (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__) -and (`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) yields - -.. math:: - - \begin{aligned} - \frac{1}{Z} = & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber \\ - + & \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi\end{aligned} - -If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then - -.. math:: - - \begin{aligned} - Z = & \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}}\label{APML_impedance}\end{aligned} - -which is the impedance of vacuum. Hence, like the PML, given some -restrictions on the parameters, the APML does not generate any reflection -at any angle and any frequency. As for the PML, this property is not -retained after discretization, as shown subsequently in this paper. - -Calling :math:`\psi` any component of the field and :math:`\psi _{0}` -its magnitude, we get from (`[Plane_wave_APML_def_1] <#Plane_wave_APML_def_1>`__), (`[Plane_wave_APML_beta_of_g] <#Plane_wave_APML_beta_of_g>`__), -(`[Plane_wave_APML_alpha_of_g] <#Plane_wave_APML_alpha_of_g>`__) and (`[APML_impedance] <#APML_impedance>`__) that - -.. math:: - - \label{Plane_wave_absorption} - \psi =\psi _{0}e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x}e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y} - -We assume that we have an APML layer of thickness :math:`\delta` (measured -along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` -and :math:`c_{y}=c.` Using (`[Plane_wave_absorption] <#Plane_wave_absorption>`__), we determine -that the coefficient of reflection given by this layer is - -.. math:: - - \begin{aligned} - R_{APML}\left( \theta \right) = & e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta }\nonumber \\ - = & e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }\end{aligned} - -which happens to be the same as the PML theoretical coefficient of -reflection if we assume :math:`c_{x}=c`. Hence, it follows that for -the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` -seems to be of no interest. However, although this conclusion is true -at the infinitesimal limit, it does not hold for the discretized counterpart. - -Discretization -~~~~~~~~~~~~~~ - -.. math:: - - \begin{aligned} - \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = & \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} \\ - % - \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = & - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n}_{j+1/2,k+1/2,l}}{2} = & - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} \\ - % - \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n}_{j+1/2,k+1/2,l}}{2} = & \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - c_x = & c e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta x}{1-e^{-\sigma_x\Delta t}} \\ - c_y = & c e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta y}{1-e^{-\sigma_y\Delta t}} \\ - c^*_x = & c e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta x}{1-e^{-\sigma^*_x\Delta t}} \\ - c^*_y = & c e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta y}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. math:: - - \begin{aligned} - E_x|^{n+1}_{j+1/2,k,l} = & E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \\ - % - E_y|^{n+1}_{j,k+1/2,l} = & E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \\ - % - H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zx}|^{n}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \\ - % - H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} = & H_{zy}|^{n}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \\ - % - H_z = & H_{zx}+H_{zy}\end{aligned} - -.. _theory-bc-pec: - -Perfect Electrical Conductor ----------------------------- - -This boundary can be used to model a dielectric or metallic surface. -For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic -field are set to 0. In the guard-cell region, the tangential electric field is set equal and -opposite to the respective field component in the mirror location across the PEC -boundary, and the normal electric field is set equal to the field component in the -mirror location in the domain across the PEC boundary. Similarly, the tangential -(and normal) magnetic field components are set equal (and opposite) to the respective -magnetic field components in the mirror locations across the PEC boundary. - -The PEC boundary condition also impacts the deposition of charge and current density. -On the boundary the charge density and parallel current density is set to zero. If -a reflecting boundary condition is used for the particles, density overlapping -with the PEC will be reflected back into the domain (for both charge and current -density). If absorbing boundaries are used, an image charge (equal weight but -opposite charge) is considered in the mirror location accross the boundary, and -the density from that charge is also deposited in the simulation domain. The -figure below shows the effect of this. The left boundary is absorbing while -the right boundary is reflecting. - -.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png - :alt: PEC boundary deposition - :width: 80% diff --git a/Docs/source/theory/amr.rst b/Docs/source/theory/amr.rst index 730593beea4..d83b1d7db9d 100644 --- a/Docs/source/theory/amr.rst +++ b/Docs/source/theory/amr.rst @@ -3,158 +3,64 @@ Mesh refinement =============== -.. raw:: latex - - \centering +.. _fig_ESAMR: .. figure:: ICNSP_2011_Vay_fig1.png :alt: Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. - :name: fig:ESAMR - :width: 15cm + :width: 95% Sketches of the implementation of mesh refinement in WarpX with the electrostatic (left) and electromagnetic (right) solvers. In both cases, the charge/current from particles are deposited at the finest levels first, then interpolated recursively to coarser levels. In the electrostatic case, the potential is calculated first at the coarsest level :math:`L_0`, the solution interpolated to the boundaries of the refined patch :math:`r` at the next level :math:`L_{1}` and the potential calculated at :math:`L_1`. The procedure is repeated iteratively up to the highest level. In the electromagnetic case, the fields are computed independently on each grid and patch without interpolation at boundaries. Patches are terminated by absorbing layers (PML) to prevent the reflection of electromagnetic waves. Additional coarse patch :math:`c` and fine grid :math:`a` are needed so that the full solution is obtained by substitution on :math:`a` as :math:`F_{n+1}(a)=F_{n+1}(r)+I[F_n( s )-F_{n+1}( c )]` where :math:`F` is the field, and :math:`I` is a coarse-to-fine interpolation operator. In both cases, the field solution at a given level :math:`L_n` is unaffected by the solution at higher levels :math:`L_{n+1}` and up, allowing for mitigation of some spurious effects (see text) by providing a transition zone via extension of the patches by a few cells beyond the desired refined area (red & orange rectangles) in which the field is interpolated onto particles from the coarser parent level only. -The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface (J. Vay et al. 2002; Colella and Norgaard 2010); b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface (Vay 2001). The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. +The mesh refinement methods that have been implemented in WarpX were developed following the following principles: i) avoidance of spurious effects from mesh refinement, or minimization of such effects; ii) user controllability of the spurious effects’ relative magnitude; iii) simplicity of implementation. The two main generic issues that were identified are: a) spurious self-force on macroparticles close to the mesh refinement interface :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`; b) reflection (and possible amplification) of short wavelength electromagnetic waves at the mesh refinement interface :cite:p:`amr-Vayjcp01`. The two effects are due to the loss of translation invariance introduced by the asymmetry of the grid on each side of the mesh refinement interface. -In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors (J. Vay et al. 2002; Colella and Norgaard 2010). As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. +In addition, for some implementations where the field that is computed at a given level is affected by the solution at finer levels, there are cases where the procedure violates the integral of Gauss’ Law around the refined patch, leading to long range errors :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. As will be shown below, in the procedure that has been developed in WarpX, the field at a given refinement level is not affected by the solution at finer levels, and is thus not affected by this type of error. Electrostatic ------------- -A cornerstone of the Particle-In-Cell method is that assuming a particle lying in a hypothetical infinite grid, then if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme (Birdsall and Langdon 1991). A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) (J. Vay et al. 2002; Colella and Norgaard 2010). - -A sketch of the implementation of mesh refinement in WarpX is given in Figure \ `[fig:ESAMR] <#fig:ESAMR>`__ (left). Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. +A cornerstone of the Particle-In-Cell method is that given a particle lying in a hypothetical infinite grid, if the grid is regular and symmetrical, and if the order of field gathering matches the order of charge (or current) deposition, then there is no self-force of the particle acting on itself: a) anywhere if using the so-called “momentum conserving” gathering scheme; b) on average within one cell if using the “energy conserving” gathering scheme :cite:p:`amr-Birdsalllangdon`. A breaking of the regularity and/or symmetry in the grid, whether it is from the use of irregular meshes or mesh refinement, and whether one uses finite difference, finite volume or finite elements, results in a net spurious self-force (which does not average to zero over one cell) for a macroparticle close to the point of irregularity (mesh refinement interface for the current purpose) :cite:p:`amr-Vaylpb2002,amr-Colellajcp2010`. -.. raw:: latex +A sketch of the implementation of mesh refinement in WarpX is given in :numref:`fig_ESAMR`. Given the solution of the electric potential at a refinement level :math:`L_n`, it is interpolated onto the boundaries of the grid patch(es) at the next refined level :math:`L_{n+1}`. The electric potential is then computed at level :math:`L_{n+1}` by solving the Poisson equation. This procedure necessitates the knowledge of the charge density at every level of refinement. For efficiency, the macroparticle charge is deposited on the highest level patch that contains them, and the charge density of each patch is added recursively to lower levels, down to the lowest. - \centering +.. _fig_ESselfforce: .. figure:: ICNSP_2011_Vay_fig2.png :alt: Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. - :name: fig:ESselfforce - :width: 15cm + :width: 95% Position history of one charged particle attracted by its image induced by a nearby metallic (dirichlet) boundary. The particle is initialized at rest. Without refinement patch (reference case), the particle is accelerated by its image, is reflected specularly at the wall, then decelerates until it reaches its initial position at rest. If the particle is initialized inside a refinement patch, the particle is initially accelerated toward the wall but is spuriously reflected before it reaches the boundary of the patch whether using the method implemented in WarpX or the MC method. Providing a surrounding transition region 2 or 4 cells wide in which the potential is interpolated from the parent coarse solution reduces significantly the effect of the spurious self-force. -The presence of the self-force is illustrated on a simple test case that was introduced in (J. Vay et al. 2002) and also used in (Colella and Norgaard 2010): a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` (“main grid”), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled “MC” in this paper) based on the algorithm given in (Mccorquodale et al. 2004), which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in Fig. `[fig:ESselfforce] <#fig:ESselfforce>`__. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - -.. raw:: latex +The presence of the self-force is illustrated on a simple test case that was introduced in :cite:t:`amr-Vaylpb2002` and also used in :cite:t:`amr-Colellajcp2010`: a single macroparticle is initialized at rest within a single refinement patch four cells away from the patch refinement boundary. The patch at level :math:`L_1` has :math:`32\times32` cells and is centered relative to the lowest :math:`64\times64` grid at level :math:`L_0` ("main grid"), while the macroparticle is centered in one direction but not in the other. The boundaries of the main grid are perfectly conducting, so that the macroparticle is attracted to the closest wall by its image. Specular reflection is applied when the particle reaches the boundary so that the motion is cyclic. The test was performed with WarpX using either linear or quadratic interpolation when gathering the main grid solution onto the refined patch boundary. It was also performed using another method from P. McCorquodale et al (labeled "MC" in this paper) based on the algorithm given in :cite:t:`amr-Mccorquodalejcp2004`, which employs a more elaborate procedure involving two-ways interpolations between the main grid and the refined patch. A reference case was also run using a single :math:`128\times128` grid with no refined patch, in which it is observed that the particle propagates toward the closest boundary at an accelerated pace, is reflected specularly at the boundary, then slows down until it reaches its initial position at zero velocity. The particle position histories are shown for the various cases in :numref:`fig_ESselfforce`. In all the cases using the refinement patch, the particle was spuriously reflected near the patch boundary and was effectively trapped in the patch. We notice that linear interpolation performs better than quadratic, and that the simple method implemented in WarpX performs better than the other proposed method for this test (see discussion below). - \centering +.. _fig_ESselfforcemap: .. figure:: ICNSP_2011_Vay_fig3.png :alt: (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). - :name: fig:ESselfforcemap - :width: 15cm + :width: 95% (left) Maps of the magnitude of the spurious self-force :math:`\epsilon` in arbitrary units within one quarter of the refined patch, defined as :math:`\epsilon=\sqrt{(E_x-E_x^{ref})^2+(E_y-E_y^{ref})^2}`, where :math:`E_x` and :math:`E_y` are the electric field components within the patch experienced by one particle at a given location and :math:`E_x^{ref}` and :math:`E_y^{ref}` are the electric field from a reference solution. The map is given for the WarpX and the MC mesh refinement algorithms and for linear and quadratic interpolation at the patch refinement boundary. (right) Lineouts of the maximum (taken over neighboring cells) of the spurious self-force. Close to the interface boundary (x=0), the spurious self-force decreases at a rate close to one order of magnitude per cell (red line), then at about one order of magnitude per six cells (green line). -The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in Fig. `[fig:ESselfforcemap] <#fig:ESselfforcemap>`__ for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. +The magnitude of the spurious self-force as a function of the macroparticle position was mapped and is shown in :numref:`fig_ESselfforcemap` for the WarpX and MC algorithms using linear or quadratic interpolations between grid levels. It is observed that the magnitude of the spurious self-force decreases rapidly with the distance between the particle and the refined patch boundary, at a rate approaching one order of magnitude per cell for the four cells closest to the boundary and about one order of magnitude per six cells beyond. The method implemented in WarpX offers a weaker spurious force on average and especially at the cells that are the closest to the coarse-fine interface where it is the largest and thus matters most. We notice that the magnitude of the spurious self-force depends strongly on the distance to the edge of the patch and to the nodes of the underlying coarse grid, but weakly on the order of deposition and size of the patch. -A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra “transition” cells are added around the “effective” refined area. -Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in Fig. \ `[fig:ESselfforce] <#fig:ESselfforce>`__, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. -We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface (Colella and Norgaard 2010). +A method was devised and implemented in WarpX for reducing the magnitude of spurious self-forces near the coarse-fine boundaries as follows. Noting that the coarse grid solution is unaffected by the presence of the patch and is thus free of self-force, extra "transition" cells are added around the "effective" refined area. +Within the effective area, the particles gather the potential in the fine grid. In the extra transition cells surrounding the refinement patch, the force is gathered directly from the coarse grid (an option, which has not yet been implemented, would be to interpolate between the coarse and fine grid field solutions within the transition zone so as to provide continuity of the force experienced by the particles at the interface). The number of cells allocated in the transition zones is controllable by the user in WarpX, giving the opportunity to check whether the spurious self-force is affecting the calculation by repeating it using different thicknesses of the transition zones. The control of the spurious force using the transition zone is illustrated in :numref:`fig_ESselfforce`, where the calculation with WarpX using linear interpolation at the patch interface was repeated using either two or four cells transition regions (measured in refined patch cell units). Using two extra cells allowed for the particle to be free of spurious trapping within the refined area and follow a trajectory that is close to the reference one, and using four extra cells improved further to the point where the resulting trajectory becomes indistinguishable from the reference one. +We note that an alternative method was devised for reducing the magnitude of self-force near the coarse-fine boundaries for the MC method, by using a special deposition procedure near the interface :cite:p:`amr-Colellajcp2010`. Electromagnetic --------------- -The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of (Vay 2001), refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. +The method that is used for electrostatic mesh refinement is not directly applicable to electromagnetic calculations. As was shown in section 3.4 of :cite:t:`amr-Vayjcp01`, refinement schemes relying solely on interpolation between coarse and fine patches lead to the reflection with amplification of the short wavelength modes that fall below the cutoff of the Nyquist frequency of the coarse grid. Unless these modes are damped heavily or prevented from occurring at their source, they may affect particle motion and their effect can escalate if trapped within a patch, via multiple successive reflections with amplification. -To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in Fig. \ `[fig:ESAMR] <#fig:ESAMR>`__-right and described in (Vay, Adam, and Heron 2004). Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user (Berenger 1996; J.-L. Vay 2002). The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. +To circumvent this issue, an additional coarse patch (with the same resolution as the parent grid) is added, as shown in :numref:`fig_ESAMR` and described in :cite:t:`amr-Vaycpc04`. Both the fine and the coarse grid patches are terminated by Perfectly Matched Layers, reducing wave reflection by orders of magnitude, controllable by the user :cite:p:`amr-Berengerjcp96,amr-Vayjcp02`. The source current resulting from the motion of charged macroparticles within the refined region is accumulated on the fine patch and is then interpolated onto the coarse patch and added onto the parent grid. The process is repeated recursively from the finest level down to the coarsest. The Maxwell equations are then solved for one time interval on the entire set of grids, by default for one time step using the time step of the finest grid. The field on the coarse and fine patches only contain the contributions from the particles that have evolved within the refined area but not from the current sources outside the area. The total contribution of the field from sources within and outside the refined area is obtained by adding the field from the refined grid :math:`F(r)`, and adding an interpolation :math:`I` of the difference between the relevant subset :math:`s` of the field in the parent grid :math:`F(s)` and the field of the coarse grid :math:`F( c )`, on an auxiliary grid :math:`a`, i.e. :math:`F(a)=F(r)+I[F(s)-F( c )]`. The field on the parent grid subset :math:`F(s)` contains contributions from sources from both within and outside of the refined area. Thus, in effect, there is substitution of the coarse field resulting from sources within the patch area by its fine resolution counterpart. The operation is carried out recursively starting at the coarsest level up to the finest. An option has been implemented in which various grid levels are pushed with different time steps, given as a fixed fraction of the individual grid Courant conditions (assuming same cell aspect ratio for all grids and refinement by integer factors). In this case, the fields from the coarse levels, which are advanced less often, are interpolated in time. The substitution method has two potential drawbacks due to the inexact cancellation between the coarse and fine patches of : (i) the remnants of ghost fixed charges created by the particles entering and leaving the patches (this effect is due to the use of the electromagnetic solver and is different from the spurious self-force that was described for the electrostatic case); (ii) if using a Maxwell solver with a low-order stencil, the electromagnetic waves traveling on each patch at slightly different velocity due to numerical dispersion. The first issue results in an effective spurious multipole field whose magnitude decreases very rapidly with the distance to the patch boundary, similarly to the spurious self-force in the electrostatic case. Hence, adding a few extra transition cells surrounding the patches mitigates this effect very effectively. -The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in (Vay, Adam, and Heron 2004). -Both effects and their mitigation are described in more detail in (Vay, Adam, and Heron 2004). +The tunability of WarpX’s electromagnetic finite-difference and pseudo-spectral solvers provides the means to optimize the numerical dispersion so as to minimize the second effect for a given application, which has been demonstrated on the laser-plasma interaction test case presented in :cite:t:`amr-Vaycpc04`. +Both effects and their mitigation are described in more detail in :cite:t:`amr-Vaycpc04`. Caustics are supported anywhere on the grid with an accuracy that is set by the local resolution, and will be adequately resolved if the grid resolution supports the necessary modes from their sources to the points of wavefront crossing. The mesh refinement method that is implemented in WarpX has the potential to provide higher efficiency than the standard use of fixed gridding, by offering a path toward adaptive gridding following wavefronts. -.. raw:: html - -
- -.. raw:: html - -
- -Berenger, Jp. 1996. “Three-Dimensional Perfectly Matched Layer for the Absorption of Electromagnetic Waves.” *Journal of Computational Physics* 127 (2): 363–79. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Colella, Phillip, and Peter C Norgaard. 2010. “Controlling Self-Force Errors at Refinement Boundaries for Amr-Pic.” *Journal of Computational Physics* 229 (4): 947–57. https://doi.org/10.1016/J.Jcp.2009.07.004. - -.. raw:: html - -
- -.. raw:: html - -
- -Mccorquodale, P, P Colella, Dp Grote, and Jl Vay. 2004. “A Node-Centered Local Refinement Algorithm For Poisson’s Equation In Complex Geometries.” *Journal of Computational Physics* 201 (1): 34–60. https://doi.org/10.1016/J.Jcp.2004.04.022. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2001. “An Extended Fdtd Scheme for the Wave Equation: Application to Multiscale Electromagnetic Simulation.” *Journal of Computational Physics* 167 (1): 72–98. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “Asymmetric Perfectly Matched Layer for the Absorption of Waves.” *Journal of Computational Physics* 183 (2): 367–99. https://doi.org/10.1006/Jcph.2002.7175. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, P Colella, P Mccorquodale, B Van Straalen, A Friedman, and Dp Grote. 2002. “Mesh Refinement for Particle-in-Cell Plasma Simulations: Applications to and Benefits for Heavy Ion Fusion.” *Laser and Particle Beams* 20 (4): 569–75. https://doi.org/10.1017/S0263034602204139. - -.. raw:: html - -
- -.. raw:: html - -
+.. bibliography:: + :keyprefix: amr- diff --git a/Docs/source/theory/boosted_frame.rst b/Docs/source/theory/boosted_frame.rst index 4b80c3d2ccb..ea1f662bd30 100644 --- a/Docs/source/theory/boosted_frame.rst +++ b/Docs/source/theory/boosted_frame.rst @@ -5,16 +5,18 @@ Moving window and optimal Lorentz boosted frame The simulations of plasma accelerators from first principles are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated particle beam into a plasma structure that is orders of magnitude longer and wider than the accelerated beam. As is customary in the modeling of particle beam dynamics in standard particle accelerators, a moving window is commonly used to follow the driver, the wake and the accelerated beam. This results in huge savings, by avoiding the meshing of the entire plasma that is orders of magnitude longer than the other length scales of interest. +.. _fig_Boosted_frame: + .. figure:: Boosted_frame.png - :alt: [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + :alt: [fig:Boosted-frame] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. - [fig:PIC] A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. + A first principle simulation of a short driver beam (laser or charged particles) propagating through a plasma that is orders of magnitude longer necessitates a very large number of time steps. Recasting the simulation in a frame of reference that is moving close to the speed of light in the direction of the driver beam leads to simulating a driver beam that appears longer propagating through a plasma that appears shorter than in the laboratory. Thus, this relativistic transformation of space and time reduces the disparity of scales, and thereby the number of time steps to complete the simulation, by orders of magnitude. -Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude (Vay 2007; J -L. Vay et al. 2011). This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in Fig. `[fig:PIC] <#fig:PIC>`__, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). +Even using a moving window, however, a full PIC simulation of a plasma accelerator can be extraordinarily demanding computationally, as many time steps are needed to resolve the crossing of the short driver beam with the plasma column. As it turns out, choosing an optimal frame of reference that travels close to the speed of light in the direction of the laser or particle beam (as opposed to the usual choice of the laboratory frame) enables speedups by orders of magnitude :cite:p:`bf-Vayprl07,bf-Vaypop2011`. This is a result of the properties of Lorentz contraction and dilation of space and time. In the frame of the laboratory, a very short driver (laser or particle) beam propagates through a much longer plasma column, necessitating millions to tens of millions of time steps for parameters in the range of the BELLA or FACET-II experiments. As sketched in :numref:`fig_Boosted_frame`, in a frame moving with the driver beam in the plasma at velocity :math:`v=\beta c` (where :math:`c` is the speed of light in vacuum), the beam length is now elongated by :math:`\approx(1+\beta)\gamma` while the plasma contracts by :math:`\gamma` (where :math:`\gamma=1/\sqrt{1-\beta^2}` is the relativistic factor associated with the frame velocity). The number of time steps that is needed to simulate a “longer” beam through a “shorter” plasma is now reduced by up to :math:`\approx(1+\beta) \gamma^2` (a detailed derivation of the speedup is given below). The modeling of a plasma acceleration stage in a boosted frame involves the fully electromagnetic modeling of a plasma propagating at near the speed of light, for which Numerical Cerenkov -(Boris and Lee 1973; Haber et al. 1973) is a potential issue, as explained in more details below. +:cite:p:`bf-Borisjcp73,bf-Habericnsp73` is a potential issue, as explained in more details below. In addition, for a frame of reference moving in the direction of the accelerated beam (or equivalently the wake of the laser), waves emitted by the plasma in the forward direction expand while the ones emitted in the backward direction contract, following the properties of the Lorentz transformation. @@ -25,16 +27,15 @@ Backscatter is weak in the short-pulse regime, and does not interact as strongly with the beam as do the forward propagating waves which stay in phase for a long period. It is thus often assumed that the backward propagating waves can be neglected in the modeling of plasma accelerator stages. The accuracy of this assumption has been demonstrated by -comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves -(Geddes et al. 2008; Geddes et al. 2009; Cowan et al. 2009). +comparison between explicit codes which include both forward and backward waves and envelope or quasistatic codes which neglect backward waves :cite:p:`bf-Geddesjp08,bf-Geddespac09,bf-Cowanaac08`. Theoretical speedup dependency with the frame boost --------------------------------------------------- -The derivation that is given here reproduces the one given in (J -L. Vay et al. 2011), where the obtainable speedup is derived as an extension of the formula that was derived earlier(Vay 2007), taking in addition into account the group velocity of the laser as it traverses the plasma. +The derivation that is given here reproduces the one given in :cite:t:`bf-Vaypop2011`, where the obtainable speedup is derived as an extension of the formula that was derived earlier :cite:p:`bf-Vayprl07`, taking in addition into account the group velocity of the laser as it traverses the plasma. Assuming that the simulation box is a fixed number of plasma periods long, which implies the use (which is standard) of a moving window following -the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by (Esarey, Schroeder, and Leemans 2009): +the wake and accelerated beam, the speedup is given by the ratio of the time taken by the laser pulse and the plasma to cross each other, divided by the shortest time scale of interest, that is the laser period. To first order, the wake velocity :math:`v_w` is set by the 1D group velocity of the laser driver, which in the linear (low intensity) limit, is given by :cite:p:`bf-Esareyrmp09`: .. math:: v_w/c=\beta_w=\left(1-\frac{\omega_p^2}{\omega^2}\right)^{1/2} @@ -58,76 +59,78 @@ In a frame moving at :math:`\beta c`, the quantities become .. math:: \begin{aligned} - \lambda_p^*&=&\lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] \\ - L^*&=&L/\gamma \\ - \lambda^*&=& \gamma\left(1+\beta\right) \lambda\\ - \beta_w^*&=&\left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) \\ - v_p^*&=&-\beta c \\ - T^*&=&\frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} \\ - R_t^*&=&\frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*}\end{aligned} + \lambda_p^* & = \lambda_p/\left[\gamma \left(1-\beta_w \beta\right)\right] + \\ + L^* & = L/\gamma + \\ + \lambda^* & = \gamma\left(1+\beta\right) \lambda + \\ + \beta_w^* & = \left(\beta_w-\beta\right)/\left(1-\beta_w\beta\right) + \\ + v_p^* & = -\beta c + \\ + T^* & = \frac{L^*+\eta \lambda_p^*}{v_w^*-v_p^*} + \\ + R_t^* & = \frac{T^* c}{\lambda^*} = \frac{\left(L^*+\eta \lambda_p^*\right)}{\left(\beta_w^*+\beta\right) \lambda^*} + \end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. The expected speedup from performing the simulation in a boosted frame is given by the ratio of :math:`R_{lab}` and :math:`R_t^*` .. math:: - S=\frac{R_{lab}}{R_t^*}=\frac{\left(1+\beta\right)\left(L+\eta \lambda_p\right)}{\left(1-\beta\beta_w\right)L+\eta \lambda_p} - \label{Eq_scaling1d0} + :label: Eq_scaling1d0 -We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier (Vay 2007) for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. +We note that assuming that :math:`\beta_w\approx1` (which is a valid approximation for most practical cases of interest) and that :math:`\gamma<<\gamma_w`, this expression is consistent with the expression derived earlier :cite:p:`bf-Vayprl07` for the laser-plasma acceleration case, which states that :math:`R_t^*=\alpha R_t/\left(1+\beta\right)` with :math:`\alpha=\left(1-\beta+l/L\right)/\left(1+l/L\right)`, where :math:`l` is the laser length which is generally proportional to :math:`\eta \lambda_p`, and :math:`S=R_t/R_T^*`. However, higher values of :math:`\gamma` are of interest for maximum speedup, as shown below. -For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing (Schroeder et al. 2011), which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. -Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(\ `[Eq_scaling1d0] <#Eq_scaling1d0>`__) becomes +For intense lasers (:math:`a\sim 1`) typically used for acceleration, the energy gain is limited by dephasing :cite:p:`bf-Schroederprl2011`, which occurs over a scale length :math:`L_d \sim \lambda_p^3/2\lambda^2`. +Acceleration is compromised beyond :math:`L_d` and in practice, the plasma length is proportional to the dephasing length, i.e. :math:`L= \xi L_d`. In most cases, :math:`\gamma_w^2>>1`, which allows the approximations :math:`\beta_w\approx1-\lambda^2/2\lambda_p^2`, and :math:`L=\xi \lambda_p^3/2\lambda^2\approx \xi \gamma_w^2 \lambda_p/2>>\eta \lambda_p`, so that Eq.(:eq:`Eq_scaling1d0`) becomes .. math:: - S=\left(1+\beta\right)^2\gamma^2\frac{\xi\gamma_w^2}{\xi\gamma_w^2+\left(1+\beta\right)\gamma^2\left(\xi\beta/2+2\eta\right)} - \label{Eq_scaling1d} + :label: Eq_scaling1d -For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) reduces to +For low values of :math:`\gamma`, i.e. when :math:`\gamma<<\gamma_w`, Eq.(:eq:`Eq_scaling1d`) reduces to .. math:: - S_{\gamma<<\gamma_w}=\left(1+\beta\right)^2\gamma^2 - \label{Eq_scaling1d_simpl2} + :label: Eq_scaling1d_simpl2 -Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) becomes +Conversely, if :math:`\gamma\rightarrow\infty`, Eq.(`Eq_scaling1d`) becomes .. math:: - S_{\gamma\rightarrow\infty}=\frac{4}{1+4\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_inf} + :label: Eq_scaling_gamma_inf -Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(\ `[Eq_scaling1d] <#Eq_scaling1d>`__) gives +Finally, in the frame of the wake, i.e. when :math:`\gamma=\gamma_w`, assuming that :math:`\beta_w\approx1`, Eq.(:eq:`Eq_scaling1d`) gives .. math:: - S_{\gamma=\gamma_w}\approx\frac{2}{1+2\eta/\xi}\gamma_w^2 - \label{Eq_scaling_gamma_wake} + :label: Eq_scaling_gamma_wake -Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(\ `[Eq_scaling_gamma_inf] <#Eq_scaling_gamma_inf>`__). +Since :math:`\eta` and :math:`\xi` are of order unity, and the practical regimes of most interest satisfy :math:`\gamma_w^2>>1`, the speedup that is obtained by using the frame of the wake will be near the maximum obtainable value given by Eq.(:eq:`Eq_scaling_gamma_inf`). -Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain (Vay 2007), and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. +Note that without the use of a moving window, the relativistic effects that are at play in the time domain would also be at play in the spatial domain :cite:p:`bf-Vayprl07`, and the :math:`\gamma^2` scaling would transform to :math:`\gamma^4`. Hence, it is important to use a moving window even in simulations in a Lorentz boosted frame. For very high values of the boosted frame, the optimal velocity of the moving window may vanish (i.e. no moving window) or even reverse. .. _theory-boostedframe-galilean: Numerical Stability and alternate formulation in a Galilean frame ----------------------------------------------------------------- -The numerical Cherenkov instability (NCI) (Godfrey 1974) +The numerical Cherenkov instability (NCI) :cite:p:`bf-Godfreyjcp74` is the most serious numerical instability affecting multidimensional PIC simulations of relativistic particle beams and streaming plasmas -(Martins et al. 2010; Vay et al. 2010; J L Vay et al. 2011; Sironi and Spitkovsky 2011; Godfrey and Vay 2013; Xu et al. 2013). +:cite:p:`bf-Martinscpc10,bf-VayAAC2010,bf-Vayjcp2011,bf-Spitkovsky:Icnsp2011,bf-GodfreyJCP2013,bf-XuJCP2013`. It arises from coupling between possibly numerically distorted electromagnetic modes and spurious beam modes, the latter due to the mismatch between the Lagrangian -treatment of particles and the Eulerian treatment of fields (Godfrey 1975). +treatment of particles and the Eulerian treatment of fields :cite:p:`bf-Godfreyjcp75`. In recent papers the electromagnetic dispersion -relations for the numerical Cherenkov instability were derived and solved for both FDTD (Godfrey and Vay 2013; Brendan B. Godfrey and Vay 2014) -and PSATD (Brendan B. Godfrey, Vay, and Haber 2014a, 2014b) algorithms. +relations for the numerical Cherenkov instability were derived and solved for both FDTD :cite:p:`bf-GodfreyJCP2013,bf-GodfreyJCP2014_FDTD` +and PSATD :cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyIEEE2014` algorithms. -Several solutions have been proposed to mitigate the NCI (Brendan B Godfrey, Vay, and Haber 2014; Brendan B. Godfrey, Vay, and Haber 2014b, 2014a; Godfrey and Vay 2015; Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015). Although +Several solutions have been proposed to mitigate the NCI :cite:p:`bf-GodfreyJCP2014,bf-GodfreyIEEE2014,bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015,bf-YuCPC2015,bf-YuCPC2015-Circ`. Although these solutions efficiently reduce the numerical instability, they typically introduce either strong smoothing of the currents and fields, or arbitrary numerical corrections, which are @@ -137,47 +140,46 @@ it is sometimes unclear to what extent these added corrections could impact the physics at stake for a given resolution. For instance, NCI-specific corrections include periodically smoothing -the electromagnetic field components (Martins et al. 2010), -using a special time step (Vay et al. 2010; J L Vay et al. 2011) or -applying a wide-band smoothing of the current components (Vay et al. 2010; J L Vay et al. 2011; J. Vay et al. 2011). Another set of mitigation methods +the electromagnetic field components :cite:p:`bf-Martinscpc10`, +using a special time step :cite:p:`bf-VayAAC2010,bf-Vayjcp2011` or +applying a wide-band smoothing of the current components :cite:p:`bf-VayAAC2010,bf-Vayjcp2011,bf-VayPOPL2011`. Another set of mitigation methods involve scaling the deposited currents by a carefully-designed wavenumber-dependent factor -(Brendan B. Godfrey and Vay 2014; Brendan B. Godfrey, Vay, and Haber 2014b) or slightly modifying the +:cite:p:`bf-GodfreyJCP2014_FDTD,bf-GodfreyIEEE2014` or slightly modifying the ratio of electric and magnetic fields (:math:`E/B`) before gathering their value onto the macroparticles -(Brendan B. Godfrey, Vay, and Haber 2014a; Godfrey and Vay 2015). +:cite:p:`bf-GodfreyJCP2014_PSATD,bf-GodfreyCPC2015`. Yet another set of NCI-specific corrections -(Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015) consists +:cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` consists in combining a small timestep :math:`\Delta t`, a sharp low-pass spatial filter, and a spectral or high-order scheme that is tuned so as to create a small, artificial “bump” in the dispersion relation -(Yu, Xu, Decyk, et al. 2015). While most mitigation methods have only been applied +:cite:p:`bf-YuCPC2015`. While most mitigation methods have only been applied to Cartesian geometry, this last -set of methods ((Yu, Xu, Decyk, et al. 2015; Yu, Xu, Tableman, et al. 2015)) +set of methods :cite:p:`bf-YuCPC2015,bf-YuCPC2015-Circ` has the remarkable property that it can be applied -(Yu, Xu, Tableman, et al. 2015) to both Cartesian geometry and +:cite:p:`bf-YuCPC2015-Circ` to both Cartesian geometry and quasi-cylindrical geometry (i.e. cylindrical geometry with -azimuthal Fourier decomposition (Lifschitz et al. 2009; Davidson et al. 2015; R. Lehe et al. 2016)). However, +azimuthal Fourier decomposition :cite:p:`bf-LifschitzJCP2009,bf-DavidsonJCP2015,bf-Lehe2016`). However, the use of a small timestep proportionally slows down the progress of the simulation, and the artificial “bump” is again an arbitrary correction that departs from the underlying physics. -A new scheme was recently proposed, in (Kirchen et al. 2016; Lehe et al. 2016), which +A new scheme was recently proposed, in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, which completely eliminates the NCI for a plasma drifting at a uniform relativistic velocity – with no arbitrary correction – by simply integrating the PIC equations in *Galilean coordinates* (also known as *comoving coordinates*). More precisely, in the new method, the Maxwell equations *in Galilean coordinates* are integrated analytically, using only natural hypotheses, within the PSATD -framework (Pseudo-Spectral-Analytical-Time-Domain (Haber et al. 1973; Vay, Haber, and Godfrey 2013)). +framework (Pseudo-Spectral-Analytical-Time-Domain :cite:p:`bf-Habericnsp73,bf-VayJCP2013`). The idea of the proposed scheme is to perform a Galilean change of coordinates, and to carry out the simulation in the new coordinates: .. math:: - - \label{eq:change-var} \boldsymbol{x}' = \boldsymbol{x} - \boldsymbol{v}_{gal}t + :label: change-var where :math:`\boldsymbol{x} = x\,\boldsymbol{u}_x + y\,\boldsymbol{u}_y + z\,\boldsymbol{u}_z` and :math:`\boldsymbol{x}' = x'\,\boldsymbol{u}_x + y'\,\boldsymbol{u}_y + z'\,\boldsymbol{u}_z` are the @@ -190,10 +192,10 @@ plasma, the plasma does not move with respect to the grid in the Galilean coordinates :math:`\boldsymbol{x}'` – or, equivalently, in the standard coordinates :math:`\boldsymbol{x}`, the grid moves along with the plasma. The heuristic intuition behind this scheme is that these coordinates should prevent the discrepancy between the Lagrangian and -Eulerian point of view, which gives rise to the NCI (Godfrey 1975). +Eulerian point of view, which gives rise to the NCI :cite:p:`bf-Godfreyjcp75`. An important remark is that the Galilean change of -coordinates (`[eq:change-var] <#eq:change-var>`__) is a simple translation. Thus, when used in +coordinates in Eq. (:eq:`change-var`) is a simple translation. Thus, when used in the context of Lorentz-boosted simulations, it does of course preserve the relativistic dilatation of space and time which gives rise to the characteristic computational speedup of the boosted-frame technique. @@ -206,378 +208,81 @@ translate the boundaries, in the Galilean scheme the gridpoints *themselves* are not only translated but in this case, the physical equations are modified accordingly. Most importantly, the assumed time evolution of the current :math:`\boldsymbol{J}` within one timestep is different in a standard PSATD scheme with moving -window and in a Galilean PSATD scheme (Lehe et al. 2016). +window and in a Galilean PSATD scheme :cite:p:`bf-LehePRE2016`. In the Galilean coordinates :math:`\boldsymbol{x}'`, the equations of particle motion and the Maxwell equations take the form .. math:: + \frac{d\boldsymbol{x}'}{dt} = \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal} + :label: motion1 - \begin{aligned} - \frac{d\boldsymbol{x}'}{dt} &= \frac{\boldsymbol{p}}{\gamma m} - \boldsymbol{v}_{gal}\label{eq:motion1} \\ - \frac{d\boldsymbol{p}}{dt} &= q \left( \boldsymbol{E} + - \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) \label{eq:motion2}\\ - \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} &= -\boldsymbol{\nabla'}\times\boldsymbol{E} \label{eq:maxwell1}\\ - \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} &= \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} \label{eq:maxwell2}\end{aligned} +.. math:: + \frac{d\boldsymbol{p}}{dt} = q \left( \boldsymbol{E} + \frac{\boldsymbol{p}}{\gamma m} \times \boldsymbol{B} \right) + :label: motion2 + +.. math:: + \left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{B} = -\boldsymbol{\nabla'}\times\boldsymbol{E} + :label: maxwell1 + +.. math:: + \frac{1}{c^2}\left( \frac{\partial \;}{\partial t} - \boldsymbol{v}_{gal}\cdot\boldsymbol{\nabla'}\right)\boldsymbol{E} = \boldsymbol{\nabla'}\times\boldsymbol{B} - \mu_0\boldsymbol{J} + :label: maxwell2 where :math:`\boldsymbol{\nabla'}` denotes a spatial derivative with respect to the Galilean coordinates :math:`\boldsymbol{x}'`. Integrating these equations from :math:`t=n\Delta t` to :math:`t=(n+1)\Delta t` results in the following update equations (see -(Lehe et al. 2016) for the details of the derivation): +:cite:t:`bf-LehePRE2016` for the details of the derivation): .. math:: - \begin{aligned} - \mathbf{\tilde{B}}^{n+1} &= \theta^2 C \mathbf{\tilde{B}}^n - -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber \\ - & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times - \mathbf{\tilde{J}}^{n+1/2} \label{eq:disc-maxwell1}\\ - \mathbf{\tilde{E}}^{n+1} &= \theta^2 C \mathbf{\tilde{E}}^n - +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber \\ - & +\frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber \\ - & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} \label{eq:disc-maxwell2}\end{aligned} - -where we used the short-hand notations :math:`\mathbf{\tilde{E}}^n \equiv -% -\mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, :math:`\mathbf{\tilde{B}}^n \equiv -\mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: + \mathbf{\tilde{B}}^{n+1} & = \theta^2 C \mathbf{\tilde{B}}^n -\frac{\theta^2 S}{ck}i\boldsymbol{k}\times \mathbf{\tilde{E}}^n \nonumber + \\ + & + \;\frac{\theta \chi_1}{\epsilon_0c^2k^2}\;i\boldsymbol{k} \times \mathbf{\tilde{J}}^{n+1/2} + \end{aligned} + :label: disc-maxwell1 .. math:: - \begin{aligned} - &C = \cos(ck\Delta t) \quad S = \sin(ck\Delta t) \quad k - = |\boldsymbol{k}| \label{eq:def-C-S}\\& - \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck} \quad \theta = - e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \quad \theta^* = - e^{-i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2} \label{eq:def-nu-theta}\\& - \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i - \nu \theta S \right) \label{eq:def-chi1}\\& - \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} \quad - \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} \label{eq:def-chi23}\end{aligned} - -Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, -(`[eq:disc-maxwell1] <#eq:disc-maxwell1>`__) and (`[eq:disc-maxwell2] <#eq:disc-maxwell2>`__) reduce to the standard PSATD -equations (Haber et al. 1973), as expected. -As shown in (Kirchen et al. 2016; Lehe et al. 2016), -the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp, and R Lee. 1973. “Nonphysical Self Forces in Some Electromagnetic Plasma-Simulation Algorithms.” Note. *Journal of Computational Physics* 12 (1). 525 B St, Ste 1900, San Diego, Ca 92101-4495: Academic Press Inc Jnl-Comp Subscriptions: 131–36. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, B, D Bruhwiler, E Cormier-Michel, E Esarey, C G R Geddes, P Messmer, and K Paul. 2009. “Laser Wakefield Simulation Using A Speed-of-Light Frame Envelope Model.” In *Aip Conference Proceedings*, 1086:309–14. - -.. raw:: html - -
- -.. raw:: html - -
- -Davidson, A., A. Tableman, W. An, F.S. Tsung, W. Lu, J. Vieira, R.A. Fonseca, L.O. Silva, and W.B. Mori. 2015. “Implementation of a hybrid particle code with a PIC description in r–z and a gridless description in :math:`\Phi` into OSIRIS.” *Journal of Computational Physics* 281: 1063–77. https://doi.org/10.1016/j.jcp.2014.10.064. - -.. raw:: html - -
- -.. raw:: html - -
- -Esarey, E, C B Schroeder, and W P Leemans. 2009. “Physics of Laser-Driven Plasma-Based Electron Accelerators.” *Rev. Mod. Phys.* 81 (3): 1229–85. https://doi.org/10.1103/Revmodphys.81.1229. - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes, C G R, D L Bruhwiler, J R Cary, W B Mori, J.-L. Vay, S F Martins, T Katsouleas, et al. 2008. “Computational Studies and Optimization of Wakefield Accelerators.” In *Journal of Physics: Conference Series*, 125:012002 (11 Pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Geddes et al., C G R. 2009. “Scaled Simulation Design of High Quality Laser Wakefield Accelerator Stages.” In *Proc. Particle Accelerator Conference*. Vancouver, Canada. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Bb. 1974. “Numerical Cherenkov Instabilities in Electromagnetic Particle Codes.” *Journal of Computational Physics* 15 (4): 504–21. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 1975. “Canonical Momenta and Numerical Instabilities in Particle Codes.” *Journal of Computational Physics* 19 (1): 58–76. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, and Jean-Luc Vay. 2013. “Numerical stability of relativistic beam multidimensional {PIC} simulations employing the Esirkepov algorithm.” *Journal of Computational Physics* 248 (0): 33–46. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.04.006. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., and Jean Luc Vay. 2014. “Suppressing the numerical Cherenkov instability in FDTD PIC codes.” *Journal of Computational Physics* 267: 1–6. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2015. “Improved numerical Cherenkov instability suppression in the generalized PSTD PIC algorithm.” *Computer Physics Communications* 196. Elsevier: 221–25. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B., Jean Luc Vay, and Irving Haber. 2014a. “Numerical stability analysis of the pseudo-spectral analytical time-domain PIC algorithm.” *Journal of Computational Physics* 258: 689–704. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2014b. “Numerical stability improvements for the pseudospectral EM PIC algorithm.” *IEEE Transactions on Plasma Science* 42 (5). Institute of Electrical; Electronics Engineers Inc.: 1339–44. - -.. raw:: html - -
- -.. raw:: html - -
- -Godfrey, Brendan B, Jean-Luc Vay, and Irving Haber. 2014. “Numerical stability analysis of the pseudo-spectral analytical time-domain {PIC} algorithm.” *Journal of Computational Physics* 258 (0): 689–704. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2013.10.053. - -.. raw:: html + \mathbf{\tilde{E}}^{n+1} & = \theta^2 C \mathbf{\tilde{E}}^n +\frac{\theta^2 S}{k} \,c i\boldsymbol{k}\times \mathbf{\tilde{B}}^n \nonumber + \\ + & + \frac{i\nu \theta \chi_1 - \theta^2S}{\epsilon_0 ck} \; \mathbf{\tilde{J}}^{n+1/2}\nonumber + \\ + & - \frac{1}{\epsilon_0k^2}\left(\; \chi_2\;\hat{\mathcal{\rho}}^{n+1} - \theta^2\chi_3\;\hat{\mathcal{\rho}}^{n} \;\right) i\boldsymbol{k} + \end{aligned} + :label: disc-maxwell2 + +where we used the short-hand notations +:math:`\mathbf{\tilde{E}}^n \equiv \mathbf{\tilde{E}}(\boldsymbol{k}, n\Delta t)`, +:math:`\mathbf{\tilde{B}}^n \equiv \mathbf{\tilde{B}}(\boldsymbol{k}, n\Delta t)` as well as: -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Kirchen, M., R. Lehe, B. B. Godfrey, I. Dornmair, S. Jalas, K. Peters, J.-L. Vay, and A. R. Maier. 2016. “Stable discrete representation of relativistically drifting plasmas.” *arXiv:1608.00215*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, Rémi, Manuel Kirchen, Igor A. Andriyash, Brendan B. Godfrey, and Jean-Luc Vay. 2016. “A spectral, quasi-cylindrical and dispersion-free Particle-In-Cell algorithm.” *Computer Physics Communications* 203: 66–82. https://doi.org/10.1016/j.cpc.2016.02.007. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R., M. Kirchen, B. B. Godfrey, A. R. Maier, and J.-L. Vay. 2016. “Elimination of Numerical Cherenkov Instability in flowing-plasma Particle-In-Cell simulations by using Galilean coordinates.” *arXiv:1608.00227*. - -.. raw:: html - -
- -.. raw:: html - -
- -Lifschitz, A F, X Davoine, E Lefebvre, J Faure, C Rechatin, and V Malka. 2009. “Particle-in-Cell modelling of laser-plasma interaction using Fourier decomposition.” *Journal of Computational Physics* 228 (5): 1803–14. https://doi.org/http://dx.doi.org/10.1016/j.jcp.2008.11.017. - -.. raw:: html - -
- -.. raw:: html - -
- -Martins, Samuel F, Ricardo A Fonseca, Luis O Silva, Wei Lu, and Warren B Mori. 2010. “Numerical Simulations of Laser Wakefield Accelerators in Optimal Lorentz Frames.” *Computer Physics Communications* 181 (5): 869–75. https://doi.org/10.1016/J.Cpc.2009.12.023. - -.. raw:: html - -
- -.. raw:: html - -
- -Schroeder, C B, C Benedetti, E Esarey, and W P Leemans. 2011. “Nonlinear Pulse Propagation and Phase Velocity of Laser-Driven Plasma Waves.” *Physical Review Letters* 106 (13): 135002. https://doi.org/10.1103/Physrevlett.106.135002. - -.. raw:: html - -
- -.. raw:: html - -
- -Sironi, L, and A Spitkovsky. 2011. “No Title.” - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -. L, C G R Geddes, C Benedetti, D L Bruhwiler, E Cormier-Michel, B M Cowan, J R Cary, and D P Grote. 2010. “Modeling Laser Wakefield Accelerators in A Lorentz Boosted Frame.” *Aip Conference Proceedings* 1299: 244–49. https://doi.org/10.1063/1.3520322. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jl, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Effects of Hyperbolic Rotation in Minkowski Space on the Modeling of Plasma Accelerators in A Lorentz Boosted Frame.” *Physics of Plasmas* 18 (3): 30701. https://doi.org/10.1063/1.3559483. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html - -
- -Xu, Xinlu, Peicheng Yu, Samual F Martins, Frank S Tsung, Viktor K Decyk, Jorge Vieira, Ricardo A Fonseca, Wei Lu, Luis O Silva, and Warren B Mori. 2013. “Numerical instability due to relativistic plasma drift in EM-PIC simulations.” *Computer Physics Communications* 184 (11): 2503–14. https://doi.org/http://dx.doi.org/10.1016/j.cpc.2013.07.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Yu, Peicheng, Xinlu Xu, Viktor K. Decyk, Frederico Fiuza, Jorge Vieira, Frank S. Tsung, Ricardo A. Fonseca, Wei Lu, Luis O. Silva, and Warren B. Mori. 2015. “Elimination of the numerical Cerenkov instability for spectral EM-PIC codes.” *Computer Physics Communications* 192 (July). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 32–47. https://doi.org/10.1016/j.cpc.2015.02.018. - -.. raw:: html - -
- -.. raw:: html +.. math:: + C = \cos(ck\Delta t), \quad S = \sin(ck\Delta t), \quad k = |\boldsymbol{k}|, + :label: def-C-S -
+.. math:: + \nu = \frac{\boldsymbol{k}\cdot\boldsymbol{v}_{gal}}{ck}, \quad \theta = e^{i\boldsymbol{k}\cdot\boldsymbol{v}_{gal}\Delta t/2}, + :label: def-nu-theta -Yu, Peicheng, Xinlu Xu, Adam Tableman, Viktor K. Decyk, Frank S. Tsung, Frederico Fiuza, Asher Davidson, et al. 2015. “Mitigation of numerical Cerenkov radiation and instability using a hybrid finite difference-FFT Maxwell solver and a local charge conserving current deposit.” *Computer Physics Communications* 197 (December). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 144–52. https://doi.org/10.1016/j.cpc.2015.08.026. +.. math:: + \chi_1 = \frac{1}{1 -\nu^2} \left( \theta^* - C \theta + i \nu \theta S \right), + :label: def-chi1 -.. raw:: html +.. math:: + \chi_2 = \frac{\chi_1 - \theta(1-C)}{\theta^*-\theta} + :label: def-chi2 -
+.. math:: + \chi_3 = \frac{\chi_1-\theta^*(1-C)}{\theta^*-\theta} + :label: def-chi3 -.. raw:: html +Note that, in the limit :math:`\boldsymbol{v}_{gal}=\boldsymbol{0}`, +Eqs. (:eq:`disc-maxwell1`) and (:eq:`disc-maxwell2`) reduce to the standard PSATD +equations :cite:p:`bf-Habericnsp73`, as expected. +As shown in :cite:t:`bf-KirchenPOP2016,bf-LehePRE2016`, +the elimination of the NCI with the new Galilean integration is verified empirically via PIC simulations of uniform drifting plasmas and laser-driven plasma acceleration stages, and confirmed by a theoretical analysis of the instability. -
+.. bibliography:: + :keyprefix: bf- diff --git a/Docs/source/theory/boundary_conditions.rst b/Docs/source/theory/boundary_conditions.rst new file mode 100644 index 00000000000..395b072ccbe --- /dev/null +++ b/Docs/source/theory/boundary_conditions.rst @@ -0,0 +1,303 @@ +.. _theory-bc: + +Boundary conditions +=================== + +.. _theory-bc-PML: + +Perfectly Matched Layer: open boundary condition for electromagnetic waves +-------------------------------------------------------------------------- + +For the transverse electric (TE) case, the original Berenger’s Perfectly Matched Layer (PML) paper :cite:p:`bc-Berengerjcp94` writes + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{\partial H_{z}}{\partial y} + :label: PML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{\partial H_{z}}{\partial x} + :label: PML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{\partial E_{y}}{\partial x} + :label: PML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{\partial E_{x}}{\partial y} + :label: PML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: PML_def_5 + +This can be generalized to + +.. math:: + \varepsilon _{0}\frac{\partial E_{x}}{\partial t}+\sigma _{y}E_{x} = \frac{c_{y}}{c}\frac{\partial H_{z}}{\partial y}+\overline{\sigma }_{y}H_{z} + :label: APML_def_1 + +.. math:: + \varepsilon _{0}\frac{\partial E_{y}}{\partial t}+\sigma _{x}E_{y} = -\frac{c_{x}}{c}\frac{\partial H_{z}}{\partial x}+\overline{\sigma }_{x}H_{z} + :label: APML_def_2 + +.. math:: + \mu _{0}\frac{\partial H_{zx}}{\partial t}+\sigma ^{*}_{x}H_{zx} = -\frac{c^{*}_{x}}{c}\frac{\partial E_{y}}{\partial x}+\overline{\sigma }_{x}^{*}E_{y} + :label: APML_def_3 + +.. math:: + \mu _{0}\frac{\partial H_{zy}}{\partial t}+\sigma ^{*}_{y}H_{zy} = \frac{c^{*}_{y}}{c}\frac{\partial E_{x}}{\partial y}+\overline{\sigma }_{y}^{*}E_{x} + :label: APML_def_4 + +.. math:: + H_{z} = H_{zx}+H_{zy} + :label: APML_def_5 + +For :math:`c_{x}=c_{y}=c^{*}_{x}=c^{*}_{y}=c` and :math:`\overline{\sigma }_{x}=\overline{\sigma }_{y}=\overline{\sigma }_{x}^{*}=\overline{\sigma }_{y}^{*}=0`, +this system reduces to the Berenger PML medium, while adding the additional +constraint :math:`\sigma _{x}=\sigma _{y}=\sigma _{x}^{*}=\sigma _{y}^{*}=0` +leads to the system of Maxwell equations in vacuum. + +.. _theory-bc-propa-plane-wave: + +Propagation of a Plane Wave in an APML Medium +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We consider a plane wave of magnitude (:math:`E_{0},H_{zx0},H_{zy0}`) +and pulsation :math:`\omega` propagating in the APML medium with an +angle :math:`\varphi` relative to the x axis + +.. math:: + E_{x} = -E_{0}\sin \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_1 + +.. math:: + E_{y} = E_{0}\cos \varphi \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_2 + +.. math:: + H_{zx} = H_{zx0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_AMPL_def_3 + +.. math:: + H_{zy} = H_{zy0} \: e^{i\omega \left( t-\alpha x-\beta y\right) } + :label: Plane_wave_APML_def_4 + +where :math:`\alpha` and :math:`\beta` are two complex constants to +be determined. + +Introducing Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_def_2`), +(:eq:`Plane_wave_AMPL_def_3`) and (:eq:`Plane_wave_APML_def_4`) +into Eqs. (:eq:`APML_def_1`), (:eq:`APML_def_2`), (:eq:`APML_def_3`) +and (:eq:`APML_def_4`) gives + +.. math:: + \varepsilon _{0}E_{0}\sin \varphi -i\frac{\sigma _{y}}{\omega }E_{0}\sin \varphi = \beta \frac{c_{y}}{c}\left( H_{zx0}+H_{zy0}\right) +i\frac{\overline{\sigma }_{y}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_1 + +.. math:: + \varepsilon _{0}E_{0}\cos \varphi -i\frac{\sigma _{x}}{\omega }E_{0}\cos \varphi = \alpha \frac{c_{x}}{c}\left( H_{zx0}+H_{zy0}\right) -i\frac{\overline{\sigma }_{x}}{\omega }\left( H_{zx0}+H_{zy0}\right) + :label: Plane_wave_APML_1_2 + +.. math:: + \mu _{0}H_{zx0}-i\frac{\sigma ^{*}_{x}}{\omega }H_{zx0} = \alpha \frac{c^{*}_{x}}{c}E_{0}\cos \varphi -i\frac{\overline{\sigma }^{*}_{x}}{\omega }E_{0}\cos \varphi + :label: Plane_wave_APML_1_3 + +.. math:: + \mu _{0}H_{zy0}-i\frac{\sigma ^{*}_{y}}{\omega }H_{zy0} = \beta \frac{c^{*}_{y}}{c}E_{0}\sin \varphi +i\frac{\overline{\sigma }^{*}_{y}}{\omega }E_{0}\sin \varphi + :label: Plane_wave_APML_1_4 + +Defining :math:`Z=E_{0}/\left( H_{zx0}+H_{zy0}\right)` and using Eqs. (:eq:`Plane_wave_APML_1_1`) +and (:eq:`Plane_wave_APML_1_2`), we get + +.. math:: + \beta = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi -i\frac{\overline{\sigma }_{y}}{\omega }\right] \frac{c}{c_{y}} + :label: Plane_wave_APML_beta_of_g + +.. math:: + \alpha = \left[ Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi +i\frac{\overline{\sigma }_{x}}{\omega }\right] \frac{c}{c_{x}} + :label: Plane_wave_APML_alpha_of_g + +Adding :math:`H_{zx0}` and :math:`H_{zy0}` from Eqs. (:eq:`Plane_wave_APML_1_3`) +and (:eq:`Plane_wave_APML_1_4`) and substituting the expressions +for :math:`\alpha` and :math:`\beta` from Eqs. (:eq:`Plane_wave_APML_beta_of_g`) +and (:eq:`Plane_wave_APML_alpha_of_g`) yields + +.. math:: + + \begin{aligned} + \frac{1}{Z} & = \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{x}}{\omega }\right) \cos \varphi \frac{c^{*}_{x}}{c_{x}}+i\frac{\overline{\sigma }_{x}}{\omega }\frac{c^{*}_{x}}{c_{x}}-i\frac{\overline{\sigma }^{*}_{x}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{x}}{\omega }}\cos \varphi \nonumber + \\ + & + \frac{Z\left( \varepsilon _{0}-i\frac{\sigma _{y}}{\omega }\right) \sin \varphi \frac{c^{*}_{y}}{c_{y}}-i\frac{\overline{\sigma }_{y}}{\omega }\frac{c^{*}_{y}}{c_{y}}+i\frac{\overline{\sigma }^{*}_{y}}{\omega }}{\mu _{0}-i\frac{\sigma ^{*}_{y}}{\omega }}\sin \varphi + \end{aligned} + +If :math:`c_{x}=c^{*}_{x}`, :math:`c_{y}=c^{*}_{y}`, :math:`\overline{\sigma }_{x}=\overline{\sigma }^{*}_{x}`, :math:`\overline{\sigma }_{y}=\overline{\sigma }^{*}_{y}`, :math:`\frac{\sigma _{x}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{x}}{\mu _{0}}` and :math:`\frac{\sigma _{y}}{\varepsilon _{0}}=\frac{\sigma ^{*}_{y}}{\mu _{0}}` then + +.. math:: + Z = \pm \sqrt{\frac{\mu _{0}}{\varepsilon _{0}}} + :label: APML_impedance + +which is the impedance of vacuum. Hence, like the PML, given some +restrictions on the parameters, the APML does not generate any reflection +at any angle and any frequency. As for the PML, this property is not +retained after discretization, as shown subsequently. + +Calling :math:`\psi` any component of the field and :math:`\psi _{0}` +its magnitude, we get from Eqs. (:eq:`Plane_wave_APML_def_1`), (:eq:`Plane_wave_APML_beta_of_g`), +(:eq:`Plane_wave_APML_alpha_of_g`) and (:eq:`APML_impedance`) that + +.. math:: + \psi =\psi _{0} \: e^{i\omega \left( t\mp x\cos \varphi /c_{x}\mp y\sin \varphi /c_{y}\right) }e^{-\left( \pm \frac{\sigma _{x}\cos \varphi }{\varepsilon _{0}c_{x}}+\overline{\sigma }_{x}\frac{c}{c_{x}}\right) x} e^{-\left( \pm \frac{\sigma _{y}\sin \varphi }{\varepsilon _{0}c_{y}}+\overline{\sigma }_{y}\frac{c}{c_{y}}\right) y}. + :label: Plane_wave_absorption + +We assume that we have an APML layer of thickness :math:`\delta` (measured +along :math:`x`) and that :math:`\sigma _{y}=\overline{\sigma }_{y}=0` +and :math:`c_{y}=c.` Using (:eq:`Plane_wave_absorption`), we determine +that the coefficient of reflection given by this layer is + +.. math:: + + \begin{aligned} + R_{\mathrm{APML}}\left( \theta \right) & = e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}+\overline{\sigma }_{x}c/c_{x}\right) \delta }e^{-\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}-\overline{\sigma }_{x}c/c_{x}\right) \delta },\nonumber + \\ + & = e^{-2\left( \sigma _{x}\cos \varphi /\varepsilon _{0}c_{x}\right) \delta }, + \end{aligned} + +which happens to be the same as the PML theoretical coefficient of +reflection if we assume :math:`c_{x}=c`. Hence, it follows that for +the purpose of wave absorption, the term :math:`\overline{\sigma }_{x}` +seems to be of no interest. However, although this conclusion is true +at the infinitesimal limit, it does not hold for the discretized counterpart. + +Discretization +~~~~~~~~~~~~~~ + +In the following we set :math:`\varepsilon_0 = \mu_0 = 1`. We discretize Eqs. (:eq:`PML_def_1`), (:eq:`PML_def_2`), (:eq:`PML_def_3`), and (:eq:`PML_def_4`) to obtain + +.. math:: + \frac{E_x|^{n+1}_{j+1/2,k,l}-E_x|^{n}_{j+1/2,k,l}}{\Delta t} + \sigma_y \frac{E_x|^{n+1}_{j+1/2,k,l}+E_x|^{n}_{j+1/2,k,l}}{2} = \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}}{\Delta y} + +.. math:: + \frac{E_y|^{n+1}_{j,k+1/2,l}-E_y|^{n}_{j,k+1/2,l}}{\Delta t} + \sigma_x \frac{E_y|^{n+1}_{j,k+1/2,l}+E_y|^{n}_{j,k+1/2,l}}{2} = - \frac{H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_x \frac{H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = - \frac{E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}}{\Delta x} + +.. math:: + \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}-H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{\Delta t} + \sigma^*_y \frac{H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l}+H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l}}{2} = \frac{E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}}{\Delta y} + +and this can be solved to obtain the following leapfrog integration equations + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = \left(\frac{1-\sigma_y \Delta t/2}{1+\sigma_y \Delta t/2}\right) E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t/\Delta y}{1+\sigma_y \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = \left(\frac{1-\sigma_x \Delta t/2}{1+\sigma_x \Delta t/2}\right) E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma_x \Delta t/2} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_x \Delta t/2}{1+\sigma^*_x \Delta t/2}\right) H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t/\Delta x}{1+\sigma^*_x \Delta t/2} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = \left(\frac{1-\sigma^*_y \Delta t/2}{1+\sigma^*_y \Delta t/2}\right) H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t/\Delta y}{1+\sigma^*_y \Delta t/2} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we account for higher order :math:`\Delta t` terms, a better approximation is given by + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +More generally, this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} E_x|^{n}_{j+1/2,k,l} + \frac{1-e^{-\sigma_y\Delta t}}{\sigma_y \Delta y}\frac{c_y}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} E_y|^{n}_{j,k+1/2,l} - \frac{1-e^{-\sigma_x\Delta t}}{\sigma_x \Delta x}\frac{c_x}{c} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{1-e^{-\sigma^*_x\Delta t}}{\sigma^*_x \Delta x}\frac{c^*_x}{c} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{1-e^{-\sigma^*_y\Delta t}}{\sigma^*_y \Delta y}\frac{c^*_y}{c} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +If we set + +.. math:: + + \begin{aligned} + c_x & = c \: e^{-\sigma_x\Delta t} \frac{\sigma_x \Delta t}{1-e^{-\sigma_x\Delta t}} \\ + c_y & = c \: e^{-\sigma_y\Delta t} \frac{\sigma_y \Delta t}{1-e^{-\sigma_y\Delta t}} \\ + c^*_x & = c \: e^{-\sigma^*_x\Delta t} \frac{\sigma^*_x \Delta t}{1-e^{-\sigma^*_x\Delta t}} \\ + c^*_y & = c \: e^{-\sigma^*_y\Delta t} \frac{\sigma^*_y \Delta t}{1-e^{-\sigma^*_y\Delta t}}\end{aligned} + +then this becomes + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = e^{-\sigma_y\Delta t} \left[ E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) \right] + \\ + E_y|^{n+1}_{j,k+1/2,l} & = e^{-\sigma_x\Delta t} \left[ E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) \right] + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_x\Delta t} \left[ H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) \right] + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = e^{-\sigma^*_y\Delta t} \left[ H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) \right] + \end{aligned} + +When the generalized conductivities are zero, the update equations are + +.. math:: + + \begin{aligned} + E_x|^{n+1}_{j+1/2,k,l} & = E_x|^{n}_{j+1/2,k,l} + \frac{\Delta t}{\Delta y} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j+1/2,k-1/2,l}\right) + \\ + E_y|^{n+1}_{j,k+1/2,l} & = E_y|^{n}_{j,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(H_z|^{n+1/2}_{j+1/2,k+1/2,l}-H_z|^{n+1/2}_{j-1/2,k+1/2,l}\right) + \\ + H_{zx}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zx}|^{n+1/2}_{j+1/2,k+1/2,l} - \frac{\Delta t}{\Delta x} \left(E_y|^{n+1}_{j+1,k+1/2,l}-E_y|^{n+1}_{j,k+1/2,l}\right) + \\ + H_{zy}|^{n+3/2}_{j+1/2,k+1/2,l} & = H_{zy}|^{n+1/2}_{j+1/2,k+1/2,l} + \frac{\Delta t}{\Delta y} \left(E_x|^{n+1}_{j+1/2,k+1,l}-E_x|^{n+1}_{j+1/2,k,l}\right) + \end{aligned} + +as expected. + +.. _theory-bc-pec: + +Perfect Electrical Conductor +---------------------------- + +This boundary can be used to model a dielectric or metallic surface. +For the electromagnetic solve, at PEC, the tangential electric field and the normal magnetic +field are set to 0. In the guard-cell region, the tangential electric field is set equal and +opposite to the respective field component in the mirror location across the PEC +boundary, and the normal electric field is set equal to the field component in the +mirror location in the domain across the PEC boundary. Similarly, the tangential +(and normal) magnetic field components are set equal (and opposite) to the respective +magnetic field components in the mirror locations across the PEC boundary. + +The PEC boundary condition also impacts the deposition of charge and current density. +On the boundary the charge density and parallel current density is set to zero. If +a reflecting boundary condition is used for the particles, density overlapping +with the PEC will be reflected back into the domain (for both charge and current +density). If absorbing boundaries are used, an image charge (equal weight but +opposite charge) is considered in the mirror location accross the boundary, and +the density from that charge is also deposited in the simulation domain. :numref:`fig_PEC_boundary_deposition` +shows the effect of this. The left boundary is absorbing while +the right boundary is reflecting. + +.. _fig_PEC_boundary_deposition: + +.. figure:: https://user-images.githubusercontent.com/40245517/221491318-b0a2bcbc-b04f-4b8c-8ec5-e9c92e55ee53.png + :alt: Plot of PEC boundary current deposition showing current vs position along the ``x``-axis. + :width: 100% + + PEC boundary current deposition along the ``x``-axis. The left boundary is absorbing while the right boundary is reflecting. + +.. bibliography:: + :keyprefix: bc- diff --git a/Docs/source/theory/cold_fluid_model.rst b/Docs/source/theory/cold_fluid_model.rst index 971ebdba062..813a568c540 100644 --- a/Docs/source/theory/cold_fluid_model.rst +++ b/Docs/source/theory/cold_fluid_model.rst @@ -42,15 +42,19 @@ where the particle quantities are calculated by the PIC algorithm. Implementation details ---------------------- +.. _fig_fluid_loop: + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 + :alt: Figure showing fluid Loop embedded within the overall PIC loop. + + Fluid Loop embedded within the overall PIC loop. + The fluid timeloop is embedded inside the standard PIC timeloop and consists of the following steps: 1. Higuera and Cary push of the momentum 2. Non-inertial (momentum source) terms (only in cylindrical geometry) 3. boundary conditions and MPI Communications 4. MUSCL -scheme for advection terms 5. Current and Charge Deposition. The figure here gives +scheme for advection terms 5. Current and Charge Deposition. :numref:`fig_fluid_loop` gives a visual representation of these steps, and we describe each of these in more detail. -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/69021085/dcbcc0e4-7899-43e4-b580-f57eb359b457 - :alt: Fluid Loop embedded within the overall PIC loop. - Step 0: **Preparation** Before the fluid loop begins, it is assumed that the program is in the state where fields :math:`\mathbf{E}` and :math:`\mathbf{B}` are available integer timestep. The @@ -59,14 +63,14 @@ Step 0: **Preparation** on a nodal grid and at half-integer timestep. Step 1: **Higuera and Cary Push** - The time staggering of the fields is used by the momentum source term, which is solved with a the - Higeura and Cary push (Higuera et al, 2017). We do not adopt spatial + The time staggering of the fields is used by the momentum source term, which is solved with a + Higuera and Cary push :cite:p:`cfm-HigueraPOP2017`. We do not adopt spatial grid staggering, all discretized fluid quantities exist on the nodal grid. External fields can be included at this step. Step 2: **Non-inertial Terms** In RZ, the divergence of the flux terms has additional non-zero elements outside of the - derivatives. These terms are Strang split and are time integrated via equation 2.18 from (Osher et al, 1988), + derivatives. These terms are Strang split and are time integrated via equation 2.18 from :cite:t:`cfm-ShuJCP1988`, which is the SSP-RK3 integrator. Step 3: **Boundary Conditions and Communications** @@ -79,7 +83,7 @@ Step 4: **Advective Push** limiting is used. We further simplify the conservative equations in terms of primitive variables, :math:`\{ N, U_x, U_y, U_z \}`. Which we found to be more stable than conservative variables for the MUSCL reconstruction. Details of - the scheme can be found here (Van Leer et al, 1984). + the scheme can be found in :cite:t:`cfm-VanLeerBookChapter1997`. Step 5: **Current and Charge Deposition** Once this series of steps is complete and the fluids have been evolved by an entire @@ -95,7 +99,7 @@ Step 5: **Current and Charge Deposition** Mesh refinement is not supported for the fluids. - The implemented MUSCL scheme has a simplifed slope averaging, see the extended writeup for details. + The implemented MUSCL scheme has a simplified slope averaging, see the extended writeup for details. More details on the precise implementation are available here, `WarpX_Cold_Rel_Fluids.pdf`_. .. _WarpX_Cold_Rel_Fluids.pdf: https://github.com/ECP-WarpX/WarpX/files/12886437/WarpX_Cold_Rel_Fluids.pdf @@ -104,33 +108,5 @@ Step 5: **Current and Charge Deposition** If using the fluid model with the Kinetic-Fluid Hybrid model or the electrostatic solver, there is a known issue that the fluids deposit at a half-timestep offset in the charge-density. -.. raw:: html - -
- -.. raw:: html - -
- -Higuera, Adam V., and John R. Cary. "Structure-preserving second-order integration of relativistic charged particle trajectories in electromagnetic fields." Physics of Plasmas 24.5 (2017). - -.. raw:: html - -
- -.. raw:: html - -
- -Osher, Stanley, and Chi-Wang Shu. "Efficient implementation of essentially non-oscillatory shock-capturing schemes." J. Comput. Phys 77.2 (1988): 439-471. - - -.. raw:: html - -
- -.. raw:: html - -
- -Van Leer, Bram. "On the relation between the upwind-differencing schemes of Godunov, Engquist–Osher and Roe." SIAM Journal on Scientific and statistical Computing 5.1 (1984): 1-20. +.. bibliography:: + :keyprefix: cfm- diff --git a/Docs/source/theory/collisions.rst b/Docs/source/theory/collisions.rst index 3f24cd8f331..52e36521125 100644 --- a/Docs/source/theory/collisions.rst +++ b/Docs/source/theory/collisions.rst @@ -3,18 +3,36 @@ Collisions ========== -Monte Carlo Collisions ----------------------- - -The Monte Carlo collisions (MCC) module can be found in *Source/Particles/Collisions/BackgroundMCC/*. -Several types of collisions between simulation particles and a neutral background gas are supported including elastic scattering, back scattering, charge exchange, excitation collisions and impact ionization. -An instance of the class :cpp:class:`MCCProcess` is created for each type of collision included in a simulation. This class saves information about the type of collision and the collision cross-section as a function of energy. - -The so-called null collision strategy is used in order to minimize the computational burden of the MCC module. -This strategy is standard in PIC-MCC and a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. -In short the maximum collision probability is found over a sensible range of energies and is used to pre-select the appropriate number of macroparticles for collision consideration. Only these pre-selected particles are then individually considered for a collision based on their energy and the cross-sections of all the different collisional processes included. - -The MCC implementation assumes that the background neutral particles are **thermal**, and are moving at non-relativistic velocities in the lab frame. For each simulation particle considered for a collision, a velocity vector for a neutral particle is randomly chosen. The particle velocity is then boosted to the stationary frame of the neutral through a Galilean transformation. The energy of the collision is calculated using the particle utility function, ``ParticleUtils::getCollisionEnergy()``, as +WarpX includes several different models to capture collisional processes +including collisions between kinetic particles (Coulomb collisions, DSMC, +nuclear fusion) as well as collisions between kinetic particles and a fixed +(i.e. non-evolving) background species (MCC, background stopping). + +.. _theory-collisions-mcc: + +Background Monte Carlo Collisions (MCC) +--------------------------------------- + +Several types of collisions between simulation particles and a neutral +background gas are supported including elastic scattering, back scattering, +charge exchange, excitation collisions and impact ionization. + +The so-called null collision strategy is used in order to minimize the +computational burden of the MCC module. This strategy is standard in PIC-MCC and +a detailed description can be found elsewhere, for example in :cite:t:`b-Birdsall1991`. +In short the maximum collision probability is found over a sensible range of +energies and is used to pre-select the appropriate number of macroparticles for +collision consideration. Only these pre-selected particles are then individually +considered for a collision based on their energy and the cross-sections of all +the different collisional processes included. + +The MCC implementation assumes that the background neutral particles are **thermal**, +and are moving at non-relativistic velocities in the lab frame. For each +simulation particle considered for a collision, a velocity vector for a neutral +particle is randomly chosen given the user specified neutral temperature. The +particle velocity is then boosted to the stationary frame of the neutral through +a Galilean transformation. The energy of the collision is calculated using the +particle utility function, ``ParticleUtils::getCollisionEnergy()``, as .. math:: @@ -23,19 +41,55 @@ The MCC implementation assumes that the background neutral particles are **therm &= \frac{2Mmu^2}{M + m + \sqrt{M^2+m^2+2\gamma mM}}\frac{1}{\gamma + 1} \end{aligned} -where :math:`u` is the speed of the particle as tracked in WarpX (i.e. :math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and :math:`M` are the rest masses of the simulation and background species, respectively. The Lorentz factor is defined in the usual way, :math:`\gamma = \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above expression clearly reduces to the classical equation :math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections for all scattering processes are evaluated at the energy as calculated above. +where :math:`u` is the speed of the particle as tracked in WarpX (i.e. +:math:`u = \gamma v` with :math:`v` the particle speed), while :math:`m` and +:math:`M` are the rest masses of the simulation and background species, +respectively. The Lorentz factor is defined in the usual way, +:math:`\gamma \def \sqrt{1 + u^2/c^2}`. Note that if :math:`\gamma\to1` the above +expression reduces to the classical equation +:math:`E_{coll} = \frac{1}{2}\frac{Mm}{M+m} u^2`. The collision cross-sections +for all scattering processes are evaluated at the energy as calculated above. Once a particle is selected for a specific collision process, that process determines how the particle is scattered as outlined below. +.. _theory-collisions-dsmc: + +Direct Simulation Monte Carlo (DSMC) +------------------------------------ + +The algorithm by which binary collisions are treated is outlined below. The +description assumes collisions between different species. + +1. Particles from both species are sorted by grid-cells. +2. The order of the particles in each cell is shuffled. +3. Within each cell, particles are paired to form collision partners. Particles + of the species with fewer members in a given cell is split in half so that + each particle has exactly one partner of the other species. +4. Each collision pair is considered for a collision using the same logic as in + the MCC description above. +5. Particles that are chosen for collision are scattered according to the + selected collision process. + +Scattering processes +-------------------- + Charge exchange ^^^^^^^^^^^^^^^ -This is the simplest scattering process. Under charge exchange the simulation particle's velocity is simply replaced with the sampled velocity of the neutral particle. Charge exchange usually has a cooling effect on the ions in the simulation by exchanging energetic ions for lower energy neutrals. +This process can occur when an ion and neutral (of the same species) collide +and results in the exchange of an electron. The ion velocity is simply replaced +with the neutral velocity and vice-versa. Elastic scattering ^^^^^^^^^^^^^^^^^^ -This scattering process as well as the ones below that relate to it, are all performed in the center-of-momentum (COM) frame. Designating the COM velocity of the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, the transformation from lab frame to COM frame is done with a general Lorentz boost (see function ``ParticleUtils::doLorentzTransform()``): +The ``elastic`` option uses isotropic scattering, i.e., with a differential +cross section that is independent of angle. +This scattering process as well as the ones below that relate to it, are all +performed in the center-of-momentum (COM) frame. Designating the COM velocity of +the particle as :math:`\vec{u}_c` and its labframe velocity as :math:`\vec{u}_l`, +the transformation from lab frame to COM frame is done with a general Lorentz +boost (see function ``ParticleUtils::doLorentzTransform()``): .. math:: \begin{bmatrix} @@ -74,6 +128,12 @@ Excitation The process is also the same as for elastic scattering except the excitation energy cost is subtracted from the particle energy. This is done by reducing the velocity before a scattering angle is chosen. +Benchmarks +---------- + +See the :ref:`MCC example ` for a benchmark of the MCC +implementation against literature results. + Particle cooling due to elastic collisions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Docs/source/theory/input_output.rst b/Docs/source/theory/input_output.rst index 75427ad9c6d..21a5f5c8d2c 100644 --- a/Docs/source/theory/input_output.rst +++ b/Docs/source/theory/input_output.rst @@ -3,7 +3,7 @@ Inputs and Outputs ================== -Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings (Vay 2008; Cowan et al. 2013). +Initialization of the plasma columns and drivers (laser or particle beam) is performed via the specification of multidimensional functions that describe the initial state with, if needed, a time dependence, or from reconstruction of distributions based on experimental data. Care is needed when initializing quantities in parallel to avoid double counting and ensure smoothness of the distributions at the interface of computational domains. When the sum of the initial distributions of charged particles is not charge neutral, initial fields are computed using generally a static approximation with Poisson solves accompanied by proper relativistic scalings :cite:p:`io-Vaypop2008, io-CowanPRSTAB13`. Outputs include dumps of particle and field quantities at regular intervals, histories of particle distributions moments, spectra, etc, and plots of the various quantities. In parallel simulations, the diagnostic subroutines need to handle additional complexity from the domain decomposition, as well as large amount of data that may necessitate data reduction in some form before saving to disk. @@ -12,16 +12,17 @@ Simulations in a Lorentz boosted frame require additional considerations, as des Inputs and outputs in a boosted frame simulation ------------------------------------------------ -.. _Fig_inputoutput: +.. _fig_inputoutput: + .. figure:: Input_output.png :alt: (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. - :width: 120mm + :width: 100% (top) Snapshot of a particle beam showing “frozen" (grey spheres) and “active" (colored spheres) macroparticles traversing the injection plane (red rectangle). (bottom) Snapshot of the beam macroparticles (colored spheres) passing through the background of electrons (dark brown streamlines) and the diagnostic stations (red rectangles). The electrons, the injection plane and the diagnostic stations are fixed in the laboratory plane, and are thus counter-propagating to the beam in a boosted frame. The input and output data are often known from, or compared to, experimental data. Thus, calculating in a frame other than the laboratory entails transformations of the data between the calculation frame and the laboratory -frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp (Grote et al. 2005) to handle the input and output of data between the frame of calculation and the laboratory frame (Vay et al. 2011). Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. +frame. This section describes the procedures that have been implemented in the Particle-In-Cell framework Warp :cite:p:`io-Warp` to handle the input and output of data between the frame of calculation and the laboratory frame :cite:p:`io-Vaypop2011`. Simultaneity of events between two frames is valid only for a plane that is perpendicular to the relative motion of the frame. As a result, the input/output processes involve the input of data (particles or fields) through a plane, as well as output through a series of planes, all of which are perpendicular to the direction of the relative velocity between the frame of calculation and the other frame of choice. Input in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -29,17 +30,17 @@ Input in a boosted frame simulation Particles - ^^^^^^^^^^^^ -Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability (Vay 2007). In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): +Particles are launched through a plane using a technique that is generic and applies to Lorentz boosted frame simulations in general, including plasma acceleration, and is illustrated using the case of a positively charged particle beam propagating through a background of cold electrons in an assumed continuous transverse focusing system, leading to a well-known growing transverse “electron cloud” instability :cite:p:`io-Vayprl07`. In the laboratory frame, the electron background is initially at rest and a moving window is used to follow the beam progression. Traditionally, the beam macroparticles are initialized all at once in the window, while background electron macroparticles are created continuously in front of the beam on a plane that is perpendicular to the beam velocity. In a frame moving at some fraction of the beam velocity in the laboratory frame, the beam initial conditions at a given time in the calculation frame are generally unknown and one must initialize the beam differently. However, it can be taken advantage of the fact that the beam initial conditions are often known for a given plane in the laboratory, either directly, or via simple calculation or projection from the conditions at a given time in the labortory frame. Given the position and velocity :math:`\{x,y,z,v_x,v_y,v_z\}` for each beam macroparticle at time :math:`t=0` for a beam moving at the average velocity :math:`v_b=\beta_b c` (where :math:`c` is the speed of light) in the laboratory, and using the standard synchronization (:math:`z=z'=0` at :math:`t=t'=0`) between the laboratory and the calculation frames, the procedure for transforming the beam quantities for injection in a boosted frame moving at velocity :math:`\beta c` in the laboratory is as follows (the superscript :math:`'` relates to quantities known in the boosted frame while the superscript :math:`^*` relates to quantities that are know at a given longitudinal position :math:`z^*` but different times of arrival): #. project positions at :math:`z^*=0` assuming ballistic propagation .. math:: \begin{aligned} - t^* &=& \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ - x^* &=& x-v_x t^* \label{Eq:x*}\\ - y^* &=& y-v_y t^* \label{Eq:y*}\\ - z^* &=& 0 \label{Eq:z*}\end{aligned} + t^* &= \left(z-\bar{z}\right)/v_z \label{Eq:t*}\\ + x^* &= x-v_x t^* \label{Eq:x*}\\ + y^* &= y-v_y t^* \label{Eq:y*}\\ + z^* &= 0 \label{Eq:z*}\end{aligned} the velocity components being left unchanged, @@ -48,13 +49,13 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - t'^* &=& -\gamma t^* \label{Eq:tp*}\\ - x'^* &=& x^* \label{Eq:xp*}\\ - y'^* &=& y^* \label{Eq:yp*}\\ - z'^* &=& \gamma\beta c t^* \label{Eq:zp*}\\ - v'^*_x&=&\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ - v'^*_y&=&\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ - v'^*_z&=&\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} + t'^* &= -\gamma t^* \label{Eq:tp*}\\ + x'^* &= x^* \label{Eq:xp*}\\ + y'^* &= y^* \label{Eq:yp*}\\ + z'^* &= \gamma\beta c t^* \label{Eq:zp*}\\ + v'^*_x&=\frac{v_x^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vxp*}\\ + v'^*_y&=\frac{v_y^*}{\gamma\left(1-\beta \beta_b\right)} \label{Eq:vyp*}\\ + v'^*_z&=\frac{v_z^*-\beta c}{1-\beta \beta_b} \label{Eq:vzp*}\end{aligned} where :math:`\gamma=1/\sqrt{1-\beta^2}`. With the knowledge of the time at which each beam macroparticle crosses the plane into consideration, one can inject each beam macroparticle in the simulation at the appropriate location and time. @@ -63,11 +64,11 @@ Particles are launched through a plane using a technique that is generic and app .. math:: \begin{aligned} - z' &=& z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} + z' &= z'^*-\bar{v}'^*_z t'^* \label{Eq:zp}\end{aligned} - This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other (Vay 2008)) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. + This additional step is needed for setting the electrostatic or electromagnetic fields at the plane of injection. In a Particle-In-Cell code, the three-dimensional fields are calculated by solving the Maxwell equations (or static approximation like Poisson, Darwin or other :cite:p:`io-Vaypop2008`) on a grid on which the source term is obtained from the macroparticles distribution. This requires generation of a three-dimensional representation of the beam distribution of macroparticles at a given time before they cross the injection plane at :math:`z'^*`. This is accomplished by expanding the beam distribution longitudinally such that all macroparticles (so far known at different times of arrival at the injection plane) are synchronized to the same time in the boosted frame. To keep the beam shape constant, the particles are “frozen” until they cross that plane: the three velocity components and the two position components perpendicular to the boosted frame velocity are kept constant, while the remaining position component is advanced at the average beam velocity. As particles cross the plane of injection, they become regular “active” particles with full 6-D dynamics. -Figure :numref:`Fig_inputoutput` (top) shows a snapshot of a beam that has passed partly through the injection plane. As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. +A snapshot of a beam that has passed partly through the injection plane in shown in :numref:`fig_inputoutput` (top). As the frozen beam macroparticles pass through the injection plane (which moves opposite to the beam in the boosted frame), they are converted to “active" macroparticles. The charge or current density is accumulated from the active and the frozen particles, thus ensuring that the fields at the plane of injection are consistent. Laser - ^^^^^^^^ @@ -89,93 +90,28 @@ If, for convenience, the injection plane is moving at constant velocity :math:`\ .. math:: \begin{aligned} - E_\perp\left(x,y,t\right)&=&\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\nonumber \\ - &\times& \sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right].\end{aligned} + E_\perp\left(x,y,t\right)&=\left(1-\beta_s\right)E_0 f\left(x,y,t\right)\sin\left[\left(1-\beta_s\right)\omega t+\phi\left(x,y,\omega\right)\right] + \end{aligned} The injection of a laser of frequency :math:`\omega` is considered for a simulation using a boosted frame moving at :math:`\beta c` with respect to the laboratory. Assuming that the laser is injected at a plane that is fixed in the laboratory, and thus moving at :math:`\beta_s=-\beta` in the boosted frame, the injection in the boosted frame is given by .. math:: \begin{aligned} - E_\perp\left(x',y',t'\right)&=&\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\nonumber \\ - &\times&\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right]\\ - &=&\left(E_0/\gamma\right) f\left(x',y',t'\right) \nonumber\\ - &\times&\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right]\end{aligned} + E_\perp\left(x',y',t'\right)&=\left(1-\beta_s\right)E'_0 f\left(x',y',t'\right)\sin\left[\left(1-\beta_s\right)\omega' t'+\phi\left(x',y',\omega'\right)\right] + \\ + &=\left(E_0/\gamma\right) f\left(x',y',t'\right)\sin\left[\omega t'/\gamma+\phi\left(x',y',\omega'\right)\right] + \end{aligned} since :math:`E'_0/E_0=\omega'/\omega=1/\left(1+\beta\right)\gamma`. -The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme (Esirkepov 2001). +The electric field is then converted into currents that get injected via a 2D array of macro-particles, with one positive and one dual negative macro-particle for each array cell in the plane of injection, whose weights and motion are governed by :math:`E_\perp\left(x',y',t'\right)`. Injecting using this dual array of macroparticles offers the advantage of automatically including the longitudinal component that arises from emitting into a boosted frame, and to automatically verify the discrete Gauss’ law thanks to using charge conserving (e.g. Esirkepov) current deposition scheme :cite:p:`io-Esirkepovcpc01`. Output in a boosted frame simulation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some quantities, e.g. charge or dimensions perpendicular to the boost velocity, are Lorentz invariant. -Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on Fig. :numref:`Fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Grote, D P, A Friedman, J.-L. Vay, and I Haber. 2005. “The Warp Code: Modeling High Intensity Ion Beams.” In *Aip Conference Proceedings*, 55–58. 749. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L. 2007. “Noninvariance of Space- and Time-Scale Ranges Under A Lorentz Transformation and the Implications for the Study of Relativistic Interactions.” *Physical Review Letters* 98 (13): 130405/1–4. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J -L., C G R Geddes, E Esarey, C B Schroeder, W P Leemans, E Cormier-Michel, and D P Grote. 2011. “Modeling of 10 Gev-1 Tev Laser-Plasma Accelerators Using Lorentz Boosted Simulations.” *Physics of Plasmas* 18 (12). https://doi.org/10.1063/1.3663841. - -.. raw:: html - -
- -.. raw:: html +Those quantities are thus readily available from standard diagnostics in the boosted frame calculations. Quantities that do not fall in this category are recorded at a number of regularly spaced “stations", immobile in the laboratory frame, at a succession of time intervals to record data history, or averaged over time. A visual example is given on :numref:`fig_inputoutput` (bottom). Since the space-time locations of the diagnostic grids in the laboratory frame generally do not coincide with the space-time positions of the macroparticles and grid nodes used for the calculation in a boosted frame, some interpolation is performed at runtime during the data collection process. As a complement or an alternative, selected particle or field quantities can be dumped at regular intervals and quantities are reconstructed in the laboratory frame during a post-processing phase. The choice of the methods depends on the requirements of the diagnostics and particular implementations. -
+.. bibliography:: + :keyprefix: io- diff --git a/Docs/source/theory/intro.rst b/Docs/source/theory/intro.rst index cbf56c919df..2101d9d81cb 100644 --- a/Docs/source/theory/intro.rst +++ b/Docs/source/theory/intro.rst @@ -8,16 +8,16 @@ Introduction Plasma laser-driven (top) and charged-particles-driven (bottom) acceleration (rendering from 3-D Particle-In-Cell simulations). A laser beam (red and blue disks in top picture) or a charged particle beam (red dots in bottom picture) propagating (from left to right) through an under-dense plasma (not represented) displaces electrons, creating a plasma wakefield that supports very high electric fields (pale blue and yellow). These electric fields, which can be orders of magnitude larger than with conventional techniques, can be used to accelerate a short charged particle beam (white) to high-energy over a very short distance. -Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`Tsung2006,Geddes2008,Geddes2009,Geddes2010,Huang2009`, with -accurate modeling of wake formation, electron self-trapping and acceleration requiring fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`Leemans2014,Blumenfeld2007,Bulanov2014,Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. +Computer simulations have had a profound impact on the design and understanding of past and present plasma acceleration experiments :cite:p:`i-Tsungpop06,i-Geddesjp08,i-Geddesscidac09,i-Geddespac09,i-Huangscidac09`. Accurate modeling of wake formation, electron self-trapping and acceleration require fully kinetic methods (usually Particle-In-Cell) using large computational resources due to the wide range of space and time scales involved. Numerical modeling complements and guides the design and analysis of advanced accelerators, and can reduce development costs significantly. Despite the major recent experimental successes :cite:p:`i-LeemansPRL2014,i-Blumenfeld2007,i-BulanovSV2014,i-Steinke2016`, the various advanced acceleration concepts need significant progress to fulfill their potential. To this end, large-scale simulations will continue to be a key component toward reaching a detailed understanding of the complex interrelated physics phenomena at play. For such simulations, the most popular algorithm is the Particle-In-Cell (or PIC) technique, which represents electromagnetic fields on a grid and particles by a sample of macroparticles. However, these simulations are extremely computationally intensive, due to the need to resolve the evolution of a driver (laser or particle beam) and an accelerated beam into a structure that is orders of magnitude longer and wider than the accelerated beam. -Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`Sprangle1990,Antonsen1992,Krall1993,Mora1997,Huang2006`, -ponderomotive guiding center (PGC) models :cite:p:`Antonsen1992,Krall1993,Huang2006,Benedetti2010,Cowan2011`, simulation in an optimal Lorentz boosted frame :cite:p:`Vay2007,Bruhwiler2009,Vay2009a,Vay2009b,Vay2010,Martins2010a,Martins2010b,Martins2010c,Vay2011a,Vay2011b,Vay2011c,Yu2016`, -expanding the fields into a truncated series of azimuthal modes :cite:p:`Godfrey1985,Lifschitz2009,Davidson2015,Lehe2016,Andriyash2016`, fluid approximation :cite:p:`Krall1993,Shadwick2009,Benedetti2010` and scaled parameters :cite:p:`CormierMichel2009`. +Various techniques or reduced models have been developed to allow multidimensional simulations at manageable computational costs: quasistatic approximation :cite:p:`i-Sprangleprl90,i-Antonsenprl1992,i-Krallpre1993,i-Morapop1997,i-Quickpic`, +ponderomotive guiding center (PGC) models :cite:p:`i-Antonsenprl1992,i-Krallpre1993,i-Quickpic,i-Benedettiaac2010,i-Cowanjcp11`, simulation in an optimal Lorentz boosted frame :cite:p:`i-Vayprl07,i-Bruhwileraac08,i-Vayscidac09,i-Vaypac09,i-Martinspac09,i-VayAAC2010,i-Martinsnaturephysics10,i-Martinspop10,i-Martinscpc10,i-Vayjcp2011,i-VayPOPL2011,i-Vaypop2011,i-Yu2016`, +expanding the fields into a truncated series of azimuthal modes :cite:p:`i-godfrey1985iprop,i-LifschitzJCP2009,i-DavidsonJCP2015,i-Lehe2016,i-AndriyashPoP2016`, fluid approximation :cite:p:`i-Krallpre1993,i-Shadwickpop09,i-Benedettiaac2010` and scaled parameters :cite:p:`i-Cormieraac08,i-Geddespac09`. .. bibliography:: + :keyprefix: i- diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 37e4955d665..3fb8320c531 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -18,8 +18,8 @@ has to resolve the electron Debye length and CFL-condition based on the speed of light. Many authors have described variations of the kinetic ion & fluid electron model, -generally referred to as particle-fluid hybrid or just hybrid-PIC models. The implementation -in WarpX follows the outline from :cite:t:`c-winske2022hybrid`. +generally referred to as particle-fluid hybrid or just hybrid-PIC models. The +implementation in WarpX is described in detail in :cite:t:`kfhm-Groenewald2023`. This description follows mostly from that reference. Model @@ -29,31 +29,57 @@ The basic justification for the hybrid model is that the system to which it is applied is dominated by ion kinetics, with ions moving much slower than electrons and photons. In this scenario two critical approximations can be made, namely, neutrality (:math:`n_e=n_i`) and the Maxwell-Ampere equation can be simplified by -neglecting the displacement current term :cite:p:`c-NIELSON1976`, giving, +neglecting the displacement current term :cite:p:`kfhm-Nielson1976`, giving, .. math:: \mu_0\vec{J} = \vec{\nabla}\times\vec{B}, -where :math:`\vec{J} = \vec{J}_i - \vec{J}_e` is the total electrical current, i.e. -the sum of electron and ion currents. Since ions are treated in the regular -PIC manner, the ion current, :math:`\vec{J}_i`, is known during a simulation. Therefore, +where :math:`\vec{J} = \sum_{s\neq e}\vec{J}_s + \vec{J}_e + \vec{J}_{ext}` is the total electrical current, +i.e. the sum of electron and ion currents as well as any external current (not captured through plasma +particles). Since ions are treated in the regular +PIC manner, the ion current, :math:`\sum_{s\neq e}\vec{J}_s`, is known during a simulation. Therefore, given the magnetic field, the electron current can be calculated. -If we now further assume electrons are inertialess, the electron momentum -equation yields, +The electron momentum transport equation (obtained from multiplying the Vlasov equation by mass and +integrating over velocity), also called the generalized Ohm's law, is given by: .. math:: - \frac{d(n_em_e\vec{V}_e)}{dt} = 0 = -en_e\vec{E}-\vec{J}_e\times\vec{B}-\nabla\cdot\vec{P}_e+en_e\vec{\eta}\cdot\vec{J}, + en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e -where :math:`\vec{V_e}=\vec{J}_e/(en_e)`, :math:`\vec{P}_e` is the electron pressure -tensor and :math:`\vec{\eta}` is the resistivity tensor. An expression for the electric field -(generalized Ohm's law) can be obtained from the above as: +where :math:`\vec{U}_e = \vec{J}_e/(en_e)` is the electron fluid velocity, +:math:`{\overleftrightarrow P}_e` is the electron pressure tensor and +:math:`\vec{R}_e` is the drag force due to collisions between electrons and ions. +Applying the above momentum equation to the Maxwell-Faraday equation (:math:`\frac{\partial\vec{B}}{\partial t} = -\nabla\times\vec{E}`) +and substituting in :math:`\vec{J}` calculated from the Maxwell-Ampere equation, gives, .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla\cdot\vec{P}_e \right)+\vec{\eta}\cdot\vec{J}. + \frac{\partial\vec{J}_e}{\partial t} = -\frac{1}{\mu_0}\nabla\times\left(\nabla\times\vec{E}\right) - \frac{\partial\vec{J}_{ext}}{\partial t} - \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t}. + +Plugging this back into the generalized Ohm' law gives: + + .. math:: + + \left(en_e +\frac{m}{e\mu_0}\nabla\times\nabla\times\right)\vec{E} =& + - \frac{m}{e}\left( \frac{\partial\vec{J}_{ext}}{\partial t} + \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t} \right) \\ + &+ \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. + +If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above equation simplifies to, + + .. math:: + + en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. + +Making the further simplifying assumptions that the electron pressure is isotropic and that +the electron drag term can be written as a simple resistance +i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +Ohm's law: + + .. math:: + + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial @@ -90,8 +116,7 @@ be interpolated to the correct time, using :math:`\vec{J}_i^n = 1/2(\vec{J}_i^{n The electron pressure is simply calculated using :math:`\rho^n` and the B-field is also already known at the correct time since it was calculated for :math:`t=t_n` at the end of the last step. Once :math:`\vec{E}^n` is calculated, it is used to push :math:`\vec{B}^n` forward in time -(using the Maxwell-Faraday equation, i.e. the same as in the regular PIC routine with ``WarpX::EvolveB()``) -to :math:`\vec{B}^{n+1/2}`. +(using the Maxwell-Faraday equation) to :math:`\vec{B}^{n+1/2}`. Second half step """""""""""""""" @@ -108,7 +133,7 @@ Extrapolation step Obtaining the E-field at timestep :math:`t=t_{n+1}` is a well documented issue for the hybrid model. Currently the approach in WarpX is to simply extrapolate -:math:`\vec{J}_i` foward in time, using +:math:`\vec{J}_i` forward in time, using .. math:: @@ -121,10 +146,11 @@ Sub-stepping ^^^^^^^^^^^^ It is also well known that hybrid PIC routines require the B-field to be -updated with a smaller timestep than needed for the particles. The update steps -as outlined above are therefore wrapped in loops that enable the B-field to be -sub-stepped. The exact number of sub-steps used can be specified by the user -through a runtime simulation parameter (see :ref:`input parameters section `). +updated with a smaller timestep than needed for the particles. A 4th order +Runge-Kutta scheme is used to update the B-field. The RK scheme is repeated a +number of times during each half-step outlined above. The number of sub-steps +used can be specified by the user through a runtime simulation parameter +(see :ref:`input parameters section `). .. _theory-hybrid-model-elec-temp: @@ -142,4 +168,4 @@ The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. .. bibliography:: - :keyprefix: c- + :keyprefix: kfhm- diff --git a/Docs/source/theory/pic.rst b/Docs/source/theory/pic.rst new file mode 100644 index 00000000000..820cdba50e6 --- /dev/null +++ b/Docs/source/theory/pic.rst @@ -0,0 +1,695 @@ +.. _theory-pic: + +Particle-in-Cell Method +======================= + +.. _fig-pic: + +.. figure:: PIC.png + :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + + The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). + +In the *electromagnetic particle-in-cell method* :cite:p:`pt-Birdsalllangdon,pt-HockneyEastwoodBook`, +the electromagnetic fields are solved on a grid, usually using Maxwell’s +equations + +.. math:: + \frac{\mathbf{\partial B}}{\partial t} = -\nabla\times\mathbf{E} + :label: Faraday-1 + +.. math:: + \frac{\mathbf{\partial E}}{\partial t} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-1 + +.. math:: + \nabla\cdot\mathbf{E} = \rho + :label: Gauss-1 + +.. math:: + \nabla\cdot\mathbf{B} = 0 + :label: divb-1 + +given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and +:math:`\mathbf{B}` are the electric and magnetic field components, and +:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The +charged particles are advanced in time using the Newton-Lorentz equations +of motion + +.. math:: + \frac{d\mathbf{x}}{dt} = \mathbf{v}, + :label: Lorentz_x-1 + +.. math:: + \frac{d\left(\gamma\mathbf{v}\right)}{dt} = \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right), + :label: Lorentz_v-1 + +where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` +are respectively the mass, charge, position, velocity and relativistic +factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated +on the grid from the particles’ positions and velocities, while the +electric and magnetic field components are interpolated from the grid +to the particles’ positions for the velocity update. + +.. _theory-pic-push: + +Particle push +------------- + +A centered finite-difference discretization of the Newton-Lorentz +equations of motion is given by + +.. math:: + \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t} = \mathbf{v}^{i+1/2}, + :label: leapfrog_x + +.. math:: + \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t} = \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right). + :label: leapfrog_v + +In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be +expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. + +.. _theory-pic-push-boris: + +Boris relativistic velocity rotation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The solution proposed by Boris :cite:p:`pt-BorisICNSP70` is given by + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}} + :label: boris_v + +where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. + +The system (:eq:`leapfrog_v`, :eq:`boris_v`) is solved very +efficiently following Boris’ method, where the electric field push +is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the +velocity is updated using the following sequence: + +.. math:: + + \begin{aligned} + \mathbf{u^{-}} & = \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \\ + \mathbf{u'} & = \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t} + \\ + \mathbf{u}^{+} & = \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+\mathbf{t}^{2}) + \\ + \mathbf{u}^{i+1/2} & = \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i} + \end{aligned} + +where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where +:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. + +The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. + +.. _theory-pic-push-vay: + +Vay Lorentz-invariant formulation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It was shown in :cite:t:`pt-Vaypop2008` that the Boris formulation is +not Lorentz invariant and can lead to significant errors in the treatment +of relativistic dynamics. A Lorentz invariant formulation is obtained +by considering the following velocity average + +.. math:: + \mathbf{\bar{v}}^{i} = \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2}. + :label: new_v + +This gives a system that is solvable analytically (see :cite:t:`pt-Vaypop2008` +for a detailed derivation), giving the following velocity update: + +.. math:: + \mathbf{u^{*}} = \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right), + :label: pusher_gamma + +.. math:: + \mathbf{u}^{i+1/2} = \frac{\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}}{1+\mathbf{t}^{2}}, + :label: pusher_upr + +where + +.. math:: + + \begin{align} + \mathbf{t} & = \boldsymbol{\tau}/\gamma^{i+1/2}, + \\ + \boldsymbol{\tau} & = \left(q\Delta t/2m\right)\mathbf{B}^{i}, + \\ + \gamma^{i+1/2} & = \sqrt{\sigma+\sqrt{\sigma^{2}+\left(\boldsymbol{\tau}^{2}+w^{2}\right)}}, + \\ + w & = \mathbf{u^{*}}\cdot\boldsymbol{\tau}, + \\ + \sigma & = \left(\gamma'^{2}-\boldsymbol{\tau}^{2}\right)/2, + \\ + \gamma' & = \sqrt{1+(\mathbf{u}^{*}/c)^{2}}. + \end{align} + +This Lorentz invariant formulation +is particularly well suited for the modeling of ultra-relativistic +charged particle beams, where the accurate account of the cancellation +of the self-generated electric and magnetic fields is essential, as +shown in :cite:t:`pt-Vaypop2008`. + +.. _theory-pic-mwsolve: + +Field solve +----------- + +Various methods are available for solving Maxwell’s equations on a +grid, based on finite-differences, finite-volume, finite-element, +spectral, or other discretization techniques that apply most commonly +on single structured or unstructured meshes and less commonly on multiblock +multiresolution grid structures. In this chapter, we summarize the widespread +second order finite-difference time-domain (FDTD) algorithm, its extension +to non-standard finite-differences as well as the pseudo-spectral +analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) +algorithms. Extension to multiresolution (or mesh refinement) PIC +is described in, e.g., :cite:t:`pt-VayCSD12,pt-Vaycpc04`. + +.. _fig_yee_grid: + +.. figure:: Yee_grid.png + :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + + (left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. + +.. _theory-pic-mwsolve-fdtd: + +Finite-Difference Time-Domain (FDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The most popular algorithm for electromagnetic PIC codes is the Finite-Difference +Time-Domain (or FDTD) solver + +.. math:: + D_{t}\mathbf{B} = -\nabla\times\mathbf{E} + :label: Faraday-2 + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere-2 + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss-2 + +.. math:: + \left[\nabla\cdot\mathbf{B} = 0\right]. + :label: divb-2 + +The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` +and the finite-difference operators in time and space are defined +respectively as + +.. math:: + + \begin{align} + D_{t}G|_{i,j,k}^{n} & = \frac{(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2})}{\Delta t}, + \\ + D_{x}G|_{i,j,k}^{n} & = \frac{G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}}{\Delta x}, + \end{align} + +where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and +the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` +and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. +The difference operators along :math:`y` and :math:`z` are obtained by circular +permutation. The equations in brackets are given for completeness, +as they are often not actually solved, thanks to the usage of a so-called +charge conserving algorithm, as explained below. As shown in :numref:`fig_yee_grid`, +the quantities are given on a staggered (or “Yee”) +grid :cite:p:`pt-Yee`, where the electric field components are located +between nodes and the magnetic field components are located in the +center of the cell faces. Knowing the current densities at half-integer steps, +the electric field components are updated alternately with the magnetic +field components at integer and half-integer steps respectively. + +.. _theory-pic-mwsolve-nsfdtd: + +Non-Standard Finite-Difference Time-Domain (NSFDTD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An implementation of the source-free Maxwell’s wave equations for narrow-band +applications based on non-standard finite-differences (NSFD) +was introduced in :cite:t:`pt-Coleieee1997,pt-Coleieee2002`, and +was adapted for wideband applications in :cite:t:`pt-Karkicap06`. At +the Courant limit for the time step and for a given set of parameters, +the stencil proposed in :cite:t:`pt-Karkicap06` has no numerical dispersion +along the principal axes, provided that the cell size is the same +along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainen” +(or CK) solver uses the non-standard finite difference formulation +(based on extended stencils) of the Maxwell-Ampere equation and can be +implemented as follows :cite:p:`pt-Vayjcp2011`: + +.. math:: + D_{t}\mathbf{B} = -\nabla^{*}\times\mathbf{E} + :label: Faraday + +.. math:: + D_{t}\mathbf{E} = \nabla\times\mathbf{B}-\mathbf{J} + :label: Ampere + +.. math:: + \left[\nabla\cdot\mathbf{E} = \rho\right] + :label: Gauss + +.. math:: + \left[\nabla^{*}\cdot\mathbf{B}= 0\right] + :label: divb + +Eqs. (:eq:`Gauss`) and (:eq:`divb`) are not being solved explicitly +but verified via appropriate initial conditions and current deposition +procedure. The NSFD differential operator is given by + +.. math:: + + \nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}} + +where + +.. math:: + + D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x} + +with + +.. math:: + + \begin{align} + S_{x}^{1}G|_{i,j,k}^{n} & = G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}, + \\ + S_{x}^{2}G|_{i,j,k}^{n} & = G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}. + \end{align} + +Here :math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` +are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with +the FDTD algorithm, the quantities with half-integer are located between +the nodes (electric field components) or in the center of the cell +faces (magnetic field components). The operators along :math:`y` and :math:`z`, +i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, +:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular +permutation of the indices. + +Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients +given in :cite:t:`pt-Karkicap06` (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) +allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which +equates to having no numerical dispersion along the principal axes. +The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. +An extension to non-cubic cells is provided in 3-D by :cite:t:`pt-CowanPRSTAB13` and in 2-D by +:cite:t:`pt-PukhovJPP99`. An alternative NSFDTD implementation that enables superluminous waves is also +given in :cite:t:`pt-LehePRSTAB13`. + +As mentioned above, a key feature of the algorithms based on NSFDTD +is that some implementations :cite:p:`pt-Karkicap06,pt-CowanPRSTAB13` enable the time step :math:`\Delta t=\Delta x` along one or +more axes and no numerical dispersion along those axes. However, as +shown in :cite:t:`pt-Vayjcp2011`, an instability develops at the Nyquist +wavelength at (or very near) such a timestep. It is also shown in +the same paper that removing the Nyquist component in all the source +terms using a bilinear filter (see description of the filter below) +suppresses this instability. + +.. _theory-pic-mwsolve-psatd: + +Pseudo Spectral Analytical Time Domain (PSATD) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Maxwell’s equations in Fourier space are given by + +.. math:: \frac{\partial\mathbf{\tilde{E}}}{\partial t} = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}} +.. math:: \frac{\partial\mathbf{\tilde{B}}}{\partial t} = -i\mathbf{k}\times\mathbf{\tilde{E}} +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{E}} = \tilde{\rho}] +.. math:: {}[i\mathbf{k}\cdot\mathbf{\tilde{B}} = 0] + +where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. +As with the real space formulation, provided that the continuity equation +:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then +the last two equations will automatically be satisfied at any time +if satisfied initially and do not need to be explicitly integrated. + +Decomposing the electric field and current between longitudinal and +transverse components + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}} & = \mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}}) + \\ + \mathbf{\tilde{J}} & = \mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}}) + \end{aligned} + +gives + +.. math:: + + \begin{aligned} + \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}} + \\ + \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = -\mathbf{\tilde{J}_{L}} + \\ + \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = -i\mathbf{k}\times\mathbf{\tilde{E}} + \end{aligned} + +with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. + +If the sources are assumed to be constant over a time interval :math:`\Delta t`, +the system of equations is solvable analytically and is given by (see :cite:t:`pt-Habericnsp73` for the original formulation and :cite:t:`pt-VayJCP2013` +for a more detailed derivation): + +.. math:: + \mathbf{\tilde{E}}_{T}^{n+1} = C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2} + :label: PSATD_transverse_1 + +.. math:: + \mathbf{\tilde{E}}_{L}^{n+1} = \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2} + :label: PSATD_longitudinal + +.. math:: + \mathbf{\tilde{B}}^{n+1} = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2} + :label: PSATD_transverse_2 + +with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. + +Combining the transverse and longitudinal components, gives + +.. math:: + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2} + \\ + & + (1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber + \\ + & + \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right), + \end{aligned} + :label: Eq_PSATD_1 + +.. math:: + \begin{aligned} + \mathbf{\tilde{B}}^{n+1} & = C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n} + \\ + & + i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}. + \end{aligned} + :label: Eq_PSATD_2 + +For fields generated by the source terms without the self-consistent +dynamics of the charged particles, this algorithm is free of numerical +dispersion and is not subject to a Courant condition. Furthermore, +this solution is exact for any time step size subject to the assumption +that the current source is constant over that time step. + +As shown in :cite:t:`pt-VayJCP2013`, by expanding the coefficients :math:`S_{h}` +and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD +formulation reduces to the perhaps better known pseudo-spectral time-domain +(PSTD) formulation :cite:p:`pt-DawsonRMP83,pt-Liumotl1997`: + +.. math:: + + \begin{aligned} + \mathbf{\tilde{E}}^{n+1} & = \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2}, + \\ + \mathbf{\tilde{B}}^{n+3/2} & = \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}. + \end{aligned} + +The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` +In contrast to the PSATD solver, the PSTD solver is subject to numerical +dispersion for a finite time step and to a Courant condition that +is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta z^{2}}\right)^{-1/2}`. + +The PSATD and PSTD formulations that were just given apply to the +field components located at the nodes of the grid. As noted in :cite:t:`pt-Ohmurapiers2010`, +they can also be easily recast on a staggered Yee grid by multiplication +of the field components by the appropriate phase factors to shift +them from the collocated to the staggered locations. The choice between +a collocated and a staggered formulation is application-dependent. + +Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods :cite:p:`pt-VayJCP2013`, at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in :cite:t:`pt-VincentiCPC2017a` for stencils of arbitrary order (up-to the infinite “spectral” order). + +WarpX also includes a kinetic-fluid hybrid model in which the electric field is +calculated using Ohm's law instead of directly evolving Maxwell's equations. This +approach allows reduced physics simulations to be done with significantly lower +spatial and temporal resolution than in the standard, fully kinetic, PIC. Details +of this model can be found in the section +:ref:`Kinetic-fluid hybrid model `. + +.. _current_deposition: + +Current deposition +------------------ + +The current densities are deposited on the computational grid from +the particle position and velocities, employing splines of various +orders :cite:p:`pt-Abejcp86`. + +.. math:: + + \begin{aligned} + \rho & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n + \\ + \mathbf{J} & = \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n + \end{aligned} + +In most applications, it is essential to prevent the accumulation +of errors resulting from the violation of the discretized Gauss’ Law. +This is accomplished by providing a method for depositing the current +from the particles to the grid that preserves the discretized Gauss’ +Law, or by providing a mechanism for “divergence cleaning” :cite:p:`pt-Birdsalllangdon,pt-Langdoncpc92,pt-Marderjcp87,pt-Vaypop98,pt-Munzjcp2000`. +For the former, schemes that allow a deposition of the current that +is exact when combined with the Yee solver is given in :cite:t:`pt-Villasenorcpc92` +for linear splines and in :cite:t:`pt-Esirkepovcpc01` for splines of arbitrary order. + +The NSFDTD formulations given above and in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13` +apply to the Maxwell-Faraday +equation, while the discretized Maxwell-Ampere equation uses the FDTD +formulation. Consequently, the charge conserving algorithms developed +for current deposition :cite:p:`pt-Villasenorcpc92,pt-Esirkepovcpc01` apply +readily to those NSFDTD-based formulations. More details concerning +those implementations, including the expressions for the numerical +dispersion and Courant condition are given +in :cite:t:`pt-PukhovJPP99,pt-Vayjcp2011,pt-CowanPRSTAB13,pt-LehePRSTAB13`. + +Current correction +~~~~~~~~~~~~~~~~~~ + +In the case of the pseudospectral solvers, the current deposition +algorithm generally does not satisfy the discretized continuity equation +in Fourier space: + +.. math:: + \tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}. + +In this case, a Boris correction :cite:p:`pt-Birdsalllangdon` can be applied +in :math:`k` space in the form + +.. math:: + \mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\frac{\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}}{k}\mathbf{\hat{k}}, + +where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction +to the current can be applied (with some similarity to the current +deposition presented by Morse and Nielson in their potential-based +model in :cite:t:`pt-Morsenielson1971`) using + +.. math:: \mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k, + +where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse +component of the current is left untouched while the longitudinal +component is effectively replaced by the one obtained from integration +of the continuity equation, ensuring that the corrected current satisfies +the continuity equation. The advantage of correcting the current rather than +the electric field is that it is more local and thus more compatible with +domain decomposition of the fields for parallel computation :cite:p:`pt-VayJCP2013`. + +Vay deposition +~~~~~~~~~~~~~~ + +Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space :cite:p:`pt-Morsenielson1971,pt-Villasenorcpc92,pt-Esirkepovcpc01`. + +The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils :cite:p:`pt-VayJCP2013`. +The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. +The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as + +- 2D Cartesian geometry: + +.. math:: + \begin{align} + D_x & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_y & = \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) + \bigg] + \\[8pt] + D_z & = \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} + \bigg[ + \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) + \bigg] + \end{align} + +- 3D Cartesian geometry: + +.. math:: + \begin{align} + \begin{split} + D_x & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_y & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \\[8pt] + \begin{split} + D_z & = \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} + \bigg[ + 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) + \\[4pt] + & \phantom{=} \: + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) + \\[4pt] + & \phantom{=} \: - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) + \bigg] + \end{split} + \end{align} + +Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. +Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. + +Field gather +------------ + +In general, the field is gathered from the mesh onto the macroparticles +using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. +Three variations are considered: + +- “momentum conserving”: fields are interpolated from the grid nodes + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at staggered positions, + they are first interpolated to the nodes on an auxiliary grid), + +- “energy conserving (or Galerkin)”: fields are interpolated from + the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` + for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, + :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` + for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` + and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields + are known at the nodes, they are first interpolated to the staggered + positions on an auxiliary grid), + +- “uniform”: fields are interpolated directly form the Yee grid + to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` + for all field components (if the fields are known at the nodes, they + are first interpolated to the staggered positions on an auxiliary + grid). + +As shown in :cite:t:`pt-Birdsalllangdon,pt-HockneyEastwoodBook,pt-LewisJCP1972`, +the momentum and energy conserving schemes conserve momentum and energy +respectively at the limit of infinitesimal time steps and generally +offer better conservation of the respective quantities for a finite +time step. The uniform scheme does not conserve momentum nor energy +in the sense defined for the others but is given for completeness, +as it has been shown to offer some interesting properties in the modeling +of relativistically drifting plasmas :cite:p:`pt-GodfreyJCP2013`. + +.. _theory-pic-filter: + +Filtering +--------- + +It is common practice to apply digital filtering to the charge or +current density in Particle-In-Cell simulations as a complement or +an alternative to using higher order splines :cite:p:`pt-Birdsalllangdon`. +A commonly used filter in PIC simulations is the three points filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2 + +where :math:`\phi^{f}` is the filtered quantity. This filter is called +a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and +:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is +given as a function of the filtering coefficient :math:`\alpha` and +the wavenumber :math:`k` by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +The total attenuation :math:`G` for :math:`n` successive applications of filters +of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by + +.. math:: + G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. + +A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, +so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” +step :cite:p:`pt-Birdsalllangdon`. For the bilinear filter (:math:`\alpha=1/2`), +the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession +of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. + +It is sometimes necessary to filter on a relatively wide band of wavelength, +necessitating the application of a large number of passes of the bilinear +filter or on the use of filters acting on many points. The former +can become very intensive computationally while the latter is problematic +for parallel computations using domain decomposition, as the footprint +of the filter may eventually surpass the size of subdomains. A workaround +is to use a combination of filters of limited footprint. A solution +based on the combination of three point filters with various strides +was proposed in :cite:t:`pt-Vayjcp2011` and operates as follows. + +The bilinear filter provides complete suppression of the signal at +the grid Nyquist wavelength (twice the grid cell size). Suppression +of the signal at integer multiples of the Nyquist wavelength can be +obtained by using a stride :math:`s` in the filter + +.. math:: + \phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2 + +for which the gain is given by + +.. math:: + g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right). + +For a given stride, the gain is given by the gain of the bilinear +filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength +:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, +as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. +The resulting filter is pass band between the poles, but since the +poles are spread at different integer values in k space, a wide band +low pass filter can be constructed by combining filters using different +strides. As shown in :cite:t:`pt-Vayjcp2011`, the successive application +of 4-passes + compensation of filters with strides 1, 2 and 4 has +a nearly equivalent fall-off in gain as 80 passes + compensation of +a bilinear filter. Yet, the strided filter solution needs only 15 +passes of a three-point filter, compared to 81 passes for an equivalent +n-pass bilinear filter, yielding a gain of 5.4 in number of operations +in favor of the combination of filters with stride. The width of the +filter with stride 4 extends only on 9 points, compared to 81 points +for a single pass equivalent filter, hence giving a gain of 9 in compactness +for the stride filters combination in comparison to the single-pass +filter with large stencil, resulting in more favorable scaling with the number +of computational cores for parallel calculations. + +.. bibliography:: + :keyprefix: pt- diff --git a/Docs/source/theory/picsar_theory.rst b/Docs/source/theory/picsar_theory.rst deleted file mode 100644 index 0aadbd558f0..00000000000 --- a/Docs/source/theory/picsar_theory.rst +++ /dev/null @@ -1,867 +0,0 @@ -.. raw:: latex - - \markboth{J.-L. Vay, R. Lehe}{Simulations for plasma and laser acceleration.} - -.. raw:: latex - - \maketitle - -.. raw:: latex - - \linenumbers - -.. _theory-pic: - -Particle-in-Cell Method -======================= - -.. figure:: PIC.png - :alt: [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - - [fig:PIC] The Particle-In-Cell (PIC) method follows the evolution of a collection of charged macro-particles (positively charged in blue on the left plot, negatively charged in red) that evolve self-consistently with their electromagnetic (or electrostatic) fields. The core PIC algorithm involves four operations at each time step: 1) evolve the velocity and position of the particles using the Newton-Lorentz equations, 2) deposit the charge and/or current densities through interpolation from the particles distributions onto the grid, 3) evolve Maxwell’s wave equations (for electromagnetic) or solve Poisson’s equation (for electrostatic) on the grid, 4) interpolate the fields from the grid onto the particles for the next particle push. Additional “add-ons” operations are inserted between these core operations to account for additional physics (e.g. absorption/emission of particles, addition of external forces to account for accelerator focusing or accelerating component) or numerical effects (e.g. smoothing/filtering of the charge/current densities and/or fields on the grid). - -In the *electromagnetic particle-in-cell method* (Birdsall and Langdon 1991), -the electromagnetic fields are solved on a grid, usually using Maxwell’s -equations - -.. math:: - - \begin{aligned} - \frac{\mathbf{\partial B}}{\partial t} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-1}\\ - \frac{\mathbf{\partial E}}{\partial t} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-1}\\ - \nabla\cdot\mathbf{E} & = & \rho\label{Eq:Gauss-1}\\ - \nabla\cdot\mathbf{B} & = & 0\label{Eq:divb-1}\end{aligned} - -given here in natural units (:math:`\epsilon_0=\mu_0=c=1`), where :math:`t` is time, :math:`\mathbf{E}` and -:math:`\mathbf{B}` are the electric and magnetic field components, and -:math:`\rho` and :math:`\mathbf{J}` are the charge and current densities. The -charged particles are advanced in time using the Newton-Lorentz equations -of motion - -.. math:: - - \begin{aligned} - \frac{d\mathbf{x}}{dt}= & \mathbf{v},\label{Eq:Lorentz_x-1}\\ - \frac{d\left(\gamma\mathbf{v}\right)}{dt}= & \frac{q}{m}\left(\mathbf{E}+\mathbf{v}\times\mathbf{B}\right),\label{Eq:Lorentz_v-1}\end{aligned} - -where :math:`m`, :math:`q`, :math:`\mathbf{x}`, :math:`\mathbf{v}` and :math:`\gamma=1/\sqrt{1-v^{2}}` -are respectively the mass, charge, position, velocity and relativistic -factor of the particle given in natural units (:math:`c=1`). The charge and current densities are interpolated -on the grid from the particles’ positions and velocities, while the -electric and magnetic field components are interpolated from the grid -to the particles’ positions for the velocity update. - -.. _theory-pic-push: - -Particle push -------------- - -A centered finite-difference discretization of the Newton-Lorentz -equations of motion is given by - -.. math:: - - \begin{aligned} - \frac{\mathbf{x}^{i+1}-\mathbf{x}^{i}}{\Delta t}= & \mathbf{v}^{i+1/2},\label{Eq:leapfrog_x}\\ - \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}-\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{\Delta t}= & \frac{q}{m}\left(\mathbf{E}^{i}+\mathbf{\bar{v}}^{i}\times\mathbf{B}^{i}\right).\label{Eq:leapfrog_v}\end{aligned} - -In order to close the system, :math:`\bar{\mathbf{v}}^{i}` must be -expressed as a function of the other quantities. The two implementations that have become the most popular are presented below. - -.. _theory-pic-push-boris: - -Boris relativistic velocity rotation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The solution proposed by Boris (Boris 1970) is given by - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\gamma^{i+1/2}\mathbf{v}^{i+1/2}+\gamma^{i-1/2}\mathbf{v}^{i-1/2}}{2\bar{\gamma}^{i}}.\label{Eq:boris_v}\end{aligned} - -where :math:`\bar{\gamma}^{i}` is defined by :math:`\bar{\gamma}^{i} \equiv (\gamma^{i+1/2}+\gamma^{i-1/2} )/2`. - -The system (`[Eq:leapfrog_v] <#Eq:leapfrog_v>`__,\ `[Eq:boris_v] <#Eq:boris_v>`__) is solved very -efficiently following Boris’ method, where the electric field push -is decoupled from the magnetic push. Setting :math:`\mathbf{u}=\gamma\mathbf{v}`, the -velocity is updated using the following sequence: - -.. math:: - - \begin{aligned} - \mathbf{u^{-}}= & \mathbf{u}^{i-1/2}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\\ - \mathbf{u'}= & \mathbf{u}^{-}+\mathbf{u}^{-}\times\mathbf{t}\\ - \mathbf{u}^{+}= & \mathbf{u}^{-}+\mathbf{u'}\times2\mathbf{t}/(1+t^{2})\\ - \mathbf{u}^{i+1/2}= & \mathbf{u}^{+}+\left(q\Delta t/2m\right)\mathbf{E}^{i}\end{aligned} - -where :math:`\mathbf{t}=\left(q\Delta t/2m\right)\mathbf{B}^{i}/\bar{\gamma}^{i}` and where -:math:`\bar{\gamma}^{i}` can be calculated as :math:`\bar{\gamma}^{i}=\sqrt{1+(\mathbf{u}^-/c)^2}`. - -The Boris implementation is second-order accurate, time-reversible and fast. Its implementation is very widespread and used in the vast majority of PIC codes. - -.. _theory-pic-push-vay: - -Vay Lorentz-invariant formulation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It was shown in (Vay 2008) that the Boris formulation is -not Lorentz invariant and can lead to significant errors in the treatment -of relativistic dynamics. A Lorentz invariant formulation is obtained -by considering the following velocity average - -.. math:: - - \begin{aligned} - \mathbf{\bar{v}}^{i}= & \frac{\mathbf{v}^{i+1/2}+\mathbf{v}^{i-1/2}}{2},\label{Eq:new_v}\end{aligned} - -This gives a system that is solvable analytically (see (Vay 2008) -for a detailed derivation), giving the following velocity update: - -.. math:: - - \begin{aligned} - \mathbf{u^{*}}= & \mathbf{u}^{i-1/2}+\frac{q\Delta t}{m}\left(\mathbf{E}^{i}+\frac{\mathbf{v}^{i-1/2}}{2}\times\mathbf{B}^{i}\right),\label{pusher_gamma}\\ - \mathbf{u}^{i+1/2}= & \left[\mathbf{u^{*}}+\left(\mathbf{u^{*}}\cdot\mathbf{t}\right)\mathbf{t}+\mathbf{u^{*}}\times\mathbf{t}\right]/\left(1+t^{2}\right),\label{pusher_upr}\end{aligned} - -where :math:`\mathbf{t}=\boldsymbol{\tau}/\gamma^{i+1/2}`, :math:`\boldsymbol{\tau}=\left(q\Delta t/2m\right)\mathbf{B}^{i}`, -:math:`\gamma^{i+1/2}=\sqrt{\sigma+\sqrt{\sigma^{2}+\left(\tau^{2}+w^{2}\right)}}`, -:math:`w=\mathbf{u^{*}}\cdot\boldsymbol{\tau}`, :math:`\sigma=\left(\gamma'^{2}-\tau^{2}\right)/2` -and :math:`\gamma'=\sqrt{1+(\mathbf{u}^{*}/c)^{2}}`. This Lorentz invariant formulation -is particularly well suited for the modeling of ultra-relativistic -charged particle beams, where the accurate account of the cancellation -of the self-generated electric and magnetic fields is essential, as -shown in (Vay 2008). - -.. _theory-pic-mwsolve: - -Field solve ------------ - -Various methods are available for solving Maxwell’s equations on a -grid, based on finite-differences, finite-volume, finite-element, -spectral, or other discretization techniques that apply most commonly -on single structured or unstructured meshes and less commonly on multiblock -multiresolution grid structures. In this chapter, we summarize the widespread -second order finite-difference time-domain (FDTD) algorithm, its extension -to non-standard finite-differences as well as the pseudo-spectral -analytical time-domain (PSATD) and pseudo-spectral time-domain (PSTD) -algorithms. Extension to multiresolution (or mesh refinement) PIC -is described in, e.g. (Vay et al. 2012; Vay, Adam, and Heron 2004). - -.. _theory-pic-mwsolve-fdtd: - -Finite-Difference Time-Domain (FDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The most popular algorithm for electromagnetic PIC codes is the Finite-Difference -Time-Domain (or FDTD) solver - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla\times\mathbf{E}\label{Eq:Faraday-2}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere-2}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss-2}\\ - \left[\nabla\cdot\mathbf{B}\right. & = & \left.0\right].\label{Eq:divb-2}\end{aligned} - -.. figure:: Yee_grid.png - :alt: [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - - [fig:yee_grid](left) Layout of field components on the staggered “Yee” grid. Current densities and electric fields are defined on the edges of the cells and magnetic fields on the faces. (right) Time integration using a second-order finite-difference "leapfrog" integrator. - -The differential operator is defined as :math:`\nabla=D_{x}\mathbf{\hat{x}}+D_{y}\mathbf{\hat{y}}+D_{z}\mathbf{\hat{z}}` -and the finite-difference operators in time and space are defined -respectively as - -.. math:: D_{t}G|_{i,j,k}^{n}=\left(G|_{i,j,k}^{n+1/2}-G|_{i,j,k}^{n-1/2}\right)/\Delta t - -and :math:`D_{x}G|_{i,j,k}^{n}=\left(G|_{i+1/2,j,k}^{n}-G|_{i-1/2,j,k}^{n}\right)/\Delta x`, -where :math:`\Delta t` and :math:`\Delta x` are respectively the time step and -the grid cell size along :math:`x`, :math:`n` is the time index and :math:`i`, :math:`j` -and :math:`k` are the spatial indices along :math:`x`, :math:`y` and :math:`z` respectively. -The difference operators along :math:`y` and :math:`z` are obtained by circular -permutation. The equations in brackets are given for completeness, -as they are often not actually solved, thanks to the usage of a so-called -charge conserving algorithm, as explained below. As shown in Figure -`[fig:yee_grid] <#fig:yee_grid>`__, the quantities are given on a staggered (or “Yee”) -grid (Yee 1966), where the electric field components are located -between nodes and the magnetic field components are located in the -center of the cell faces. Knowing the current densities at half-integer steps, -the electric field components are updated alternately with the magnetic -field components at integer and half-integer steps respectively. - -.. _theory-pic-mwsolve-nsfdtd: - -Non-Standard Finite-Difference Time-Domain (NSFDTD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In (Cole 1997, 2002), Cole introduced an implementation -of the source-free Maxwell’s wave equations for narrow-band applications -based on non-standard finite-differences (NSFD). In (Karkkainen et al. 2006), -Karkkainen *et al.* adapted it for wideband applications. At -the Courant limit for the time step and for a given set of parameters, -the stencil proposed in (Karkkainen et al. 2006) has no numerical dispersion -along the principal axes, provided that the cell size is the same -along each dimension (i.e. cubic cells in 3D). The “Cole-Karkkainnen” -(or CK) solver uses the non-standard finite difference formulation -(based on extended stencils) of the Maxwell-Ampere equation and can be -implemented as follows (Vay et al. 2011): - -.. math:: - - \begin{aligned} - D_{t}\mathbf{B} & = & -\nabla^{*}\times\mathbf{E}\label{Eq:Faraday}\\ - D_{t}\mathbf{E} & = & \nabla\times\mathbf{B}-\mathbf{J}\label{Eq:Ampere}\\ - \left[\nabla\cdot\mathbf{E}\right. & = & \left.\rho\right]\label{Eq:Gauss}\\ - \left[\nabla^{*}\cdot\mathbf{B}\right. & = & \left.0\right]\label{Eq:divb}\end{aligned} - -Eq. `[Eq:Gauss] <#Eq:Gauss>`__ and `[Eq:divb] <#Eq:divb>`__ are not being solved explicitly -but verified via appropriate initial conditions and current deposition -procedure. The NSFD differential operators is given by :math:`\nabla^{*}=D_{x}^{*}\mathbf{\hat{x}}+D_{y}^{*}\mathbf{\hat{y}}+D_{z}^{*}\mathbf{\hat{z}}` -where :math:`D_{x}^{*}=\left(\alpha+\beta S_{x}^{1}+\xi S_{x}^{2}\right)D_{x}` -with :math:`S_{x}^{1}G|_{i,j,k}^{n}=G|_{i,j+1,k}^{n}+G|_{i,j-1,k}^{n}+G|_{i,j,k+1}^{n}+G|_{i,j,k-1}^{n}`, -:math:`S_{x}^{2}G|_{i,j,k}^{n}=G|_{i,j+1,k+1}^{n}+G|_{i,j-1,k+1}^{n}+G|_{i,j+1,k-1}^{n}+G|_{i,j-1,k-1}^{n}`. -:math:`G` is a sample vector component, while :math:`\alpha`, :math:`\beta` and :math:`\xi` -are constant scalars satisfying :math:`\alpha+4\beta+4\xi=1`. As with -the FDTD algorithm, the quantities with half-integer are located between -the nodes (electric field components) or in the center of the cell -faces (magnetic field components). The operators along :math:`y` and :math:`z`, -i.e. :math:`D_{y}`, :math:`D_{z}`, :math:`D_{y}^{*}`, :math:`D_{z}^{*}`, :math:`S_{y}^{1}`, -:math:`S_{z}^{1}`, :math:`S_{y}^{2}`, and :math:`S_{z}^{2}`, are obtained by circular -permutation of the indices. - -Assuming cubic cells (:math:`\Delta x=\Delta y=\Delta z`), the coefficients -given in (Karkkainen et al. 2006) (:math:`\alpha=7/12`, :math:`\beta=1/12` and :math:`\xi=1/48`) -allow for the Courant condition to be at :math:`\Delta t=\Delta x`, which -equates to having no numerical dispersion along the principal axes. -The algorithm reduces to the FDTD algorithm with :math:`\alpha=1` and :math:`\beta=\xi=0`. -An extension to non-cubic cells is provided by Cowan, *et al.* -in 3-D in (Cowan et al. 2013) and was given by Pukhov in 2-D in -(Pukhov 1999). An alternative NSFDTD implementation that enables superluminous waves is also -given by Lehe et al. in (Lehe et al. 2013). - -As mentioned above, a key feature of the algorithms based on NSFDTD -is that some implementations (Karkkainen et al. 2006; Cowan et al. 2013) enable the time step :math:`\Delta t=\Delta x` along one or -more axes and no numerical dispersion along those axes. However, as -shown in (Vay et al. 2011), an instability develops at the Nyquist -wavelength at (or very near) such a timestep. It is also shown in -the same paper that removing the Nyquist component in all the source -terms using a bilinear filter (see description of the filter below) -suppresses this instability. - -.. _theory-pic-mwsolve-psatd: - -Pseudo Spectral Analytical Time Domain (PSATD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Maxwell’s equations in Fourier space are given by - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{E}}& = & \tilde{\rho}]\\ - {}[i\mathbf{k}\cdot\mathbf{\tilde{B}}& = & 0]\end{aligned} - -where :math:`\tilde{a}` is the Fourier Transform of the quantity :math:`a`. -As with the real space formulation, provided that the continuity equation -:math:`\partial\tilde{\rho}/\partial t+i\mathbf{k}\cdot\mathbf{\tilde{J}}=0` is satisfied, then -the last two equations will automatically be satisfied at any time -if satisfied initially and do not need to be explicitly integrated. - -Decomposing the electric field and current between longitudinal and -transverse components :math:`\mathbf{\tilde{E}}=\mathbf{\tilde{E}}_{L}+\mathbf{\tilde{E}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{E}})` -and :math:`\mathbf{\tilde{J}}=\mathbf{\tilde{J}}_{L}+\mathbf{\tilde{J}}_{T}=\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}})-\mathbf{\hat{k}}\times(\mathbf{\hat{k}}\times\mathbf{\tilde{J}})` -gives - -.. math:: - - \begin{aligned} - \frac{\partial\mathbf{\tilde{E}}_{T}}{\partial t} & = & i\mathbf{k}\times\mathbf{\tilde{B}}-\mathbf{\tilde{J}_{T}}\\ - \frac{\partial\mathbf{\tilde{E}}_{L}}{\partial t} & = & -\mathbf{\tilde{J}_{L}}\\ - \frac{\partial\mathbf{\tilde{B}}}{\partial t} & = & -i\mathbf{k}\times\mathbf{\tilde{E}}\end{aligned} - -with :math:`\mathbf{\hat{k}}=\mathbf{k}/k`. - -If the sources are assumed to be constant over a time interval :math:`\Delta t`, -the system of equations is solvable analytically and is given by (see -(Haber et al. 1973) for the original formulation and (Jean-Luc Vay, Haber, and Godfrey 2013) -for a more detailed derivation): - -[Eq:PSATD] - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}_{T}^{n+1} & = & C\mathbf{\tilde{E}}_{T}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}_{T}^{n+1/2}\label{Eq:PSATD_transverse_1}\\ - \mathbf{\tilde{E}}_{L}^{n+1} & = & \mathbf{\tilde{E}}_{L}^{n}-\Delta t\mathbf{\tilde{J}}_{L}^{n+1/2}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}\label{Eq:PSATD_transverse_2}\end{aligned} - -with :math:`C=\cos\left(k\Delta t\right)` and :math:`S=\sin\left(k\Delta t\right)`. - -Combining the transverse and longitudinal components, gives - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & C\mathbf{\tilde{E}}^{n}+iS\mathbf{\hat{k}}\times\mathbf{\tilde{B}}^{n}-\frac{S}{k}\mathbf{\tilde{J}}^{n+1/2}\\ - & + &(1-C)\mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{E}}^{n})\nonumber \\ - & + & \mathbf{\hat{k}}(\mathbf{\hat{k}}\cdot\mathbf{\tilde{J}}^{n+1/2})\left(\frac{S}{k}-\Delta t\right),\label{Eq_PSATD_1}\\ - \mathbf{\tilde{B}}^{n+1} & = & C\mathbf{\tilde{B}}^{n}-iS\mathbf{\hat{k}}\times\mathbf{\tilde{E}}^{n}\\ - &+&i\frac{1-C}{k}\mathbf{\hat{k}}\times\mathbf{\tilde{J}}^{n+1/2}.\label{Eq_PSATD_2}\end{aligned} - -For fields generated by the source terms without the self-consistent -dynamics of the charged particles, this algorithm is free of numerical -dispersion and is not subject to a Courant condition. Furthermore, -this solution is exact for any time step size subject to the assumption -that the current source is constant over that time step. - -As shown in (Jean-Luc Vay, Haber, and Godfrey 2013), by expanding the coefficients :math:`S_{h}` -and :math:`C_{h}` in Taylor series and keeping the leading terms, the PSATD -formulation reduces to the perhaps better known pseudo-spectral time-domain -(PSTD) formulation (Dawson 1983; Liu 1997): - -.. math:: - - \begin{aligned} - \mathbf{\tilde{E}}^{n+1} & = & \mathbf{\tilde{E}}^{n}+i\Delta t\mathbf{k}\times\mathbf{\tilde{B}}^{n+1/2}-\Delta t\mathbf{\tilde{J}}^{n+1/2},\\ - \mathbf{\tilde{B}}^{n+3/2} & = & \mathbf{\tilde{B}}^{n+1/2}-i\Delta t\mathbf{k}\times\mathbf{\tilde{E}}^{n+1}.\end{aligned} - -The dispersion relation of the PSTD solver is given by :math:`\sin(\frac{\omega\Delta t}{2})=\frac{k\Delta t}{2}.` -In contrast to the PSATD solver, the PSTD solver is subject to numerical -dispersion for a finite time step and to a Courant condition that -is given by :math:`\Delta t\leq \frac{2}{\pi}\left(\frac{1}{\Delta x^{2}}+\frac{1}{\Delta y^{2}}+\frac{1}{\Delta x^{2}}\right)^{-1/2}.` - -The PSATD and PSTD formulations that were just given apply to the -field components located at the nodes of the grid. As noted in (Ohmura and Okamura 2010), -they can also be easily recast on a staggered Yee grid by multiplication -of the field components by the appropriate phase factors to shift -them from the collocated to the staggered locations. The choice between -a collocated and a staggered formulation is application-dependent. - -Spectral solvers used to be very popular in the years 1970s to early 1990s, before being replaced by finite-difference methods with the advent of parallel supercomputers that favored local methods. However, it was shown recently that standard domain decomposition with Fast Fourier Transforms that are local to each subdomain could be used effectively with PIC spectral methods (Jean-Luc Vay, Haber, and Godfrey 2013), at the cost of truncation errors in the guard cells that could be neglected. A detailed analysis of the effectiveness of the method with exact evaluation of the magnitude of the effect of the truncation error is given in (Vincenti and Vay 2016) for stencils of arbitrary order (up-to the infinite “spectral” order). - -WarpX also includes a kinetic-fluid hybrid model in which the electric field is -calculated using Ohm's law instead of directly evolving Maxwell's equations. This -approach allows reduced physics simulations to be done with significantly lower -spatial and temporal resolution than in the standard, fully kinetic, PIC. Details -of this model can be found in the section -:ref:`Kinetic-fluid hybrid model `. - -.. _current_deposition: - -Current deposition ------------------- - -The current densities are deposited on the computational grid from -the particle position and velocities, employing splines of various -orders (Abe et al. 1986). - -.. math:: - - \begin{aligned} - \rho & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_nS_n\\ - \mathbf{J} & = & \frac{1}{\Delta x \Delta y \Delta z}\sum_nq_n\mathbf{v_n}S_n\end{aligned} - -In most applications, it is essential to prevent the accumulation -of errors resulting from the violation of the discretized Gauss’ Law. -This is accomplished by providing a method for depositing the current -from the particles to the grid that preserves the discretized Gauss’ -Law, or by providing a mechanism for “divergence cleaning” (Birdsall and Langdon 1991; Langdon 1992; Marder 1987; Vay and Deutsch 1998; Munz et al. 2000). -For the former, schemes that allow a deposition of the current that -is exact when combined with the Yee solver is given in (Villasenor and Buneman 1992) -for linear splines and in (Esirkepov 2001) for splines of arbitrary order. - -The NSFDTD formulations given above and in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013) -apply to the Maxwell-Faraday -equation, while the discretized Maxwell-Ampere equation uses the FDTD -formulation. Consequently, the charge conserving algorithms developed -for current deposition (Villasenor and Buneman 1992; Esirkepov 2001) apply -readily to those NSFDTD-based formulations. More details concerning -those implementations, including the expressions for the numerical -dispersion and Courant condition are given -in (Pukhov 1999; Vay et al. 2011; Cowan et al. 2013; Lehe et al. 2013). - -Current correction -~~~~~~~~~~~~~~~~~~ - -In the case of the pseudospectral solvers, the current deposition -algorithm generally does not satisfy the discretized continuity equation -in Fourier space :math:`\tilde{\rho}^{n+1}=\tilde{\rho}^{n}-i\Delta t\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}`. -In this case, a Boris correction (Birdsall and Langdon 1991) can be applied -in :math:`k` space in the form :math:`\mathbf{\tilde{E}}_{c}^{n+1}=\mathbf{\tilde{E}}^{n+1}-\left(\mathbf{k}\cdot\mathbf{\tilde{E}}^{n+1}+i\tilde{\rho}^{n+1}\right)\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{E}}_{c}` is the corrected field. Alternatively, a correction -to the current can be applied (with some similarity to the current -deposition presented by Morse and Nielson in their potential-based -model in (Morse and Nielson 1971)) using :math:`\mathbf{\tilde{J}}_{c}^{n+1/2}=\mathbf{\tilde{J}}^{n+1/2}-\left[\mathbf{k}\cdot\mathbf{\tilde{J}}^{n+1/2}-i\left(\tilde{\rho}^{n+1}-\tilde{\rho}^{n}\right)/\Delta t\right]\mathbf{\hat{k}}/k`, -where :math:`\mathbf{\tilde{J}}_{c}` is the corrected current. In this case, the transverse -component of the current is left untouched while the longitudinal -component is effectively replaced by the one obtained from integration -of the continuity equation, ensuring that the corrected current satisfies -the continuity equation. The advantage of correcting the current rather than -the electric field is that it is more local and thus more compatible with -domain decomposition of the fields for parallel computation (Jean Luc Vay, Haber, and Godfrey 2013). - -Vay deposition -~~~~~~~~~~~~~~ - -Alternatively, an exact current deposition can be written for the pseudo-spectral solvers, following the geometrical interpretation of existing methods in real space (`Morse and Nielson, 1971 `_; `Villasenor and Buneman, 1992 `_; `Esirkepov, 2001 `_). - -The Vay deposition scheme is the generalization of the Esirkepov deposition scheme for the spectral case with arbitrary-order stencils `(Vay et al, 2013) `_. -The current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` in Fourier space is computed as :math:`\widehat{\boldsymbol{J}}^{\,n+1/2} = i \, \widehat{\boldsymbol{D}} / \boldsymbol{k}` when :math:`\boldsymbol{k} \neq 0` and set to zero otherwise. -The quantity :math:`\boldsymbol{D}` is deposited in real space by averaging the currents over all possible grid paths between the initial position :math:`\boldsymbol{x}^{\,n}` and the final position :math:`\boldsymbol{x}^{\,n+1}` and is defined as - -- 2D Cartesian geometry: - -.. math:: - \begin{align} - D_x = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},z_i^{n}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_y = & \: \sum_i \frac{v_i^y}{\Delta x \Delta z} \frac{q_i w_i}{4} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) + \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) + \Gamma(x_i^{n},z_i^{n}) - \bigg] - \\[8pt] - D_z = & \: \sum_i \frac{1}{\Delta x \Delta z} \frac{q_i w_i}{2 \Delta t} - \bigg[ - \Gamma(x_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},z_i^{n}) - \bigg] - \end{align} - -- 3D Cartesian geometry: - -.. math:: - \begin{align} - \begin{split} - D_x = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) + 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_y = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & + \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) - + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) + 2 \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} \\[8pt] - \begin{split} - D_z = & \: \sum_i \frac{1}{\Delta x\Delta y\Delta z} \frac{q_i w_i}{6\Delta t} - \bigg[ - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n+1}) - 2 \Gamma(x_i^{n+1},y_i^{n+1},z_i^{n}) \\[4pt] - & + \Gamma(x_i^{n},y_i^{n+1},z_i^{n+1}) - \Gamma(x_i^{n},y_i^{n+1},z_i^{n}) - + \Gamma(x_i^{n+1},y_i^{n},z_i^{n+1}) \\[4pt] - & - \Gamma(x_i^{n+1},y_i^{n},z_i^{n}) + 2 \Gamma(x_i^{n},y_i^{n},z_i^{n+1}) - - 2 \Gamma(x_i^{n},y_i^{n},z_i^{n}) - \bigg] - \end{split} - \end{align} - -Here, :math:`w_i` represents the weight of the :math:`i`-th macro-particle and :math:`\Gamma` represents its shape factor. -Note that in 2D Cartesian geometry, :math:`D_y` is effectively :math:`J_y` and does not require additional operations in Fourier space. - -Field gather ------------- - -In general, the field is gathered from the mesh onto the macroparticles -using splines of the same order as for the current deposition :math:`\mathbf{S}=\left(S_{x},S_{y},S_{z}\right)`. -Three variations are considered: - -- “momentum conserving”: fields are interpolated from the grid nodes - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at staggered positions, - they are first interpolated to the nodes on an auxiliary grid), - -- “energy conserving (or Galerkin)”: fields are interpolated from - the staggered Yee grid to the macroparticles using :math:`\left(S_{nx-1},S_{ny},S_{nz}\right)` - for :math:`E_{x}`, :math:`\left(S_{nx},S_{ny-1},S_{nz}\right)` for :math:`E_{y}`, - :math:`\left(S_{nx},S_{ny},S_{nz-1}\right)` for :math:`E_{z}`, :math:`\left(S_{nx},S_{ny-1},S_{nz-1}\right)` - for :math:`B_{x}`, :math:`\left(S_{nx-1},S_{ny},S_{nz-1}\right)` for :math:`B{}_{y}` - and\ :math:`\left(S_{nx-1},S_{ny-1},S_{nz}\right)` for :math:`B_{z}` (if the fields - are known at the nodes, they are first interpolated to the staggered - positions on an auxiliary grid), - -- “uniform”: fields are interpolated directly form the Yee grid - to the macroparticles using :math:`\mathbf{S}=\left(S_{nx},S_{ny},S_{nz}\right)` - for all field components (if the fields are known at the nodes, they - are first interpolated to the staggered positions on an auxiliary - grid). - -As shown in :raw-latex:`\cite{BirdsallLangdon,HockneyEastwoodBook,LewisJCP1972}`, -the momentum and energy conserving schemes conserve momentum and energy -respectively at the limit of infinitesimal time steps and generally -offer better conservation of the respective quantities for a finite -time step. The uniform scheme does not conserve momentum nor energy -in the sense defined for the others but is given for completeness, -as it has been shown to offer some interesting properties in the modeling -of relativistically drifting plasmas :raw-latex:`\cite{GodfreyJCP2013}`. - -.. _theory-pic-filter: - -Filtering -========= - -It is common practice to apply digital filtering to the charge or -current density in Particle-In-Cell simulations as a complement or -an alternative to using higher order splines (Birdsall and Langdon 1991). -A commonly used filter in PIC simulations is the three points filter -:math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-1}+\phi_{j+1}\right)/2` -where :math:`\phi^{f}` is the filtered quantity. This filter is called -a bilinear filter when :math:`\alpha=0.5`. Assuming :math:`\phi=e^{jkx}` and -:math:`\phi^{f}=g\left(\alpha,k\right)e^{jkx}`, the filter gain :math:`g` is -given as a function of the filtering coefficient :math:`\alpha` and -the wavenumber :math:`k` by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(k\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -The total attenuation :math:`G` for :math:`n` successive applications of filters -of coefficients :math:`\alpha_{1}`...\ :math:`\alpha_{n}` is given by :math:`G=\prod_{i=1}^{n}g\left(\alpha_{i},k\right)\approx1-\left(n-\sum_{i=1}^{n}\alpha_{i}\right)\frac{\left(k\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -A sharper cutoff in :math:`k` space is provided by using :math:`\alpha_{n}=n-\sum_{i=1}^{n-1}\alpha_{i}`, -so that :math:`G\approx1+O\left(k^{4}\right)`. Such step is called a “compensation” -step (Birdsall and Langdon 1991). For the bilinear filter (:math:`\alpha=1/2`), -the compensation factor is :math:`\alpha_{c}=2-1/2=3/2`. For a succession -of :math:`n` applications of the bilinear factor, it is :math:`\alpha_{c}=n/2+1`. - -It is sometimes necessary to filter on a relatively wide band of wavelength, -necessitating the application of a large number of passes of the bilinear -filter or on the use of filters acting on many points. The former -can become very intensive computationally while the latter is problematic -for parallel computations using domain decomposition, as the footprint -of the filter may eventually surpass the size of subdomains. A workaround -is to use a combination of filters of limited footprint. A solution -based on the combination of three point filters with various strides -was proposed in (Vay et al. 2011) and operates as follows. - -The bilinear filter provides complete suppression of the signal at -the grid Nyquist wavelength (twice the grid cell size). Suppression -of the signal at integer multiples of the Nyquist wavelength can be -obtained by using a stride :math:`s` in the filter :math:`\phi_{j}^{f}=\alpha\phi_{j}+\left(1-\alpha\right)\left(\phi_{j-s}+\phi_{j+s}\right)/2` -for which the gain is given by :math:`g\left(\alpha,k\right)=\alpha+\left(1-\alpha\right)\cos\left(sk\Delta x\right)\approx1-\left(1-\alpha\right)\frac{\left(sk\Delta x\right)^{2}}{2}+O\left(k^{4}\right)`. -For a given stride, the gain is given by the gain of the bilinear -filter shifted in k space, with the pole :math:`g=0` shifted from the wavelength -:math:`\lambda=2/\Delta x` to :math:`\lambda=2s/\Delta x`, with additional poles, -as given by :math:`sk\Delta x=\arccos\left(\frac{\alpha}{\alpha-1}\right)\pmod{2\pi}`. -The resulting filter is pass band between the poles, but since the -poles are spread at different integer values in k space, a wide band -low pass filter can be constructed by combining filters using different -strides. As shown in (Vay et al. 2011), the successive application -of 4-passes + compensation of filters with strides 1, 2 and 4 has -a nearly equivalent fall-off in gain as 80 passes + compensation of -a bilinear filter. Yet, the strided filter solution needs only 15 -passes of a three-point filter, compared to 81 passes for an equivalent -n-pass bilinear filter, yielding a gain of 5.4 in number of operations -in favor of the combination of filters with stride. The width of the -filter with stride 4 extends only on 9 points, compared to 81 points -for a single pass equivalent filter, hence giving a gain of 9 in compactness -for the stride filters combination in comparison to the single-pass -filter with large stencil, resulting in more favorable scaling with the number -of computational cores for parallel calculations. - -.. raw:: latex - - \IfFileExists{\jobname.bbl}{} {\typeout{} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{{*}{*} Please run \textquotedbl{}bibtex \jobname\textquotedbl{} - to optain} \typeout{{*}{*} the bibliography and then re-run LaTeX} - \typeout{{*}{*} twice to fix the references!} \typeout{{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}{*}} - \typeout{} } - -.. raw:: html - -
- -.. raw:: html - -
- -Abe, H, N Sakairi, R Itatani, and H Okuda. 1986. “High-Order Spline Interpolations in the Particle Simulation.” *Journal of Computational Physics* 63 (2): 247–67. - -.. raw:: html - -
- -.. raw:: html - -
- -Birdsall, C K, and A B Langdon. 1991. *Plasma Physics via Computer Simulation*. Adam-Hilger. - -.. raw:: html - -
- -.. raw:: html - -
- -Boris, Jp. 1970. “Relativistic Plasma Simulation-Optimization of a Hybrid Code.” In *Proc. Fourth Conf. Num. Sim. Plasmas*, 3–67. Naval Res. Lab., Wash., D. C. - -.. raw:: html - -
- -.. raw:: html - -
- -Cole, J. B. 1997. “A High-Accuracy Realization of the Yee Algorithm Using Non-Standard Finite Differences.” *Ieee Transactions on Microwave Theory and Techniques* 45 (6): 991–96. - -.. raw:: html - -
- -.. raw:: html - -
- -———. 2002. “High-Accuracy Yee Algorithm Based on Nonstandard Finite Differences: New Developments and Verifications.” *Ieee Transactions on Antennas and Propagation* 50 (9): 1185–91. https://doi.org/10.1109/Tap.2002.801268. - -.. raw:: html - -
- -.. raw:: html - -
- -Cowan, Benjamin M, David L Bruhwiler, John R Cary, Estelle Cormier-Michel, and Cameron G R Geddes. 2013. “Generalized algorithm for control of numerical dispersion in explicit time-domain electromagnetic simulations.” *Physical Review Special Topics-Accelerators and Beams* 16 (4). https://doi.org/10.1103/PhysRevSTAB.16.041303. - -.. raw:: html - -
- -.. raw:: html - -
- -Dawson, J M. 1983. “Particle Simulation of Plasmas.” *Reviews of Modern Physics* 55 (2): 403–47. https://doi.org/10.1103/RevModPhys.55.403. - -.. raw:: html - -
- -.. raw:: html - -
- -Esirkepov, Tz. 2001. “Exact Charge Conservation Scheme for Particle-in-Cell Simulation with an Arbitrary Form-Factor.” *Computer Physics Communications* 135 (2): 144–53. - -.. raw:: html - -
- -.. raw:: html - -
- -Haber, I, R Lee, Hh Klein, and Jp Boris. 1973. “Advances in Electromagnetic Simulation Techniques.” In *Proc. Sixth Conf. Num. Sim. Plasmas*, 46–48. Berkeley, Ca. - -.. raw:: html - -
- -.. raw:: html - -
- -Karkkainen, M, E Gjonaj, T Lau, and T Weiland. 2006. “Low-Dispersionwake Field Calculation Tools.” In *Proc. Of International Computational Accelerator Physics Conference*, 35–40. Chamonix, France. - -.. raw:: html - -
- -.. raw:: html - -
- -Langdon, A B. 1992. “On Enforcing Gauss Law in Electromagnetic Particle-in-Cell Codes.” *Computer Physics Communications* 70 (3): 447–50. - -.. raw:: html - -
- -.. raw:: html - -
- -Lehe, R, A Lifschitz, C Thaury, V Malka, and X Davoine. 2013. “Numerical growth of emittance in simulations of laser-wakefield acceleration.” *Physical Review Special Topics-Accelerators and Beams* 16 (2). https://doi.org/10.1103/PhysRevSTAB.16.021301. - -.. raw:: html - -
- -.. raw:: html - -
- -Liu, Qh. 1997. “The PSTD Algorithm: A Time-Domain Method Requiring Only Two Cells Per Wavelength.” *Microwave and Optical Technology Letters* 15 (3): 158–65. `https://doi.org/10.1002/(SICI)1098-2760(19970620)15\:3\<158\:\:AID-MOP11\>3.0.CO\;2-3 3.0.CO\;2-3>`_. - -.. raw:: html - -
- -.. raw:: html - -
- -Marder, B. 1987. “A Method for Incorporating Gauss Law into Electromagnetic Pic Codes.” *Journal of Computational Physics* 68 (1): 48–55. - -.. raw:: html - -
- -.. raw:: html - -
- -Morse, Rl, and Cw Nielson. 1971. “Numerical Simulation of Weibel Instability in One and 2 Dimensions.” *Phys. Fluids* 14 (4): 830 –&. https://doi.org/10.1063/1.1693518. - -.. raw:: html - -
- -.. raw:: html - -
- -Munz, Cd, P Omnes, R Schneider, E Sonnendrucker, and U Voss. 2000. “Divergence Correction Techniques for Maxwell Solvers Based on A Hyperbolic Model.” *Journal of Computational Physics* 161 (2): 484–511. https://doi.org/10.1006/Jcph.2000.6507. - -.. raw:: html - -
- -.. raw:: html - -
- -Ohmura, Y, and Y Okamura. 2010. “Staggered Grid Pseudo-Spectral Time-Domain Method for Light Scattering Analysis.” *Piers Online* 6 (7): 632–35. - -.. raw:: html - -
- -.. raw:: html - -
- -Pukhov, A. 1999. “Three-dimensional electromagnetic relativistic particle-in-cell code VLPL (Virtual Laser Plasma Lab).” *Journal of Plasma Physics* 61 (3): 425–33. https://doi.org/10.1017/S0022377899007515. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean-Luc, Irving Haber, and Brendan B Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243 (June): 260–68. https://doi.org/10.1016/j.jcp.2013.03.010. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, Jean Luc, Irving Haber, and Brendan B. Godfrey. 2013. “A domain decomposition method for pseudo-spectral electromagnetic simulations of plasmas.” *Journal of Computational Physics* 243: 260–68. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L. 2008. “Simulation of Beams or Plasmas Crossing at Relativistic Velocity.” *Physics of Plasmas* 15 (5): 56701. https://doi.org/10.1063/1.2837054. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., J.-C. Adam, and A Heron. 2004. “Asymmetric Pml for the Absorption of Waves. Application to Mesh Refinement in Electromagnetic Particle-in-Cell Plasma Simulations.” *Computer Physics Communications* 164 (1-3): 171–77. https://doi.org/10.1016/J.Cpc.2004.06.026. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., and C Deutsch. 1998. “Charge Compensated Ion Beam Propagation in A Reactor Sized Chamber.” *Physics of Plasmas* 5 (4): 1190–7. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J L, C G R Geddes, E Cormier-Michel, and D P Grote. 2011. “Numerical Methods for Instability Mitigation in the Modeling of Laser Wakefield Accelerators in A Lorentz-Boosted Frame.” *Journal of Computational Physics* 230 (15): 5908–29. https://doi.org/10.1016/J.Jcp.2011.04.003. - -.. raw:: html - -
- -.. raw:: html - -
- -Vay, J.-L., D P Grote, R H Cohen, and A Friedman. 2012. “Novel methods in the particle-in-cell accelerator code-framework warp.” Journal Paper. *Computational Science and Discovery* 5 (1): 014019 (20 pp.). - -.. raw:: html - -
- -.. raw:: html - -
- -Villasenor, J, and O Buneman. 1992. “Rigorous Charge Conservation for Local Electromagnetic-Field Solvers.” *Computer Physics Communications* 69 (2-3): 306–16. - -.. raw:: html - -
- -.. raw:: html - -
- -Vincenti, H., and J.-L. Vay. 2016. “Detailed analysis of the effects of stencil spatial variations with arbitrary high-order finite-difference Maxwell solver.” *Computer Physics Communications* 200 (March). ELSEVIER SCIENCE BV, PO BOX 211, 1000 AE AMSTERDAM, NETHERLANDS: 147–67. https://doi.org/10.1016/j.cpc.2015.11.009. - -.. raw:: html - -
- -.. raw:: html - -
- -Yee, Ks. 1966. “Numerical Solution of Initial Boundary Value Problems Involving Maxwells Equations in Isotropic Media.” *Ieee Transactions on Antennas and Propagation* Ap14 (3): 302–7. - -.. raw:: html - -
- -.. raw:: html - -
diff --git a/Docs/source/usage/examples.rst b/Docs/source/usage/examples.rst index 48d37db98c4..0492372b4e6 100644 --- a/Docs/source/usage/examples.rst +++ b/Docs/source/usage/examples.rst @@ -7,252 +7,132 @@ This section allows you to **download input files** that correspond to different We provide two kinds of inputs: -* AMReX ``inputs`` files, :ref:`with parameters described here `, * PICMI python input files, `with parameters described here `__. +* AMReX ``inputs`` files, :ref:`with parameters described here `, -For a complete list of all example input files, have a look at our ``Examples/`` directory. -It contains folders and subfolders with self-describing names that you can try. All these input files are automatically tested, so they should always be up-to-date. - -Beam-driven electron acceleration ---------------------------------- - -AMReX ``inputs``: - -* :download:`2D case <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_2d_boost>` -* :download:`3D case in boosted frame <../../../Examples/Physics_applications/plasma_acceleration/inputs_3d_boost>` - -PICMI: - -* :download:`Without mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py>` -* :download:`With mesh refinement <../../../Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration_mr.py>` - -Laser-driven electron acceleration ----------------------------------- +For a complete list of all example input files, also have a look at our `Examples/ `__ directory. +It contains folders and subfolders with self-describing names that you can try. +All these input files are automatically tested, so they should always be up-to-date. -AMReX ``inputs``: -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/inputs_1d>` -* :download:`2D case <../../../Examples/Physics_applications/laser_acceleration/inputs_2d>` -* :download:`2D case in boosted frame <../../../Examples/Physics_applications/laser_acceleration/inputs_2d_boost>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/inputs_3d>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/inputs_rz>` +Plasma-Based Acceleration +------------------------- -PICMI (Python) scripts: +.. toctree:: + :maxdepth: 1 -* :download:`1D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_1d.py>` -* :download:`2D case with mesh refinement <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_2d.py>` -* :download:`3D case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py>` -* :download:`RZ case <../../../Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py>` + examples/lwfa/README.rst + examples/pwfa/README.rst + pwfa.rst -Plasma mirror -------------- -:download:`2D case <../../../Examples/Physics_applications/plasma_mirror/inputs_2d>` +Laser-Plasma Interaction +------------------------ -Laser-ion acceleration ----------------------- +.. toctree:: + :maxdepth: 1 -:download:`2D case <../../../Examples/Physics_applications/laser_ion/inputs>` + examples/laser_ion/README.rst + examples/plasma_mirror/README.rst -.. note:: - The resolution of this 2D case is extremely low by default. - You will need a computing cluster for adequate resolution of the target density, see comments in the input file. +Particle Accelerator & Beam Physics +----------------------------------- -.. warning:: +.. toctree:: + :maxdepth: 1 - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. + examples/gaussian_beam/README.rst + examples/beam-beam_collision/README.rst -Uniform plasma --------------- -:download:`2D case <../../../Examples/Physics_applications/uniform_plasma/inputs_2d>` -:download:`3D case <../../../Examples/Physics_applications/uniform_plasma/inputs_3d>` +High Energy Astrophysical Plasma Physics +---------------------------------------- -Capacitive discharge --------------------- +.. toctree:: + :maxdepth: 1 -The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. In particular this can be used to study capacitive discharges between parallel plates. The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `here `__). + examples/ohm_solver_magnetic_reconnection/README.rst -.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png - :alt: MCC benchmark against Turner et. al. (2013). - :width: 80% -An input file to reproduce the benchmark calculations is linked below. -To run a given case ``-n``, from 1 to 4, execute: +Microelectronics +---------------- - .. code-block:: bash +`ARTEMIS (Adaptive mesh Refinement Time-domain ElectrodynaMIcs Solver) `__ is based on WarpX and couples the Maxwell's equations implementation in WarpX with classical equations that describe quantum material behavior (such as, LLG equation for micromagnetics and London equation for superconducting materials) for quantifying the performance of `next-generation microelectronics `__. - python3 PICMI_inputs_1d.py -n 1 +* `ARTEMIS examples `__ +* `ARTEMIS manual `__ -Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot above. Running case 1 on 4 processors takes roughly 20 minutes to complete. -* :download:`input file <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py>` +Nuclear Fusion +-------------- .. note:: - This example needs `additional calibration data for cross sections `__. - Download this data alongside your inputs file and update the paths in the inputs file: - - .. code-block:: bash - - git clone https://github.com/ECP-WarpX/warpx-data.git - -Test cases ----------- + TODO -PICMI (Python) test cases included that can be used as a reference: - -* :download:`Gaussian beam <../../../Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py>` -* :download:`Langmuir plasma wave test in 3d <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rt.py>` -* :download:`Langmuir plasma wave test in RZ <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir_rz_multimode_analyze.py>` -* :download:`Langmuir plasma wave test in 2D <../../../Examples/Tests/langmuir/PICMI_inputs_langmuir2d.py>` - -Manipulating fields via Python ------------------------------- - -An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. - -* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` - -An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. - -* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` - -.. _examples-hybrid-model: -Kinetic-fluid hybrid model +Fundamental Plasma Physics -------------------------- -Several examples and benchmarks of the kinetic-fluid hybrid model are shown below. The first few examples are replications -of the verification tests described in :cite:t:`ex-MUNOZ2018`. -The hybrid-PIC model was added to WarpX in `PR #3665 `_ - the figures below -were generated at that time. +.. toctree:: + :maxdepth: 1 -Electromagnetic modes -^^^^^^^^^^^^^^^^^^^^^ + examples/langmuir/README.rst + examples/capacitive_discharge/README.rst -In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the -:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are -analyzed for mode excitations. -Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except -in a region of thermal resonances as indicated on the plot below. - -.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png - :alt: Parallel EM modes in thermal ion plasma - :width: 70% - -Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. - -.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png - :alt: Perpendicular EM modes in thermal ion plasma - :width: 50% - -The input file for these examples and the corresponding analysis can be found at: - -* :download:`EM modes input <../../../Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_EM_modes/analysis.py>` - -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the parallel propagating or -ion-Bernstein modes as indicated below. - - .. code-block:: bash - - python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y/z} - -A RZ-geometry example case for normal modes propagating along an applied magnetic field in a cylinder is also available. -The analytical solution for these modes are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. - -.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png - :alt: Normal EM modes in a metallic cylinder - :width: 90% - -The input file for this example and corresponding analysis can be found at: - -* :download:`Cylinderical modes input <../../../Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_EM_modes/analysis_rz.py>` - -Ion beam R instability -^^^^^^^^^^^^^^^^^^^^^^ - -In this example a low density ion beam interacts with a "core" plasma population which induces an instability. -Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can -be accessed. The figures below show the evolution of the y-component of the magnetic field as the beam and -core plasma interact. - -.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png - :alt: Resonant ion beam R instability - :width: 70% - -.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png - :alt: Non-resonant ion beam R instability - :width: 70% - -The growth rates of the strongest growing modes for the resonant case are compared -to theory (dashed lines) in the figure below. - -.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 - :alt: Resonant ion beam R instability growth rates - :width: 50% +.. _examples-hybrid-model: -The input file for these examples and the corresponding analysis can be found at: +Kinetic-fluid Hybrid Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* :download:`Ion beam R instability input <../../../Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_beam_instability/analysis.py>` +WarpX includes a reduced plasma model in which electrons are treated as a massless +fluid while ions are kinetically evolved, and Ohm's law is used to calculate +the electric field. This model is appropriate for problems in which ion kinetics +dominate (ion cyclotron waves, for instance). See the +:ref:`theory section ` for more details. Several +examples and benchmarks of this kinetic-fluid hybrid model are provided below. +A few of the examples are replications of the verification tests described in +:cite:t:`ex-MUNOZ2018`. The hybrid-PIC model was added to WarpX in +`PR #3665 `_ - the figures in the +examples below were generated at that time. -The same input script can be used for 1d, 2d or 3d simulations as well as replicating either the resonant or non-resonant -condition as indicated below. +.. toctree:: + :maxdepth: 1 - .. code-block:: bash + examples/ohm_solver_EM_modes/README.rst + examples/ohm_solver_ion_beam_instability/README.rst + examples/ohm_solver_ion_Landau_damping/README.rst - python3 PICMI_inputs.py -dim {1/2/3} --resonant -Ion Landau damping -^^^^^^^^^^^^^^^^^^ +High-Performance Computing and Numerics +--------------------------------------- -Landau damping is a well known process in which electrostatic (acoustic) waves -are damped by transferring energy to particles satisfying a resonance condition. -The process can be simulated by seeding a plasma with a specific acoustic mode -(density perturbation) and tracking the strength of the mode as a function of -time. The figure below shows a set of such simulations with parameters matching -those described in section 4.5 of :cite:t:`ex-MUNOZ2018`. The straight lines show -the theoretical damping rate for the given temperature ratios. +The following examples are commonly used to study the performance of WarpX, e.g., for computing efficiency, scalability, and I/O patterns. +While all prior examples are used for such studies as well, the examples here need less explanation on the physics, less-detail tuning on load balancing, and often simply scale (weak or strong) by changing the number of cells, AMReX block size and number of compute units. -.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png - :alt: Ion Landau damping - :width: 70% +.. toctree:: + :maxdepth: 1 -The input file for these examples and the corresponding analysis can be found at: + examples/uniform_plasma/README.rst -* :download:`Ion Landau damping input <../../../Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py>` -The same input script can be used for 1d, 2d or 3d simulations and to sweep different -temperature ratios. +Manipulating fields via Python +------------------------------ - .. code-block:: bash +.. note:: - python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + TODO: The section needs to be sorted into either science cases (above) or later sections (:ref:`workflows and Python API details `). -Magnetic reconnection -^^^^^^^^^^^^^^^^^^^^^ +An example of using Python to access the simulation charge density, solve the Poisson equation (using ``superLU``) and write the resulting electrostatic potential back to the simulation is given in the input file below. This example uses the ``fields.py`` module included in the ``pywarpx`` library. -Hybrid-PIC codes are often used to simulate magnetic reconnection in space -plasmas. An example of magnetic reconnection from a force-free sheet is -provided, based on the simulation described in :cite:t:`ex-Le2016`. +* :download:`Direct Poisson solver example <../../../Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py>` -.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif - :alt: Magnetic reconnection - :width: 70% +An example of initializing the fields by accessing their data through Python, advancing the simulation for a chosen number of time steps, and plotting the fields again through Python. The simulation runs with 128 regular cells, 8 guard cells, and 10 PML cells, in each direction. Moreover, it uses div(E) and div(B) cleaning both in the regular grid and in the PML and initializes all available electromagnetic fields (E,B,F,G) identically. -The input file for this example and corresponding analysis can be found at: +* :download:`Unit pulse with PML <../../../Examples/Tests/python_wrappers/PICMI_inputs_2d.py>` -* :download:`Magnetic reconnection input <../../../Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py>` -* :download:`Analysis script <../../../Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py>` Many Further Examples, Demos and Tests -------------------------------------- @@ -261,5 +141,8 @@ WarpX runs over 200 integration tests on a variety of modeling cases, which vali Please see the `Examples/Tests/ `__ directory for many more examples. +Example References +------------------ + .. bibliography:: :keyprefix: ex- diff --git a/Docs/source/usage/examples/beam-beam_collision b/Docs/source/usage/examples/beam-beam_collision new file mode 120000 index 00000000000..8c6ac6b30b1 --- /dev/null +++ b/Docs/source/usage/examples/beam-beam_collision @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/beam-beam_collision \ No newline at end of file diff --git a/Docs/source/usage/examples/capacitive_discharge b/Docs/source/usage/examples/capacitive_discharge new file mode 120000 index 00000000000..dd1493f8f70 --- /dev/null +++ b/Docs/source/usage/examples/capacitive_discharge @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/capacitive_discharge \ No newline at end of file diff --git a/Docs/source/usage/examples/gaussian_beam b/Docs/source/usage/examples/gaussian_beam new file mode 120000 index 00000000000..a03c49d4dda --- /dev/null +++ b/Docs/source/usage/examples/gaussian_beam @@ -0,0 +1 @@ +../../../../Examples/Tests/gaussian_beam \ No newline at end of file diff --git a/Docs/source/usage/examples/langmuir b/Docs/source/usage/examples/langmuir new file mode 120000 index 00000000000..f0f0c5d7bc3 --- /dev/null +++ b/Docs/source/usage/examples/langmuir @@ -0,0 +1 @@ +../../../../Examples/Tests/langmuir \ No newline at end of file diff --git a/Docs/source/usage/examples/laser_ion b/Docs/source/usage/examples/laser_ion new file mode 120000 index 00000000000..8d9aa37f076 --- /dev/null +++ b/Docs/source/usage/examples/laser_ion @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_ion \ No newline at end of file diff --git a/Docs/source/usage/examples/lwfa b/Docs/source/usage/examples/lwfa new file mode 120000 index 00000000000..4ec620fb20e --- /dev/null +++ b/Docs/source/usage/examples/lwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/laser_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_EM_modes b/Docs/source/usage/examples/ohm_solver_EM_modes new file mode 120000 index 00000000000..485be7241ae --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_EM_modes @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_EM_modes \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_Landau_damping b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping new file mode 120000 index 00000000000..c70b062da77 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_Landau_damping @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_Landau_damping \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_ion_beam_instability b/Docs/source/usage/examples/ohm_solver_ion_beam_instability new file mode 120000 index 00000000000..3864ca292b5 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_ion_beam_instability @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_ion_beam_instability \ No newline at end of file diff --git a/Docs/source/usage/examples/ohm_solver_magnetic_reconnection b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection new file mode 120000 index 00000000000..1dd74c99f84 --- /dev/null +++ b/Docs/source/usage/examples/ohm_solver_magnetic_reconnection @@ -0,0 +1 @@ +../../../../Examples/Tests/ohm_solver_magnetic_reconnection \ No newline at end of file diff --git a/Docs/source/usage/examples/plasma_mirror b/Docs/source/usage/examples/plasma_mirror new file mode 120000 index 00000000000..46f96c3eb8f --- /dev/null +++ b/Docs/source/usage/examples/plasma_mirror @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_mirror \ No newline at end of file diff --git a/Docs/source/usage/examples/pwfa b/Docs/source/usage/examples/pwfa new file mode 120000 index 00000000000..044974be6ce --- /dev/null +++ b/Docs/source/usage/examples/pwfa @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/plasma_acceleration \ No newline at end of file diff --git a/Docs/source/usage/examples/uniform_plasma b/Docs/source/usage/examples/uniform_plasma new file mode 120000 index 00000000000..fe1c540dfa9 --- /dev/null +++ b/Docs/source/usage/examples/uniform_plasma @@ -0,0 +1 @@ +../../../../Examples/Physics_applications/uniform_plasma \ No newline at end of file diff --git a/Docs/source/usage/how_to_run.rst b/Docs/source/usage/how_to_run.rst index 26c56cb6007..ab5cc6e79bd 100644 --- a/Docs/source/usage/how_to_run.rst +++ b/Docs/source/usage/how_to_run.rst @@ -25,7 +25,7 @@ Where ```` by the actual path to the run directory. ------------- If you installed warpX with a :ref:`package manager `, a ``warpx``-prefixed executable will be available as a regular system command to you. -Depending on the choosen build options, the name is suffixed with more details. +Depending on the chosen build options, the name is suffixed with more details. Try it like this: .. code-block:: bash diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index b9fedb4f050..0c5fe85973f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1,7 +1,11 @@ .. _running-cpp-parameters: -Input Parameters -================ +Parameters: Inputs File +======================= + +This documents on how to use WarpX with an inputs file (e.g., ``warpx.3d input_3d``). + +Complete example input files can be found in :ref:`the examples section `. .. note:: @@ -77,6 +81,43 @@ Overall simulation parameters one should not expect to obtain the same random numbers, even if a fixed ``warpx.random_seed`` is provided. +* ``algo.evolve_scheme`` (`string`, default: `explicit`) + Specifies the evolve scheme used by WarpX. + + * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD + + * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. + Note that this method is for demonstration only. It is inefficient and does not work well when + :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. + For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + + * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. + For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, + and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in + which case charge will also be conserved). + +* ``algo.max_picard_iterations`` (`integer`, default: 10) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard + itearations that are done each time step. + +* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of + the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. + If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last + iteration (for a slight optimization). + +* ``algo.require_picard_convergence`` (`bool`, default: 1) + When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step + is required to converge. + If it is required, an abort is raised if it does not converge and the code then exits. + If not, then a warning is issued and the calculation continues. + * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating the fields at each iteration with the full Maxwell equations, the fields @@ -114,7 +155,7 @@ Overall simulation parameters \qquad \boldsymbol{B} = -\frac{1}{c}\boldsymbol{\beta}\times\boldsymbol{\nabla}\phi where :math:`\boldsymbol{\beta}` is the average (normalized) velocity of the considered species (which can be relativistic). - See e.g. `Vay et al., Physics of Plasmas 15, 056701 (2008) `__ for more information. + See, e.g., :cite:t:`param-Vaypop2008` for more information. See the `AMReX documentation `_ for details of the MLMG solver (the default solver used with electrostatic @@ -151,7 +192,7 @@ Overall simulation parameters This only applies when warpx.do_electrostatic = labframe. * ``warpx.self_fields_verbosity`` (`integer`, default: 2) - The vebosity used for MLMG solver for space-charge fields calculation. Currently + The verbosity used for MLMG solver for space-charge fields calculation. Currently MLMG solver looks for verbosity levels from 0-5. A higher number results in more verbose output. @@ -167,6 +208,12 @@ Overall simulation parameters For all regular WarpX operations, we therefore do explicit memory transfers without the need for managed memory and thus changed the AMReX default to false. `Please also see the documentation in AMReX `__. +* ``amrex.omp_threads`` (``system``, ``nosmt`` or positive integer; default is ``nosmt``) + An integer number can be set in lieu of the ``OMP_NUM_THREADS`` environment variable to control the number of OpenMP threads to use for the ``OMP`` compute backend on CPUs. + By default, we use the ``nosmt`` option, which overwrites the OpenMP default of spawning one thread per logical CPU core, and instead only spawns a number of threads equal to the number of physical CPU cores on the machine. + If set, the environment variable ``OMP_NUM_THREADS`` takes precedence over ``system`` and ``nosmt``, but not over integer numbers set in this option. + + Signal Handling ^^^^^^^^^^^^^^^ @@ -203,7 +250,7 @@ We follow the same naming, but remove the ``SIG`` prefix, e.g., the WarpX signal .. code-block:: bash - #SBATCH --signal=B:1@360 + #SBATCH --signal=1@360 srun ... \ warpx.break_signals=HUP \ @@ -340,7 +387,7 @@ Domain Boundary Conditions * ``Periodic``: This option can be used to set periodic domain boundaries. Note that if the fields for lo in a certain dimension are set to periodic, then the corresponding upper boundary must also be set to periodic. If particle boundaries are not specified in the input file, then particles boundaries by default will be set to periodic. If particles boundaries are specified, then they must be set to periodic corresponding to the periodic field boundaries. - * ``pml`` (default): This option can be used to add Perfectly Matched Layers (PML) around the simulation domain. See the :ref:`PML theory section ` for more details. + * ``pml`` (default): This option can be used to add Perfectly Matched Layers (PML) around the simulation domain. See the :ref:`PML theory section ` for more details. Additional pml algorithms can be explored using the parameters ``warpx.do_pml_in_domain``, ``warpx.pml_has_particles``, and ``warpx.do_pml_j_damping``. * ``absorbing_silver_mueller``: This option can be used to set the Silver-Mueller absorbing boundary conditions. These boundary conditions are simpler and less computationally expensive than the pml, but are also less effective at absorbing the field. They only work with the Yee Maxwell solver. @@ -359,7 +406,7 @@ Domain Boundary Conditions * ``Absorbing``: Particles leaving the boundary will be deleted. - * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistenly set to periodic and both lower and upper boundaries must be periodic. + * ``Periodic``: Particles leaving the boundary will re-enter from the opposite boundary. The field boundary condition must be consistently set to periodic and both lower and upper boundaries must be periodic. * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. @@ -370,7 +417,7 @@ Domain Boundary Conditions * ``boundary.verboncoeur_axis_correction`` (`bool`) optional (default `true`) Whether to apply the Verboncoeur correction on the charge and current density on axis when using RZ. For nodal values (rho and Jz), the cell volume for values on axis is calculated as :math:`\pi*\Delta r^2/4`. - In `Verboncoeur JCP 174, 421-427 (2001) `__, it is shown that using + In :cite:t:`param-VerboncoeurJCP2001`, it is shown that using :math:`\pi*\Delta r^2/3` instead will give a uniform density if the particle density is uniform. Additional PML parameters @@ -635,8 +682,8 @@ Particle initialization * ``particles.rigid_injected_species`` (`strings`, separated by spaces) List of species injected using the rigid injection method. The rigid injection - method is useful when injecting a relativistic particle beam, in boosted-frame - simulation ; see the :ref:`input-output section ` for more details. + method is useful when injecting a relativistic particle beam in boosted-frame + simulations; see the :ref:`input-output section ` for more details. For species injected using this method, particles are translated along the `+z` axis with constant velocity as long as their ``z`` coordinate verifies ``zzinject_plane``, @@ -676,6 +723,19 @@ Particle initialization When ``.xmin`` and ``.xmax`` are set, they delimit the region within which particles are injected. If periodic boundary conditions are used in direction ``i``, then the default (i.e. if the range is not specified) range will be the simulation box, ``[geometry.prob_hi[i], geometry.prob_lo[i]]``. +* ``.injection_sources`` (``list of strings``) optional + Names of additional injection sources. By default, WarpX assumes one injection source per species, hence all of the input + parameters below describing the injection are parameters directly of the species. However, this option allows + additional sources, the names of which are specified here. For each source, the name of the source is added to the + input parameters below. For instance, with ``.injection_sources = source1 source2`` there can be the two input + parameters ``.source1.injection_style`` and ``.source2.injection_style``. + For the parameters of each source, the parameter with the name of the source will be used. + If it is not given, the value of the parameter without the source name will be used. This allows parameters used for all + sources to be specified once. For example, if the ``source1`` and ``source2`` have the same value of ``uz_m``, then it can be + set using ``.uz_m`` instead of setting it for each source. + Note that since by default ``.injection_style = none``, all injection sources can be input this way. + Note that if a moving window is used, the bulk velocity of all of the sources must be the same since it is used when updating the window. + * ``.injection_style`` (`string`; default: ``none``) Determines how the (macro-)particles will be injected in the simulation. The number of particles per cell is always given with respect to the coarsest level (level 0/mother grid), even if particles are immediately assigned to a refined patch. @@ -690,41 +750,70 @@ Particle initialization * ``SingleParticle``: Inject a single macroparticle. This requires the additional parameters: - ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) - ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) - ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) + + * ``.single_particle_pos`` (`3 doubles`, particle 3D position [meter]) + + * ``.single_particle_u`` (`3 doubles`, particle 3D normalized momentum, i.e. :math:`\gamma \beta`) + + * ``.single_particle_weight`` ( `double`, macroparticle weight, i.e. number of physical particles it represents) * ``MultipleParticles``: Inject multiple macroparticles. This requires the additional parameters: - ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) - ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) - ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) - ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) - ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) - ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) - ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) + + * ``.multiple_particles_pos_x`` (list of `doubles`, X positions of the particles [meter]) + + * ``.multiple_particles_pos_y`` (list of `doubles`, Y positions of the particles [meter]) + + * ``.multiple_particles_pos_z`` (list of `doubles`, Z positions of the particles [meter]) + + * ``.multiple_particles_ux`` (list of `doubles`, X normalized momenta of the particles, i.e. :math:`\gamma \beta_x`) + + * ``.multiple_particles_uy`` (list of `doubles`, Y normalized momenta of the particles, i.e. :math:`\gamma \beta_y`) + + * ``.multiple_particles_uz`` (list of `doubles`, Z normalized momenta of the particles, i.e. :math:`\gamma \beta_z`) + + * ``.multiple_particles_weight`` (list of `doubles`, macroparticle weights, i.e. number of physical particles each represents) * ``gaussian_beam``: Inject particle beam with gaussian distribution in space in all directions. This requires additional parameters: - ``.q_tot`` (beam charge), - ``.npart`` (number of particles in the beam), - ``.x/y/z_m`` (average position in `x/y/z`), - ``.x/y/z_rms`` (standard deviation in `x/y/z`), - ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), - and optional arguments ``.do_symmetrize`` (whether to - symmetrize the beam) and ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + + * ``.q_tot`` (beam charge), + + * ``.npart`` (number of macroparticles in the beam), + + * ``.x/y/z_m`` (average position in `x/y/z`), + + * ``.x/y/z_rms`` (standard deviation in `x/y/z`), + + There are additional optional parameters: + + * ``.x/y/z_cut`` (optional, particles with ``abs(x-x_m) > x_cut*x_rms`` are not injected, same for y and z. ``.q_tot`` is the charge of the un-cut beam, so that cutting the distribution is likely to result in a lower total charge), + * ``.do_symmetrize`` (optional, whether to symmetrize the beam) + + * ``.symmetrization_order`` (order of symmetrization, default is 4, can be 4 or 8). + If ``.do_symmetrize`` is 0, no symmetrization occurs. If ``.do_symmetrize`` is 1, then the beam is symmetrized according to the value of ``.symmetrization_order``. If set to 4, symmetrization is in the x and y direction, (x,y) (-x,y) (x,-y) (-x,-y). If set to 8, symmetrization is also done with x and y exchanged, (y,x), (-y,x), (y,-x), (-y,-x)). + * ``.focal_distance`` (optional, distance between the beam centroid and the position of the focal plane of the beam, along the direction of the beam mean velocity; space charge is ignored in the initialization of the particles) + + If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the size of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: - ``.injection_file`` (`string`) openPMD file name and - ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. - ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. - ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + + * ``.injection_file`` (`string`) openPMD file name and + + * ``.charge`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.mass`` (`double`) optional (default is read from openPMD file) when set this will be the charge of the physical particle represented by the injected macroparticles. + + * ``.z_shift`` (`double`) optional (default is no shift) when set this value will be added to the longitudinal, ``z``, position of the particles. + + * ``.impose_t_lab_from_file`` (`bool`) optional (default is false) only read if warpx.gamma_boost > 1., it allows to set t_lab for the Lorentz Transform as being the time stored in the openPMD file. + Warning: ``q_tot!=0`` is not supported with the ``external_file`` injection style. If a value is provided, it is ignored and no re-scaling is done. The external file must include the species ``openPMD::Record`` labeled ``position`` and ``momentum`` (`double` arrays), with dimensionality and units set via ``openPMD::setUnitDimension`` and ``setUnitSI``. If the external file also contains ``openPMD::Records`` for ``mass`` and ``charge`` (constant `double` scalars) then the species will use these, unless overwritten in the input file (see ``.mass``, ``.charge`` or ``.species_type``). @@ -733,13 +822,20 @@ Particle initialization * ``NFluxPerCell``: Continuously inject a flux of macroparticles from a planar surface. This requires the additional parameters: - ``.flux_profile`` (see the description of this parameter further below) - ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) - ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) - ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) - ``.num_particles_per_cell`` (`double`) - ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) - ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) + + * ``.flux_profile`` (see the description of this parameter further below) + + * ``.surface_flux_pos`` (`double`, location of the injection plane [meter]) + + * ``.flux_normal_axis`` (`x`, `y`, or `z` for 3D, `x` or `z` for 2D, or `r`, `t`, or `z` for RZ. When `flux_normal_axis` is `r` or `t`, the `x` and `y` components of the user-specified momentum distribution are interpreted as the `r` and `t` components respectively) + + * ``.flux_direction`` (`-1` or `+1`, direction of flux relative to the plane) + + * ``.num_particles_per_cell`` (`double`) + + * ``.flux_tmin`` (`double`, Optional time at which the flux will be turned on. Ignored when negative.) + + * ``.flux_tmax`` (`double`, Optional time at which the flux will be turned off. Ignored when negative.) * ``none``: Do not inject macro-particles (for example, in a simulation that starts with neutral, ionizable atoms, one may want to create the electrons species -- where ionized electrons can be stored later on -- without injecting electron macro-particles). @@ -838,7 +934,7 @@ Particle initialization ``.ux``, ``.uy`` and ``.uz``, the normalized momenta in the x, y and z direction respectively, which are all ``0.`` by default. - * ``uniform``: uniform probability distribution between a minumum and a maximum value. + * ``uniform``: uniform probability distribution between a minimum and a maximum value. The x, y and z directions are sampled independently and the final momentum space is a cuboid. The parameters that control the minimum and maximum domain of the distribution are ``.u_min`` and ``.u_max`` in each @@ -1065,15 +1161,21 @@ Particle initialization the above mentioned function. .. note:: - When accessing the data via Python, the scraped particle buffer relies on the user - to clear the buffer after processing the data. The - buffer will grow unbounded as particles are scraped and therefore could - lead to memory issues if not periodically cleared. To clear the buffer - call ``warpx_clearParticleBoundaryBuffer()``. + + When accessing the data via Python, the scraped particle buffer relies on the user + to clear the buffer after processing the data. The + buffer will grow unbounded as particles are scraped and therefore could + lead to memory issues if not periodically cleared. To clear the buffer + call ``clear_buffer()``. * ``.do_field_ionization`` (`0` or `1`) optional (default `0`) Do field ionization for this species (using the ADK theory). +* ``.do_adk_correction`` (`0` or `1`) optional (default `0`) + Whether to apply the correction to the ADK theory proposed by Zhang, Lan and Lu in `Q. Zhang et al. (Phys. Rev. A 90, 043410, 2014) `__. + If so, the probability of ionization is modified using an empirical model that should be more accurate in the regime of high electric fields. + Currently, this is only implemented for Hydrogen, although Argon is also available in the same reference. + * ``.physical_element`` (`string`) Only read if `do_field_ionization = 1`. Symbol of chemical element for this species. Example: for Helium, use ``physical_element = He``. @@ -1138,7 +1240,7 @@ Particle initialization The algorithm used for resampling. Currently there is only one option, which is already set by default: - * ``leveling_thinning`` This algorithm is defined in `Muraviev et al., arXiv:2006.08593 (2020) `_. + * ``leveling_thinning`` This algorithm is defined in :cite:t:`param-MuravievCPC2021`. It has two parameters: * ``.resampling_algorithm_target_ratio`` (`float`) optional (default `1.5`) @@ -1166,7 +1268,7 @@ Cold Relativistic Fluid initialization * ``fluids.species_names`` (`strings`, separated by spaces) Defines the names of each fluid species. It is a required input to create and evolve fluid species using the cold relativistic fluid equations. Most of the parameters described in the section "Particle initialization" can also be used to initialize fluid properties (e.g. initial density distribution). - For fluid-specific inputs we use `` as a placeholder. Also see external fields + For fluid-specific inputs we use `` as a placeholder. Also see external fields for how to specify these for fluids as the function names differ. .. _running-cpp-parameters-laser: @@ -1219,7 +1321,7 @@ Laser initialization be parallel to ``warpx.boost_direction``, for now. * ``.e_max`` (`float` ; in V/m) - Peak amplitude of the laser field. + Peak amplitude of the laser field, in the focal plane. For a laser with a wavelength :math:`\lambda = 0.8\,\mu m`, the peak amplitude is related to :math:`a_0` by: @@ -1233,9 +1335,9 @@ Laser initialization perform the conversion to the boosted frame. * ``.a0`` (`float` ; dimensionless) - Peak normalized amplitude of the laser field (given in the lab frame, just as ``e_max`` above). + Peak normalized amplitude of the laser field, in the focal plane (given in the lab frame, just as ``e_max`` above). See the description of ``.e_max`` for the conversion between ``a0`` and ``e_max``. - Exactly one of ``a0`` and ``e_max`` must be specified. + Either ``a0`` or ``e_max`` must be specified. * ``.wavelength`` (`float`; in meters) The wavelength of the laser in vacuum. @@ -1284,14 +1386,16 @@ Laser initialization ``.e_max`` (i.e. in most cases the maximum of abs(E(x,y,t)) should be 1, so that the maximum field intensity can be set straightforwardly with ``.e_max``). The binary file has to respect the following format: - * flag to indicate the grid is uniform(1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported - * nt, number of timesteps (uint32_t, must be >=2) - * nx, number of points along x (uint32_t, must be >=2) - * ny, number of points along y (uint32_t, must be 1 for 2D simulations and >=2 for 3D simulations) - * timesteps (double[2]=[t_min,t_max]) - * x_coords (double[2]=[x_min,x_max]) - * y_coords (double[1] in 2D, double[2]=[y_min,y_max] in 3D) - * field_data (double[nt * nx * ny], with nt being the slowest coordinate). + + * ``flag`` to indicate the grid is uniform (1 byte, 0 means non-uniform, !=0 means uniform) - only uniform is supported + * ``nt``, number of timesteps (``uint32_t``, must be >=2) + * ``nx``, number of points along x (``uint32_t``, must be >=2) + * ``ny``, number of points along y (``uint32_t``, must be 1 for 2D simulations and >=2 for 3D simulations) + * ``timesteps`` (``double[2]=[t_min,t_max]``) + * ``x_coords`` (``double[2]=[x_min,x_max]``) + * ``y_coords`` (``double[1]`` in 2D, ``double[2]=[y_min,y_max]`` in 3D) + * ``field_data`` (``double[nt x nx * ny]``, with ``nt`` being the slowest coordinate). + A binary file can be generated from Python, see an example at ``Examples/Tests/laser_injection_from_file`` * ``.profile_t_peak`` (`float`; in seconds) @@ -1311,12 +1415,15 @@ Laser initialization Note that :math:`\tau` relates to the full width at half maximum (FWHM) of *intensity*, which is closer to pulse length measurements in experiments, as :math:`\tau = \mathrm{FWHM}_I / \sqrt{2\ln(2)}` :math:`\approx \mathrm{FWHM}_I / 1.1774`. + For a chirped laser pulse (i.e. with a non-zero ``.phi2``), ``profile_duration`` is the Fourier-limited duration of the pulse, not the actual duration of the pulse. See the documentation for ``.phi2`` for more detail. + When running a **boosted-frame simulation**, provide the value of ``.profile_duration`` in the laboratory frame, and use ``warpx.gamma_boost`` to automatically perform the conversion to the boosted frame. * ``.profile_waist`` (`float` ; in meters) - The waist of the transverse Gaussian laser profile, defined as :math:`w_0` : + The waist of the transverse Gaussian :math:`w_0`, i.e. defined such that the electric field of the + laser pulse in the focal plane is of the form: .. math:: @@ -1338,19 +1445,36 @@ Laser initialization * ``.stc_direction`` (`3 floats`) optional (default `1. 0. 0.`) Direction of laser spatio-temporal couplings. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + See definition in :cite:t:`param-AkturkOE2004`. * ``.zeta`` (`float`; in meters.seconds) optional (default `0.`) Spatial chirp at focus in direction ``.stc_direction``. See definition in - Akturk et al., Opt Express, vol 12, no 19 (2004). + :cite:t:`param-AkturkOE2004`. * ``.beta`` (`float`; in seconds) optional (default `0.`) Angular dispersion (or angular chirp) at focus in direction ``.stc_direction``. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + See definition in :cite:t:`param-AkturkOE2004`. * ``.phi2`` (`float`; in seconds**2) optional (default `0.`) - Temporal chirp at focus. - See definition in Akturk et al., Opt Express, vol 12, no 19 (2004). + The amount of temporal chirp :math:`\phi^{(2)}` at focus (in the lab frame). Namely, a wave packet + centered on the frequency :math:`(\omega_0 + \delta \omega)` will reach its peak intensity + at :math:`z(\delta \omega) = z_0 - c \phi^{(2)} \, \delta \omega`. Thus, a positive + :math:`\phi^{(2)}` corresponds to positive chirp, i.e. red part of the spectrum in the + front of the pulse and blue part of the spectrum in the back. More specifically, the electric + field in the focal plane is of the form: + + .. math:: + + E(\boldsymbol{x},t) \propto Re\left[ \exp\left( -\frac{(t-t_{peak})^2}{\tau^2 + 2i\phi^{(2)}} + i\omega_0 (t-t_{peak}) + i\phi_0 \right) \right] + + where :math:`\tau` is given by ``.profile_duration`` and represents the + Fourier-limited duration of the laser pulse. Thus, the actual duration of the chirped laser pulse is: + + .. math:: + + \tau' = \sqrt{ \tau^2 + 4 (\phi^{(2)})^2/\tau^2 } + + See also the definition in :cite:t:`param-AkturkOE2004`. * ``.do_continuous_injection`` (`0` or `1`) optional (default `0`). Whether or not to use continuous injection. @@ -1395,9 +1519,15 @@ Laser initialization External fields --------------- -Grid initialization +Applied to the grid ^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_grid_init_`` or ``warpx.E_ext_grid_init_`` +are applied to the grid directly. In particular, these fields can be seen in the diagnostics that output the fields on the grid. + + - When using an **electromagnetic** field solver, these fields are applied to the grid at the beginning of the simulation, and serve as initial condition for the Maxwell solver. + - When using an **electrostatic** or **magnetostatic** field solver, these fields are added to the fields computed by the Poisson solver, at each timestep. + * ``warpx.B_ext_grid_init_style`` (string) optional This parameter determines the type of initialization for the external magnetic field. By default, the @@ -1484,6 +1614,9 @@ Grid initialization Applied to Particles ^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_particle_init_`` or ``warpx.E_ext_particle_init_`` +are applied to the particles directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``particles.E_ext_particle_init_style`` & ``particles.B_ext_particle_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the particles at every timestep. @@ -1544,6 +1677,9 @@ Applied to Particles Applied to Cold Relativistic Fluids ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The external fields defined with input parameters that start with ``warpx.B_ext_init_`` or ``warpx.E_ext_init_`` +are applied to the fluids directly, at each timestep. As a results, these fields **cannot** be seen in the diagnostics that output the fields on the grid. + * ``.E_ext_init_style`` & ``.B_ext_init_style`` (string) optional (default "none") These parameters determine the type of the external electric and magnetic fields respectively that are applied directly to the cold relativistic fluids at every timestep. @@ -1637,6 +1773,7 @@ Collision models ---------------- WarpX provides several particle collision models, using varying degrees of approximation. +Details about the collision models can be found in the :ref:`theory section `. * ``collisions.collision_names`` (`strings`, separated by spaces) The name of each collision type. @@ -1646,29 +1783,29 @@ WarpX provides several particle collision models, using varying degrees of appro * ``.type`` (`string`) optional The type of collision. The types implemented are: - - ``pairwisecoulomb`` for pairwise Coulomb collisions, the default if unspecified. + - ``pairwisecoulomb`` for pair-wise Coulomb collisions, the default if unspecified. This provides a pair-wise relativistic elastic Monte Carlo binary Coulomb collision model, - following the algorithm given by `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + following the algorithm given by :cite:t:`param-PerezPOP2012`. When the RZ mode is used, `warpx.n_rz_azimuthal_modes` must be set to 1 at the moment, since the current implementation of the collision module assumes axisymmetry. - ``nuclearfusion`` for fusion reactions. - This implements the pair-wise fusion model by `Higginson et al. (JCP 388, 439-453, 2019) `__. + This implements the pair-wise fusion model by :cite:t:`param-HigginsonJCP2019`. Currently, WarpX supports deuterium-deuterium, deuterium-tritium, deuterium-helium and proton-boron fusion. When initializing the reactant and product species, you need to use ``species_type`` (see the documentation for this parameter), so that WarpX can identify the type of reaction to use. (e.g. ``.species_type = 'deuterium'``) + - ``dsmc`` for pair-wise, non-Coulomb collisions between kinetic species. + This is a "direct simulation Monte Carlo" treatment of collisions between + kinetic species. See :ref:`DSMC section `. - ``background_mcc`` for collisions between particles and a neutral background. This is a relativistic Monte Carlo treatment for particles colliding - with a neutral background gas. The implementation follows the so-called - null collision strategy discussed for example in `Birdsall (IEEE Transactions on - Plasma Science, vol. 19, no. 2, pp. 65-85, 1991) `_. - See also :ref:`collisions section `. + with a neutral background gas. See :ref:`MCC section `. - ``background_stopping`` for slowing of ions due to collisions with electrons or ions. This implements the approximate formulae as derived in Introduction to Plasma Physics, from Goldston and Rutherford, section 14.2. * ``.species`` (`strings`) - If using ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, + If using ``dsmc``, ``pairwisecoulomb`` or ``nuclearfusion``, this should be the name(s) of the species, between which the collision will be considered. (Provide only one name for intra-species collisions.) If using ``background_mcc`` or ``background_stopping`` type this should be the name of the species for which collisions with a background will be included. @@ -1691,7 +1828,7 @@ WarpX provides several particle collision models, using varying degrees of appro :math:`A` is the mass number. If this is not provided, or if a non-positive value is provided, a Coulomb logarithm will be computed automatically according to the algorithm in - `Perez et al. (Phys. Plasmas 19, 083104, 2012) `__. + :cite:t:`param-PerezPOP2012`. * ``.fusion_multiplier`` (`float`) optional. Only for ``nuclearfusion``. @@ -1700,12 +1837,12 @@ WarpX provides several particle collision models, using varying degrees of appro total number of physical particle remains the same). This can improve the statistics of the simulation, in the case where fusion reactions are very rare. More specifically, in a fusion reaction between two macroparticles with weight ``w_1`` and ``w_2``, - the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multipler``. + the weight of the product macroparticles will be ``min(w_1,w_2)/fusion_multiplier``. (And the weights of the reactant macroparticles are reduced correspondingly after the reaction.) - See `Higginson et al. (JCP 388, 439-453, 2019) `__ - for more details. The default value of ``fusion_multiplier`` is 1. + See :cite:t:`param-HigginsonJCP2019` for more details. + The default value of ``fusion_multiplier`` is 1. -* ``.fusion_probability_threshold``(`float`) optional. +* ``.fusion_probability_threshold`` (`float`) optional. Only for ``nuclearfusion``. If the fusion multiplier is too high and results in a fusion probability that approaches 1 (for a given collision between two macroparticles), then @@ -1773,25 +1910,21 @@ WarpX provides several particle collision models, using varying degrees of appro where :math:`\beta` is the term on the r.h.s except :math:`W_b`. * ``.scattering_processes`` (`strings` separated by spaces) - Only for ``background_mcc``. The MCC scattering processes that should be + Only for ``dsmc`` and ``background_mcc``. The scattering processes that should be included. Available options are ``elastic``, ``back`` & ``charge_exchange`` for ions and ``elastic``, ``excitationX`` & ``ionization`` for electrons. - The ``elastic`` option uses hard-sphere scattering, with a differential - cross section that is independent of angle. - With ``charge_exchange``, the ion velocity is replaced with the neutral - velocity, chosen from a Maxwellian based on the value of - ``.background_temperature``. Multiple excitation events can be included for electrons corresponding to excitation to different levels, the ``X`` above can be changed to a unique identifier for each excitation process. For each scattering process specified - a path to a cross-section data file must also be given. We use + a path to a cross-section data file must also be given. We use ```` as a placeholder going forward. * ``._cross_section`` (`string`) - Only for ``background_mcc``. Path to the file containing cross-section data + Only for ``dsmc`` and ``background_mcc``. Path to the file containing cross-section data for the given scattering processes. The cross-section file must have exactly 2 columns of data, the first containing equally spaced energies in eV and the - second the corresponding cross-section in :math:`m^2`. + second the corresponding cross-section in :math:`m^2`. The energy column should + represent the kinetic energy of the colliding particles in the center-of-mass frame. * ``._energy`` (`float`) Only for ``background_mcc``. If the scattering process is either @@ -1866,12 +1999,12 @@ Particle push, charge and current deposition, field gathering 2. ``esirkepov`` The current density is deposited as described in - `(Esirkepov, CPC, 2001) `_. + :cite:t:`param-Esirkepovcpc01`. This deposition scheme guarantees charge conservation for shape factors of arbitrary order. 3. ``vay`` - The current density is deposited as described in `(Vay et al, 2013) `_ (see section :ref:`current_deposition` for more details). + The current density is deposited as described in :cite:t:`param-VayJCP2013` (see section :ref:`current_deposition` for more details). This option guarantees charge conservation only when used in combination with ``psatd.periodic_single_box_fft=1``, that is, only for periodic single-box simulations with global FFTs without guard cells. The implementation for domain @@ -1898,13 +2031,13 @@ Particle push, charge and current deposition, field gathering The algorithm for the particle pusher. Available options are: - ``boris``: Boris pusher. - - ``vay``: Vay pusher (see `Vay, Phys. Plasmas (2008) `__) - - ``higuera``: Higuera-Cary pusher (see `Higuera and Cary, Phys. Plasmas (2017) `__) + - ``vay``: Vay pusher (see :cite:t:`param-Vaypop2008`) + - ``higuera``: Higuera-Cary pusher (see :cite:t:`param-HigueraPOP2017`) If ``algo.particle_pusher`` is not specified, ``boris`` is the default. -* ``algo.particle_shape`` (`integer`; `1`, `2`, or `3`) - The order of the shape factors (splines) for the macro-particles along all spatial directions: `1` for linear, `2` for quadratic, `3` for cubic. +* ``algo.particle_shape`` (`integer`; `1`, `2`, `3`, or `4`) + The order of the shape factors (splines) for the macro-particles along all spatial directions: `1` for linear, `2` for quadratic, `3` for cubic, `4` for quartic. Low-order shape factors result in faster simulations, but may lead to more noisy results. High-order shape factors are computationally more expensive, but may increase the overall accuracy of the results. For production runs it is generally safer to use high-order shape factors, such as cubic order. @@ -1921,9 +2054,9 @@ Two families of Maxwell solvers are implemented in WarpX, based on the Finite-Di - ``yee``: Yee FDTD solver. - ``ckc``: (not available in ``RZ`` geometry) Cole-Karkkainen solver with Cowan - coefficients (see `Cowan, PRSTAB 16 (2013) `_). + coefficients (see :cite:t:`param-CowanPRSTAB13`). - ``psatd``: Pseudo-spectral solver (see :ref:`theory `). - - ``ect``: Enlarged cell technique (conformal finite difference solver. See `Xiao and Liu, IEEE Antennas and Propagation Society International Symposium (2005) `_). + - ``ect``: Enlarged cell technique (conformal finite difference solver. See :cite:t:`param-XiaoIEEE2005`). - ``hybrid``: The E-field will be solved using Ohm's law and a kinetic-fluid hybrid model (see :ref:`theory `). - ``none``: No field solve will be performed. @@ -1960,14 +2093,14 @@ Maxwell solver: PSATD method If true, a current correction scheme in Fourier space is applied in order to guarantee charge conservation. The default value is ``psatd.current_correction=1``, unless a charge-conserving current deposition scheme is used (by setting ``algo.current_deposition=esirkepov`` or ``algo.current_deposition=vay``) or unless the ``div(E)`` cleaning scheme is used (by setting ``warpx.do_dive_cleaning=1``). - If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in (`Vay et al, JCP 243, 2013 `_) and the current correction reads + If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in :cite:t:`param-VayJCP2013` and the current correction reads .. math:: \widehat{\boldsymbol{J}}^{\,n+1/2}_{\mathrm{correct}} = \widehat{\boldsymbol{J}}^{\,n+1/2} - \bigg(\boldsymbol{k}\cdot\widehat{\boldsymbol{J}}^{\,n+1/2} - i \frac{\widehat{\rho}^{n+1} - \widehat{\rho}^{n}}{\Delta{t}}\bigg) \frac{\boldsymbol{k}}{k^2} - If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in (`Lehe et al, PRE 94, 2016 `_) and the current correction reads + If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in :cite:t:`param-LehePRE2016` and the current correction reads .. math:: \widehat{\boldsymbol{J}}^{\,n+1/2}_{\mathrm{correct}} = \widehat{\boldsymbol{J}}^{\,n+1/2} @@ -1983,7 +2116,7 @@ Maxwell solver: PSATD method If false, instead, the update equation for the electric field is expressed in terms of the current density :math:`\widehat{\boldsymbol{J}}^{\,n+1/2}` only. If charge is expected to be conserved (by setting, for example, ``psatd.current_correction=1``), then the two formulations are expected to be equivalent. - If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in (`Vay et al, JCP 243, 2013 `_): + If ``psatd.v_galilean`` is zero, the spectral solver used is the standard PSATD scheme described in :cite:t:`param-VayJCP2013`: 1. if ``psatd.update_with_rho=0``, the update equation for the electric field reads @@ -2009,9 +2142,9 @@ Maxwell solver: PSATD method \frac{1}{\Delta{t}}\right)\widehat{\rho}^{n+1} \boldsymbol{k} \end{split} - The coefficients :math:`C` and :math:`S` are defined in (`Vay et al, JCP 243, 2013 `_). + The coefficients :math:`C` and :math:`S` are defined in :cite:t:`param-VayJCP2013`. - If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in (`Lehe et al, PRE 94, 2016 `_): + If ``psatd.v_galilean`` is non-zero, the spectral solver used is the Galilean PSATD scheme described in :cite:t:`param-LehePRE2016`: 1. if ``psatd.update_with_rho=0``, the update equation for the electric field reads @@ -2041,7 +2174,7 @@ Maxwell solver: PSATD method - i \, \frac{\chi_2}{\epsilon_0 k^{2}} \widehat{\rho}^{\,n+1} \boldsymbol{k} \end{split} - The coefficients :math:`C`, :math:`S`, :math:`\theta`, :math:`\nu`, :math:`\chi_1`, :math:`\chi_2`, and :math:`\chi_3` are defined in (`Lehe et al, PRE 94, 2016 `_). + The coefficients :math:`C`, :math:`S`, :math:`\theta`, :math:`\nu`, :math:`\chi_1`, :math:`\chi_2`, and :math:`\chi_3` are defined in :cite:t:`param-LehePRE2016`. The default value for ``psatd.update_with_rho`` is ``1`` if ``psatd.v_galilean`` is non-zero and ``0`` otherwise. The option ``psatd.update_with_rho=0`` is not implemented with the following algorithms: @@ -2107,25 +2240,28 @@ Maxwell solver: kinetic-fluid hybrid ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * ``hybrid_pic_model.elec_temp`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the electron temperature, in eV, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.n0_ref`` (`float`) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the reference density, in :math:`m^{-3}`, used to calculate + the electron pressure (see :ref:`here `). * ``hybrid_pic_model.gamma`` (`float`) optional (default ``5/3``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate - the electron pressure (see :ref:`here `). + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the exponent used to calculate + the electron pressure (see :ref:`here `). -* ``hybrid_pic_model.plasma_resistivity(rho)`` (`float` or `str`) optional (default ``0``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. +* ``hybrid_pic_model.plasma_resistivity(rho,J)`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. + +* ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. * ``hybrid_pic_model.n_floor`` (`float`) optional (default ``1``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma density floor, in :math:`m^{-3}`, which is useful since the generalized Ohm's law used to calculate the E-field includes a :math:`1/n` term. -* ``hybrid_pic_model.substeps`` (`int`) optional (default ``100``) - If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. +* ``hybrid_pic_model.substeps`` (`int`) optional (default ``10``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the number of sub-steps to take during the B-field update. .. note:: @@ -2149,7 +2285,7 @@ Grid types (collocated, staggered, hybrid) Whether to use a Galerkin scheme when gathering fields to particles. When set to ``1``, the interpolation orders used for field-gathering are reduced for certain field components along certain directions. For example, :math:`E_z` is gathered using ``algo.particle_shape`` along :math:`(x,y)` and ``algo.particle_shape - 1`` along :math:`z`. - See equations (21)-(23) of (`Godfrey and Vay, 2013 `_) and associated references for details. + See equations (21)-(23) of :cite:t:`param-Godfrey2013` and associated references for details. Default: ``interpolation.galerkin_scheme = 0`` with collocated grids and/or momentum-conserving field gathering, ``interpolation.galerkin_scheme = 1`` otherwise. @@ -2208,9 +2344,8 @@ Additional parameters is performed at every timestep regardless of this parameter. * ``warpx.use_hybrid_QED`` (`bool`; default: 0) - Will use the Hybird QED Maxwell solver when pushing fields: a QED correction is added to the - field solver to solve non-linear Maxwell's equations, according to [Quantum Electrodynamics - vacuum polarization solver, P. Carneiro et al., `ArXiv 2016 `__]. + Will use the Hybrid QED Maxwell solver when pushing fields: a QED correction is added to the + field solver to solve non-linear Maxwell's equations, according to :cite:t:`param-GrismayerNJP2021`. Note that this option can only be used with the PSATD build. Furthermore, one must set ``warpx.grid_type = collocated`` (which otherwise would be ``staggered`` by default). @@ -2240,8 +2375,8 @@ Additional parameters ``idx_type = {0, 0, 0}``: Sort particles to a cell centered grid ``idx_type = {1, 1, 1}``: Sort particles to a node centered grid ``idx_type = {2, 2, 2}``: Compromise between a cell and node centered grid. - In 2D (XZ and RZ), only the first two elements are read. - In 1D, only the first element is read. + In 2D (XZ and RZ), only the first two elements are read. + In 1D, only the first element is read. * ``warpx.sort_bin_size`` (list of `int`) optional (default ``1 1 1``) If ``sort_intervals`` is activated and ``sort_particles_for_deposition`` is ``false``, particles are sorted in bins of ``sort_bin_size`` cells. @@ -2270,7 +2405,7 @@ Additional parameters * ``warpx.shared_tilesize`` (list of `int`) optional (default `6 6 8` in 3D; `14 14` in 2D; `1s` otherwise) Used to tune performance when ``do_shared_mem_current_deposition`` or - ``do_shared_mem_charge_depostion`` is enabled. ``shared_tilesize`` is the + ``do_shared_mem_charge_deposition`` is enabled. ``shared_tilesize`` is the size of the temporary buffer allocated in shared memory for a threadblock. A larger tilesize requires more shared memory, but gives more work to each threadblock, which can lead to higher occupancy, and allows for more @@ -2472,8 +2607,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a Only works with ``.format = plotfile``. * ``.coarsening_ratio`` (list of `int`) optional (default `1 1 1`) - Reduce size of the field output by this ratio in each dimension. - (This is done by averaging the field over 1 or 2 points along each direction, depending on the staggering). + Reduce size of the selected diagnostic fields output by this ratio in each dimension. + (For a ratio of N, this is done by averaging the fields over N or (N+1) points depending on the staggering). If ``blocking_factor`` and ``max_grid_size`` are used for the domain decomposition, as detailed in the :ref:`domain decomposition ` section, ``coarsening_ratio`` should be an integer divisor of ``blocking_factor``. If ``warpx.numprocs`` is used instead, the total number of cells in a given @@ -2621,8 +2756,13 @@ The data collected at each boundary is written out to a subdirectory of the diag By default, all of the collected particle data is written out at the end of the simulation. Optionally, the ``.intervals`` parameter can be given to specify writing out the data more often. This can be important if a large number of particles are lost, avoiding filling up memory with the accumulated lost particle data. -In addition to their usual attributes, the saved particles have an integer attribute ``timestamp``, which -indicates the PIC iteration at which each particle was absorbed at the boundary. +In addition to their usual attributes, the saved particles have + an integer attribute ``stepScraped``, which indicates the PIC iteration at which each particle was absorbed at the boundary, + a real attribute ``deltaTimeScraped``, which indicates the time between the time associated to `stepScraped` + and the exact time when each particle hits the boundary. + 3 real attributes ``nx``, ``ny``, ``nz``, which represents the three components of the normal to the boundary on the point of contact of the particles (not saved if they reach non-EB boundaries) + +``BoundaryScrapingDiagnostics`` can be used with ``..random_fraction``, ``..uniform_stride``, and ``..plot_filter_function``, which have the same behavior as for ``FullDiagnostics``. For ``BoundaryScrapingDiagnostics``, these filters are applied at the time the data is written to file. An implication of this is that more particles may initially be accumulated in memory than are ultimately written. ``t`` in ``plot_filter_function`` refers to the time the diagnostic is written rather than the time the particle crossed the boundary. .. _running-cpp-parameters-diagnostics-reduced: @@ -2755,8 +2895,9 @@ Reduced Diagnostics defaulting to ``1``. In RZ geometry, this only saves the 0'th azimuthal mode component of the fields. - Integrated electric and magnetic field components can instead be obtained by specifying + Time integrated electric and magnetic field components can instead be obtained by specifying ``.integrate = true``. + The integration is done every time step even when the data is written out less often. In a *moving window* simulation, the FieldProbe can be set to follow the moving frame by specifying ``.do_moving_window_FP = 1`` (default 0). .. warning:: @@ -2970,6 +3111,9 @@ Reduced Diagnostics A species name must be provided, such that the diagnostics are done for this species. + * ``.file_min_digits`` (`int`) optional (default `6`) + The minimum number of digits used for the iteration number appended to the diagnostic file names. + * ``.histogram_function_abs(t,x,y,z,ux,uy,uz,w)`` (`string`) A histogram function must be provided for the abscissa axis. `t` represents the physical time in seconds during the simulation. @@ -3262,7 +3406,7 @@ The checkpoint capability can be turned with regular diagnostics: ``. Name of the checkpoint file to restart from. Returns an error if the folder does not exist or if it is not properly formatted. -* ``warpx.write_diagonstics_on_restart`` (`bool`) optional (default `false`) +* ``warpx.write_diagnostics_on_restart`` (`bool`) optional (default `false`) When `true`, write the diagnostics after restart at the time of the restart. Intervals parser diff --git a/Docs/source/usage/pwfa.rst b/Docs/source/usage/pwfa.rst index 44d62384699..5119184089c 100644 --- a/Docs/source/usage/pwfa.rst +++ b/Docs/source/usage/pwfa.rst @@ -1,3 +1,5 @@ +.. _examples-pwfa-boost: + In-Depth: PWFA ============== @@ -42,7 +44,7 @@ Listed below are the key arguments and best-practices relevant for choosing the Finite Difference Time Domain ----------------------------- - For standard plasma wakefield configurations, it is possible to model the physics correctly using the Particle-In-Cell (PIC) Finite Difference Time Domain (FDTD) algorithms (:doc:`../theory/picsar_theory`). + For standard plasma wakefield configurations, it is possible to model the physics correctly using the :ref:`Particle-In-Cell (PIC) ` Finite Difference Time Domain (FDTD) algorithms. If the simulation contains localised extremely high intensity fields, however, numerical instabilities might arise, such as the numerical Cherenkov instability (:doc:`../theory/boosted_frame`). In that case, it is recommended to use the Pseudo Spectral Analytical Time Domain (PSATD) or the Pseudo-Spectral Time-Domain (PSTD) algorithms. In the example we are describing, it is sufficient to use FDTD. @@ -96,7 +98,7 @@ Time step The time step (:math:`dt`) is used to iterated over the main PIC loop and is computed by WarpX differently depending on the Maxwell field FDTD solvers used: - * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :doc:`../theory/picsar_theory` + * **For Yee** is equal to the CFL parameter chosen in the input file (:doc:`parameters`) times the Courant–Friedrichs–Lewy condition (CFL) that follows the analytical expression in :ref:`theory-pic` * **For CKC** is equal to CFL times the minimum between the boosted frame cell dimensions where CFL is chosen to be below unity and set an optimal trade-off between making the simulation faster and avoiding NCI and other spurious effects. diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index dee10651813..639b57d219e 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -1,359 +1,148 @@ .. _usage-picmi: +.. _usage-picmi-run: + +Parameters: Python (PICMI) +========================== -Python (PICMI) -============== +This documents on how to use WarpX as a Python script (e.g., ``python3 PICMI_script.py``). WarpX uses the `PICMI standard `__ for its Python input files. -Python version 3.8 or newer is required. +Complete example input files can be found in :ref:`the examples section `. -Example input files can be found in :ref:`the examples section `. In the input file, instances of classes are created defining the various aspects of the simulation. -The `Simulation` object is the central object, where the instances are passed, -defining the simulation time, field solver, registered species, etc. +A variable of type :py:class:`pywarpx.picmi.Simulation` is the central object to which all other options are passed, defining the simulation time, field solver, registered species, etc. -.. _usage-picmi-parameters: +Once the simulation is fully configured, it can be used in one of two modes. +**Interactive** use is the most common and can be :ref:`extended with custom runtime functionality `: -Classes -------- +.. tab-set:: -Simulation and grid setup -^^^^^^^^^^^^^^^^^^^^^^^^^ + .. tab-item:: Interactive -Simulation -"""""""""" -.. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :py:meth:`~pywarpx.picmi.Simulation.step`: run directly from Python -Constants -""""""""" -For convenience, the PICMI interface defines the following constants, -which can be used directly inside any PICMI script. The values are in SI units. + .. tab-item:: Preprocessor -- ``picmi.constants.c``: The speed of light in vacuum. -- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` -- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` -- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). -- ``picmi.constants.m_e``: The electron mass -- ``picmi.constants.m_p``: The proton mass + :py:meth:`~pywarpx.picmi.Simulation.write_input_file`: create an :ref:`inputs file for a WarpX executable ` -Field solvers define the updates of electric and magnetic fields. +When run directly from Python, one can also extend WarpX with further custom user logic. +See the :ref:`detailed workflow page ` on how to extend WarpX from Python. -ElectromagneticSolver -""""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectromagneticSolver -ElectrostaticSolver -""""""""""""""""""" -.. autoclass:: pywarpx.picmi.ElectrostaticSolver +.. _usage-picmi-parameters: + +Simulation and Grid Setup +------------------------- + +.. autoclass:: pywarpx.picmi.Simulation + :members: step, add_species, add_laser, write_input_file -Cartesian3DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian3DGrid -Cartesian2DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian2DGrid -Cartesian1DGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.Cartesian1DGrid -CylindricalGrid -""""""""""""""" .. autoclass:: pywarpx.picmi.CylindricalGrid -EmbeddedBoundary -"""""""""""""""" .. autoclass:: pywarpx.picmi.EmbeddedBoundary +Field solvers define the updates of electric and magnetic fields. + +.. autoclass:: pywarpx.picmi.ElectromagneticSolver + +.. autoclass:: pywarpx.picmi.ElectrostaticSolver + +Constants +--------- + +For convenience, the PICMI interface defines the following constants, +which can be used directly inside any PICMI script. The values are in SI units. + +- ``picmi.constants.c``: The speed of light in vacuum. +- ``picmi.constants.ep0``: The vacuum permittivity :math:`\epsilon_0` +- ``picmi.constants.mu0``: The vacuum permeability :math:`\mu_0` +- ``picmi.constants.q_e``: The elementary charge (absolute value of the charge of an electron). +- ``picmi.constants.m_e``: The electron mass +- ``picmi.constants.m_p``: The proton mass + Applied fields -^^^^^^^^^^^^^^ +-------------- -AnalyticInitialField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticInitialField -ConstantAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.ConstantAppliedField -AnalyticAppliedField -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticAppliedField -PlasmaLens -"""""""""" .. autoclass:: pywarpx.picmi.PlasmaLens -Mirror -"""""" .. autoclass:: pywarpx.picmi.Mirror Diagnostics -^^^^^^^^^^^ +----------- -ParticleDiagnostic -"""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleDiagnostic -FieldDiagnostic -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldDiagnostic -ElectrostaticFieldDiagnostic -"""""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ElectrostaticFieldDiagnostic +.. autoclass:: pywarpx.picmi.Checkpoint + +.. autoclass:: pywarpx.picmi.ReducedDiagnostic + Lab-frame diagnostics diagnostics are used when running boosted-frame simulations. -LabFrameFieldDiagnostic -""""""""""""""""""""""" -.. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic +.. autoclass:: pywarpx.picmi.LabFrameParticleDiagnostic -Checkpoint -"""""""""" -.. autoclass:: pywarpx.picmi.Checkpoint +.. autoclass:: pywarpx.picmi.LabFrameFieldDiagnostic Particles -^^^^^^^^^ +--------- Species objects are a collection of particles with similar properties. For instance, background plasma electrons, background plasma ions and an externally injected beam could each be their own particle species. -Species -""""""" .. autoclass:: pywarpx.picmi.Species -MultiSpecies -"""""""""""" .. autoclass:: pywarpx.picmi.MultiSpecies Particle distributions can be used for to initialize particles in a particle species. -GaussianBunchDistribution -""""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.GaussianBunchDistribution -UniformDistribution -""""""""""""""""""" .. autoclass:: pywarpx.picmi.UniformDistribution -AnalyticDistribution -"""""""""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticDistribution -ParticleListDistribution -"""""""""""""""""""""""" .. autoclass:: pywarpx.picmi.ParticleListDistribution Particle layouts determine how to microscopically place macro particles in a grid cell. -GriddedLayout -""""""""""""" .. autoclass:: pywarpx.picmi.GriddedLayout -PseudoRandomLayout -"""""""""""""""""" .. autoclass:: pywarpx.picmi.PseudoRandomLayout -Other operations related to particles +Other operations related to particles: -CoulombCollisions -""""""""""""""""" .. autoclass:: pywarpx.picmi.CoulombCollisions -MCCCollisions -""""""""""""" +.. autoclass:: pywarpx.picmi.DSMCCollisions + .. autoclass:: pywarpx.picmi.MCCCollisions -FieldIonization -""""""""""""""" .. autoclass:: pywarpx.picmi.FieldIonization -Lasers -^^^^^^ +Laser Pulses +------------ Laser profiles can be used to initialize laser pulses in the simulation. -GaussianLaser -""""""""""""" .. autoclass:: pywarpx.picmi.GaussianLaser -AnalyticLaser -""""""""""""" .. autoclass:: pywarpx.picmi.AnalyticLaser Laser injectors control where to initialize laser pulses on the simulation grid. -LaserAntenna -"""""""""""" .. autoclass:: pywarpx.picmi.LaserAntenna - - -.. _usage-picmi-run: - -Running -------- - -WarpX can be run in one of two modes. It can run as a preprocessor, using the -Python input file to generate an input file to be used by the C++ version, or -it can be run directly from Python. -The examples support running in both modes by commenting and uncommenting the appropriate lines. - -In either mode, if using a `virtual environment `__, be sure to activate it before compiling and running WarpX. - - -Running WarpX directly from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For this, a full Python installation of WarpX is required, as described in :ref:`the install documentation ` (:ref:`developers `). - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. - -* Add a **Python script** in the directory. - -The input file should have the line ``sim.step()`` which runs the simulation. - -* **Run** the script with Python: - -.. code-block:: bash - - mpirun -np python - -where ```` is the number of MPI ranks used, and ```` -is the name of the script. - - -.. _usage-picmi-extend: - -Extending a Simulation from Python -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When running WarpX directly from Python it is possible to interact with the simulation -by installing ``CallbackFunctions``, which will execute a given Python function at a -specific location in the WarpX simulation loop. - -.. autoclass:: pywarpx.callbacks.CallbackFunctions - -Places in the WarpX loop where callbacks are available include: -``afterinit``, ``beforecollisions``, ``aftercollisions``, ``beforeEsolve``, ``afterEsolve``, -``beforeInitEsolve``, ``afterInitEsolve``, ``beforedeposition``, ``afterdeposition``, -``beforestep``, ``afterstep``, ``afterdiagnostics``,``afterrestart`` and ``oncheckpointsignal``. -See the examples in ``Examples/Tests/ParticleDataPython`` for references on how to use -``callbacks``. - -There are several "hooks" available via the ``libwarpx`` shared library to access and manipulate -simulation objects (particles, fields and memory buffers) as well as general properties -(such as processor number). These "hooks" are accessible through the `Simulation.extension` object. - -An important object is ``Simulation.extension.warpx``, which is available during simulation run. -This object is the Python equivalent to the central ``WarpX`` simulation class and provides access to -field ``MultiFab`` and ``ParticleContainer`` data. - -.. function:: pywarpx.picmi.Simulation.extension.warpx.getistep - -.. function:: pywarpx.picmi.Simulation.extension.warpx.gett_new - -.. function:: pywarpx.picmi.Simulation.extension.warpx.evolve - -.. autofunction:: pywarpx.picmi.Simulation.extension.finalize - -These and other classes are provided through `pyAMReX `__. -After the simulation is initialized, pyAMReX can be accessed via - -.. code-block:: python - - from pywarpx import picmi, libwarpx - - # ... simulation definition ... - - # equivalent to - # import amrex.space3d as amr - # for a 3D simulation - amr = libwarpx.amr # picks the right 1d, 2d or 3d variant - -.. function:: amr.ParallelDescriptor.NProcs() - -.. function:: amr.ParallelDescriptor.MyProc() - -.. function:: amr.ParallelDescriptor.IOProcessor() - -.. function:: amr.ParallelDescriptor.IOProcessorNumber() - -Particles can be added to the simulation at specific positions and with specific -attribute values: - -.. code-block:: python - - from pywarpx import particle_containers, picmi - - # ... - - electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_particles - -Properties of the particles already in the simulation can be obtained with various -functions. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_count - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_structs - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_particle_arrays - -The ``get_particle_structs()`` and ``get_particle_arrays()`` functions are called -by several utility functions of the form ``get_particle_{comp_name}`` where -``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, -``weight``, ``ux``, ``uy`` or ``uz``. - -New components can be added via Python. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.add_real_comp - -Various diagnostics are also accessible from Python. -This includes getting the deposited or total charge density from a given species -as well as accessing the scraped particle buffer. See the example in -``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact -with scraped particle data. - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.get_species_charge_sum - -.. autofunction:: pywarpx.particle_containers.ParticleContainerWrapper.deposit_charge_density - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_size - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer_structs - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.get_particle_boundary_buffer - -.. autofunction:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper.clearParticleBoundaryBuffer - -The embedded boundary conditions can be modified when using the electrostatic solver. - -.. function:: pywarpx.picmi.Simulation.extension.warpx.set_potential_on_eb - -Using Python Input as a Preprocessor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this case, only the pure Python version needs to be installed, as described :ref:`here `. - -In order to run a new simulation: - -* Create a **new directory**, where the simulation will be run. - -* Add a **Python script** in the directory. - -The input file should have the line like ``sim.write_input_file(file_name = 'inputs_from_PICMI')`` -which runs the preprocessor, generating the AMReX inputs file. - -* **Run** the script with Python: - -.. code-block:: bash - - python - -where ```` is the name of the script. -This creates the WarpX input file that you can run as normal with the WarpX executable. diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 6fefcd9a243..e68ec9391be 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -8,9 +8,12 @@ This section collects typical user workflows and best practices for WarpX. .. toctree:: :maxdepth: 2 + workflows/python_extend + workflows/domain_decomposition + workflows/plot_distribution_mapping workflows/debugging workflows/libensemble workflows/plot_timestep_duration - workflows/plot_distribution_mapping workflows/psatd_stencil workflows/archiving + workflows/ml_dataset_training diff --git a/Docs/source/usage/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst similarity index 98% rename from Docs/source/usage/domain_decomposition.rst rename to Docs/source/usage/workflows/domain_decomposition.rst index d3df5b78f5c..1340ecc10d9 100644 --- a/Docs/source/usage/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -66,7 +66,7 @@ and the details of the on-node parallelization and computer architecture used fo Because these parameters put additional constraints on the domain size for a simulation, it can be cumbersome to calculate the number of cells and the physical size of the computational domain for a given resolution. This -:download:`Python script <../../../Tools/DevUtils/compute_domain.py>` does it +:download:`Python script <../../../../Tools/DevUtils/compute_domain.py>` does it automatically. When using the RZ spectral solver, the values of ``amr.max_grid_size`` and ``amr.blocking_factor`` are constrained since the solver diff --git a/Docs/source/usage/workflows/ml_dataset_training.rst b/Docs/source/usage/workflows/ml_dataset_training.rst new file mode 100644 index 00000000000..6e60a318bee --- /dev/null +++ b/Docs/source/usage/workflows/ml_dataset_training.rst @@ -0,0 +1,267 @@ +.. _ml_dataset_training: + +Training a Surrogate Model from WarpX Data +========================================== + +Suppose we have a WarpX simulation that we wish to replace with a neural network surrogate model. +For example, a simulation determined by the following input script + +.. dropdown:: Python Input for Training Simulation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/run_warpx_training.py + :language: python + +In this section we walk through a workflow for data processing and model training. +This workflow was developed and first presented in :cite:t:`ml-SandbergIPAC23,ml-SandbergPASC24`. + +This assumes you have an up-to-date environment with PyTorch and openPMD. + +Data Cleaning +------------- + +It is important to inspect the data for artifacts to +check that input/output data make sense. +If we plot the final phase space for beams 1-8, +the particle data is distributed in a single blob, +as shown by :numref:`fig_phase_space_beam_1` for beam 1. +This is as we expect and what is optimal for training neural networks. + +.. _fig_phase_space_beam_1: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010209-c55baf1c-dd98-4d56-a675-ad3729481eee.png + :alt: Plot showing the final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + + The final phase space projections for beam 1 of the training data, for a surrogate to stage 1. + +.. _fig_phase_space_beam_0: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010282-40560ac4-8509-4599-82ca-167bb1739cff.png + :alt: Plot showing the final phase space projections for beam 0 of the training data, for a surrogate to stage 0. + + The final phase space projections for beam 0 of the training data, for a surrogate to stage 0 + +On the other hand, the final phase space for beam 0, shown in :numref:`fig_phase_space_beam_1`, +has a halo of outlying particles. +Looking closer at the z-pz space, we see that some particles got caught in a decelerating +region of the wake, have slipped back and are much slower than the rest of the beam. +To assist our neural network in learning dynamics of interest, we filter out these particles. +It is sufficient for our purposes to select particles that are not too far back, setting +``particle_selection={'z':[0.28002, None]}``. Then a particle tracker is set up to make sure +we consistently filter out these particles from both the initial and final data. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Particle tracking START + :end-before: # Manual: Particle tracking END + +Create Normalized Dataset +------------------------- + +Having chosen training data we are content with, we now need to format the data, +normalize it, and store the normalized data as well as the normalizations. +The script below will take the openPMD data we have selected and +format, normalize, and store it. + +.. dropdown:: Python dataset creation + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/create_dataset.py + :language: python + +Load openPMD Data +^^^^^^^^^^^^^^^^^ + +First the openPMD data is loaded, using the particle selector as chosen above. +The neural network will make predictions from the initial phase space coordinates, +using the final phase space coordinates to measure how well it is making predictions. +Hence we load two sets of particle data, the source and target particle arrays. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Load openPMD START + :end-before: # Manual: Load openPMD END + +Normalize Data +^^^^^^^^^^^^^^ + +Neural networks learn better on appropriately normalized data. +Here we subtract out the mean in each coordinate direction and +divide by the standard deviation in each coordinate direction, +for normalized data that is centered on the origin with unit variance. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Normalization START + :end-before: # Manual: Normalization END + +openPMD to PyTorch Data +^^^^^^^^^^^^^^^^^^^^^^^ + +With the data normalized, it must be stored in a form PyTorch recognizes. +The openPMD data are 6 lists of arrays, for each of the 6 phase space coordinates +:math:`x, y, z, p_x, p_y,` and :math:`p_z`. +This data are converted to an :math:`N\times 6` numpy array and then to a PyTorch :math:`N\times 6` tensor. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Format data START + :end-before: # Manual: Format data END + +Save Normalizations and Normalized Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With the data properly normalized, it and the normalizations are saved to file for +use in training and inference. + +.. literalinclude:: ml_materials/create_dataset.py + :language: python + :dedent: 4 + :start-after: # Manual: Save dataset START + :end-before: # Manual: Save dataset END + +Neural Network Structure +------------------------ + +It was found in :cite:t:`ml-SandbergPASC24` that reasonable surrogate models are obtained with +shallow feedforward neural networks consisting of fewer than 10 hidden layers and +just under 1000 nodes per layer. +The example shown here uses 3 hidden layers and 20 nodes per layer +and is trained for 10 epochs. + + + +Train and Save Neural Network +----------------------------- + +The script below trains the neural network on the dataset just created. +In subsequent sections we discuss the various parts of the training process. + +.. dropdown:: Python neural network training + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/train.py + :language: python3 + +Training Function +^^^^^^^^^^^^^^^^^ + +In the training function, the model weights are updated. +Iterating through batches, the loss function is evaluated on each batch. +PyTorch provides automatic differentiation, so the direction of steepest descent +is determined when the loss function is evaluated and the ``loss.backward()`` function +is invoked. +The optimizer uses this information to update the weights in the ``optimizer.step()`` call. +The training loop then resets the optimizer and updates the summed error for the whole dataset +with the error on the batch and continues iterating through batches. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Train function START + :end-before: # Manual: Train function END + +Testing Function +^^^^^^^^^^^^^^^^ + +The testing function just evaluates the neural network on the testing data that has not been used +to update the model parameters. +This testing function requires that the testing dataset is small enough to be loaded all at once. +The PyTorch dataloader can load data in batches if this size assumption is not satisfied. +The error, measured by the loss function, is returned by the testing function to be aggregated and stored. +Note that this function returns the sum of all errors across the entire dataset, +which is later divided by the size of the dataset in the training loop. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Test function START + :end-before: # Manual: Test function END + +Train Loop +^^^^^^^^^^ + +The full training loop performs ``n_epochs`` number of iterations. +At each iteration the training and testing functions are called, +the respective errors are divided by the size of the dataset and recorded, +and a status update is printed to the console. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Training loop START + :end-before: # Manual: Training loop END + +Save Neural Network Parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The model weights are saved after training to record the updates to the model parameters. +Additionally, we save some model metainformation with the model for convenience, +including the model hyperparameters, the training and testing losses, and how long the training took. + +.. literalinclude:: ml_materials/train.py + :language: python + :start-after: # Manual: Save model START + :end-before: # Manual: Save model END + +Evaluate +-------- + +In this section we show two ways to diagnose how well the neural network is learning the data. +First we consider the train-test loss curves, shown in :numref:`fig_train_test_loss`. +This figure shows the model error on the training data (in blue) and testing data (in green) as a function of the number of epochs seen. +The training data is used to update the model parameters, so training error should be lower than testing error. +A key feature to look for in the train-test loss curve is the inflection point in the test loss trend. +The testing data is set aside as a sample of data the neural network hasn't seen before. +The testing error serves as a metric of model generalizability, indicating how well the model performs +on data it hasn't seen yet. +When the test-loss starts to trend flat or even upward, the neural network is no longer improving its ability to generalize to new data. + +.. _fig_train_test_loss: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010428-f83725ab-a08f-494c-b075-314b0d26cb9a.png + :alt: Plot of training and testing loss curves versus number of training epochs. + + Training (in blue) and testing (in green) loss curves versus number of training epochs. + +.. _fig_train_evaluation: + +.. figure:: https://user-images.githubusercontent.com/10621396/290010486-4a3541e7-e0be-4cf1-b33b-57d5e5985196.png + :alt: Plot comparing model prediction with simulation output. + + A comparison of model prediction (yellow-red dots, colored by mean-squared error) with simulation output (black dots). + +A visual inspection of the model prediction can be seen in :numref:`fig_train_evaluation`. +This plot compares the model prediction, with dots colored by mean-square error, on the testing data with the actual simulation output in black. +The model obtained with the hyperparameters chosen here trains quickly but is not very accurate. +A more accurate model is obtained with 5 hidden layers and 800 nodes per layer, +as discussed in :cite:t:`ml-SandbergPASC24`. + +These figures can be generated with the following Python script. + +.. dropdown:: Python visualization of progress training neural network + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ml_materials/visualize.py + :language: python3 + + +Surrogate Usage in Accelerator Physics +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A neural network such as the one we trained here can be incorporated in other BLAST codes. +`Consider the example using neural networks in ImpactX `__. + +.. bibliography:: + :keyprefix: ml- diff --git a/Docs/source/usage/workflows/ml_materials/create_dataset.py b/Docs/source/usage/workflows/ml_materials/create_dataset.py new file mode 100644 index 00000000000..08000105479 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/create_dataset.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import tarfile +from urllib import request + +import numpy as np + +c = 2.998e8 + +from openpmd_viewer import OpenPMDTimeSeries, ParticleTracker +import torch + +############### + +def sanitize_dir_strings(*dir_strings): + """append '/' to a string for concatenation in building up file tree descriptions + """ + dir_strings = list(dir_strings) + for ii, dir_string in enumerate(dir_strings): + if dir_string[-1] != '/': + dir_strings[ii] = dir_string + '/' + + return dir_strings + +def download_and_unzip(url, data_dir): + request.urlretrieve(url, data_dir) + with tarfile.open(data_dir) as tar_dataset: + tar_dataset.extractall() + +def create_source_target_data(data_dir, + species, + source_index=0, + target_index=-1, + particle_selection=None + ): + """Create dataset from openPMD files + + Parameters + --- + data_dir : string, location of diagnostic data + source_index : int, which index to take source data from + target_index : int, which index to take target data from + particle_selection: dictionary, optional, selection criterion for dataset + + Returns + --- + source_data: Nx6 array of source particle data + source_means: 6 element array of source particle coordinate means + source_stds: 6 element array of source particle coordinate standard deviations + target_data: Nx6 array of target particle data + target_means: 6 element array of target particle coordinate means + target_stds: 6 element array of source particle coordinate standard deviations + relevant times: 2 element array of source and target times + """ + data_dir, = sanitize_dir_strings(data_dir) + data_path = data_dir + print('loading openPMD data from', data_path) + ts = OpenPMDTimeSeries(data_path) + relevant_times = [ts.t[source_index], ts.t[target_index]] + + # Manual: Particle tracking START + iteration = ts.iterations[target_index] + pt = ParticleTracker( ts, + species=species, + iteration=iteration, + select=particle_selection) + # Manual: Particle tracking END + + #### create normalized source, target data sets #### + print('creating data sets') + + # Manual: Load openPMD START + iteration = ts.iterations[source_index] + source_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + + iteration = ts.iterations[target_index] + target_data = ts.get_particle(species=species, + iteration=iteration, + var_list=['x','y','z','ux','uy','uz'], + select=pt) + # Manual: Load openPMD END + + # Manual: Normalization START + target_means = np.zeros(6) + target_stds = np.zeros(6) + source_means = np.zeros(6) + source_stds = np.zeros(6) + for jj in range(6): + source_means[jj] = source_data[jj].mean() + source_stds[jj] = source_data[jj].std() + source_data[jj] -= source_means[jj] + source_data[jj] /= source_stds[jj] + + for jj in range(6): + target_means[jj] = target_data[jj].mean() + target_stds[jj] = target_data[jj].std() + target_data[jj] -= target_means[jj] + target_data[jj] /= target_stds[jj] + # Manual: Normalization END + + # Manual: Format data START + source_data = torch.tensor(np.column_stack(source_data)) + target_data = torch.tensor(np.column_stack(target_data)) + # Manual: Format data END + + return source_data, source_means, source_stds, target_data, target_means, target_stds, relevant_times + + +def save_warpx_surrogate_data(dataset_fullpath_filename, + diag_dir, + species, + training_frac, + batch_size, + source_index, + target_index, + survivor_select_index, + particle_selection=None + ): + + source_target_data = create_source_target_data(data_dir=diag_dir, + species=species, + source_index=source_index, + target_index=target_index, + particle_selection=particle_selection + ) + source_data, source_means, source_stds, target_data, target_means, target_stds, times = source_target_data + + # Manual: Save dataset START + full_dataset = torch.utils.data.TensorDataset(source_data.float(), target_data.float()) + + n_samples = full_dataset.tensors[0].size(0) + n_train = int(training_frac*n_samples) + n_test = n_samples - n_train + + train_data, test_data = torch.utils.data.random_split(full_dataset, [n_train, n_test]) + + torch.save({'dataset':full_dataset, + 'train_indices':train_data.indices, + 'test_indices':test_data.indices, + 'source_means':source_means, + 'source_stds':source_stds, + 'target_means':target_means, + 'target_stds':target_stds, + 'times':times, + }, + dataset_fullpath_filename + ) + # Manual: Save dataset END + +######## end utility functions ############# +######## start dataset creation ############ + +data_url = "https://zenodo.org/records/10368972/files/ml_example_training.tar.gz?download=1" +download_and_unzip(data_url, "training_dataset.tar.gz") +data_dir = "lab_particle_diags/lab_particle_diags/" + + +# create data set + +source_index = 0 +target_index = 1 +survivor_select_index = 1 +batch_size=1200 +training_frac = 0.7 + +os.makedirs('datasets', exist_ok=True) + +# improve stage 0 dataset +stage_i = 0 +select = {'z':[0.28002, None]} +species = f'beam_stage_{stage_i}' +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename +save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index, + particle_selection=select + ) + +for stage_i in range(1,9): + species = f'beam_stage_{stage_i}' + dataset_filename = f'dataset_{species}.pt' + dataset_file = 'datasets/' + dataset_filename + save_warpx_surrogate_data(dataset_fullpath_filename=dataset_file, + diag_dir=data_dir, + species=species, + training_frac=training_frac, + batch_size=batch_size, + source_index=source_index, + target_index=target_index, + survivor_select_index=survivor_select_index + ) diff --git a/Docs/source/usage/workflows/ml_materials/neural_network_classes.py b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py new file mode 100644 index 00000000000..58b51a1d364 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/neural_network_classes.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from enum import Enum + +from torch import nn + + +class ActivationType(Enum): + """ + Activation class provides an enumeration type for the supported activation layers + """ + ReLU = 1 + Tanh = 2 + PReLU = 3 + Sigmoid = 4 + +def get_enum_type(type_to_test, EnumClass): + """ + Returns the enumeration type associated to type_to_test in EnumClass + + Parameters + ---------- + type_to_test: EnumClass, int or str + object whose Enum class is to be obtained + EnumClass: Enum class + Enum class to test + """ + if type(type_to_test) is EnumClass: + return type_to_test + if type(type_to_test) is int: + return EnumClass(type_to_test) + if type(type_to_test) is str: + return getattr(EnumClass, type_to_test) + else: + raise Exception("unsupported type entered") + + + +class ConnectedNN(nn.Module): + """ + ConnectedNN is a class of fully connected neural networks + """ + def __init__(self, layers): + super().__init__() + self.stack = nn.Sequential(*layers) + def forward(self, x): + return self.stack(x) + +class OneActNN(ConnectedNN): + """ + OneActNN is class of fully connected neural networks admitting only one activation function + """ + def __init__(self, + n_in, + n_out, + n_hidden_nodes, + n_hidden_layers, + act): + + self.n_in = n_in + self.n_out = n_out + self.n_hidden_layers = n_hidden_layers + self.n_hidden_nodes = n_hidden_nodes + + self.act = get_enum_type(act, ActivationType) + + layers = [nn.Linear(self.n_in, self.n_hidden_nodes)] + + for ii in range(self.n_hidden_layers): + if self.act is ActivationType.ReLU: + layers += [nn.ReLU()] + if self.act is ActivationType.Tanh: + layers += [nn.Tanh()] + if self.act is ActivationType.PReLU: + layers += [nn.PReLU()] + if self.act is ActivationType.Sigmoid: + layers += [nn.Sigmoid()] + + if ii < self.n_hidden_layers - 1: + layers += [nn.Linear(self.n_hidden_nodes,self.n_hidden_nodes)] + + layers += [nn.Linear(self.n_hidden_nodes, self.n_out)] + + super().__init__(layers) diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py new file mode 100644 index 00000000000..8ee098e3e29 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# +# Copyright 2022-2023 WarpX contributors +# Authors: WarpX team +# License: BSD-3-Clause-LBNL +# +# -*- coding: utf-8 -*- + +import math + +import numpy as np + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e +m_e = picmi.constants.m_e +m_p = picmi.constants.m_p +ep0 = picmi.constants.ep0 + +# Number of cells +dim = '3' +nx = ny = 128 +nz = 8832 +if dim == 'rz': + nr = nx//2 + +# Computational domain +rmin = 0. +rmax = 128e-6 +zmin = -180e-6 +zmax = 0. + +# Number of processes for static load balancing +# Check with your submit script +num_procs = [1, 1, 16*4] +if dim == 'rz': + num_procs = [1, 16] + +# Number of time steps +gamma_boost = 60. +beta_boost = np.sqrt(1.-gamma_boost**-2) + +# Create grid +if dim == 'rz': + grid = picmi.CylindricalGrid( + number_of_cells=[nr, nz], + guard_cells=[32, 32], + n_azimuthal_modes=2, + lower_bound=[rmin, zmin], + upper_bound=[rmax, zmax], + lower_boundary_conditions=['none', 'damped'], + upper_boundary_conditions=['none', 'damped'], + lower_boundary_conditions_particles=['absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing'], + moving_window_velocity=[0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=64) +else: + grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + guard_cells=[11, 11, 12], + lower_bound=[-rmax, -rmax, zmin], + upper_bound=[rmax, rmax, zmax], + lower_boundary_conditions=['periodic', 'periodic', 'damped'], + upper_boundary_conditions=['periodic', 'periodic', 'damped'], + lower_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + upper_boundary_conditions_particles=['periodic', 'periodic', 'absorbing'], + moving_window_velocity=[0., 0., c], + warpx_max_grid_size=256, + warpx_blocking_factor=32) + + +# plasma region +plasma_rlim = 100.e-6 +N_stage = 9 +L_plasma_bulk = 0.28 +L_ramp = 1.e-9 +L_ramp_up = L_ramp +L_ramp_down = L_ramp +L_stage = L_plasma_bulk + 2*L_ramp + +# focusing +# lens external fields +beam_gamma1 = 15095 +lens_focal_length = 0.015 +lens_width = 0.003 + +stage_spacing = L_plasma_bulk + 2*lens_focal_length + +def get_species_of_accelerator_stage(stage_idx, stage_zmin, stage_zmax, + stage_xmin=-plasma_rlim, stage_xmax=plasma_rlim, + stage_ymin=-plasma_rlim, stage_ymax=plasma_rlim, + Lplus = L_ramp_up, Lp = L_plasma_bulk, + Lminus = L_ramp_down): + # Parabolic density profile + n0 = 1.7e23 + Rc = 40.e-6 + Lstage = Lplus + Lp + Lminus + if not np.isclose(stage_zmax-stage_zmin, Lstage): + print('Warning: zmax disagrees with stage length') + parabolic_distribution = picmi.AnalyticDistribution( + density_expression= + f'n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.-cos(pi*(z-{stage_zmin})/Lplus)))*((z-{stage_zmin})=Lplus)*((z-{stage_zmin})<(Lplus+Lp))' \ + + f'+n0*(1.+4.*(x**2+y**2)/(kp**2*Rc**4))*(0.5*(1.+cos(pi*((z-{stage_zmin})-Lplus-Lp)/Lminus)))*((z-{stage_zmin})>=(Lplus+Lp))*((z-{stage_zmin})<(Lplus+Lp+Lminus))', + pi=3.141592653589793, + n0=n0, + kp=q_e/c*math.sqrt(n0/(m_e*ep0)), + Rc=Rc, + Lplus=Lplus, + Lp=Lp, + Lminus=Lminus, + lower_bound=[stage_xmin, stage_ymin, stage_zmin], + upper_bound=[stage_xmax, stage_ymax, stage_zmax], + fill_in=True) + + electrons = picmi.Species( + particle_type='electron', + name=f'electrons{stage_idx}', + initial_distribution=parabolic_distribution) + + ions = picmi.Species( + particle_type='proton', + name=f'ions{stage_idx}', + initial_distribution=parabolic_distribution) + + return electrons, ions + +species_list = [] +for i_stage in range(1): + # Add plasma + zmin_stage = i_stage * stage_spacing + zmax_stage = zmin_stage + L_stage + electrons, ions = get_species_of_accelerator_stage(i_stage+1, zmin_stage, zmax_stage) + species_list.append(electrons) + species_list.append(ions) + +# add beam to species_list +beam_charge = -10.e-15 # in Coulombs +N_beam_particles = int(1e6) +beam_centroid_z = -107.e-6 +beam_rms_z = 2.e-6 +#beam_gammas = [2000 + 13000 * i_stage for i_stage in range(N_stage)] +beam_gammas = [1957, 15188, 28432, 41678, 54926, 68174, 81423,94672, 107922,121171] # From 3D run +beams = [] +for i_stage in range(N_stage): + beam_gamma = beam_gammas[i_stage] + sigma_gamma = 0.10 * beam_gamma + gaussian_distribution = picmi.GaussianBunchDistribution( + n_physical_particles= abs(beam_charge) / q_e, + rms_bunch_size=[2.e-6, 2.e-6, beam_rms_z], + rms_velocity=[8*c, 8*c, sigma_gamma*c], + centroid_position=[0., 0., beam_centroid_z], + centroid_velocity=[0., 0., beam_gamma*c], + ) + beam = picmi.Species( + particle_type='electron', + name=f'beam_stage_{i_stage}', + initial_distribution= gaussian_distribution + ) + beams.append(beam) + +# Laser +antenna_z = -1e-9 +profile_t_peak = 1.46764864e-13 +def get_laser(antenna_z, profile_t_peak, fill_in=True): + profile_focal_distance = 0. + laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=36e-06, + duration=7.33841e-14, + focal_position=[0., 0., profile_focal_distance + antenna_z], + centroid_position=[0., 0., antenna_z - c*profile_t_peak], + propagation_direction=[0., 0., 1.], + polarization_direction=[0., 1., 0.], + a0=2.36, + fill_in=fill_in) + laser_antenna = picmi.LaserAntenna( + position=[0., 0., antenna_z], + normal_vector=[0., 0., 1.]) + return (laser, laser_antenna) +lasers = [] +for i_stage in range(1): + fill_in = True + if i_stage == 0: + fill_in = False + lasers.append( + get_laser(antenna_z + i_stage*stage_spacing, + profile_t_peak + i_stage*stage_spacing/c, + fill_in) + ) + +# Electromagnetic solver + +psatd_algo = 'multij' +if psatd_algo == 'galilean': + galilean_velocity = [0.,0.] if dim=='3' else [0.] + galilean_velocity += [-c*beta_boost] + n_pass_z = 1 + do_multiJ = None + do_multi_J_n_depositions=None + J_in_time = None + current_correction = True + divE_cleaning = False +elif psatd_algo == 'multij': + n_pass_z = 4 + galilean_velocity = None + do_multiJ = True + do_multi_J_n_depositions = 2 + J_in_time = "linear" + current_correction = False + divE_cleaning = True +else: + raise Exception(f'PSATD algorithm \'{psatd_algo}\' is not recognized!\n'\ + 'Valid options are \'multiJ\' or \'galilean\'.') +if dim == 'rz': + stencil_order = [8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,n_pass_z]) + grid_type = 'collocated' +else: + stencil_order = [8, 8, 16] + smoother = picmi.BinomialSmoother(n_pass=[1,1,n_pass_z]) + grid_type = 'hybrid' + + +solver = picmi.ElectromagneticSolver( + grid=grid, + method='PSATD', + cfl=0.9999, + source_smoother=smoother, + stencil_order=stencil_order, + galilean_velocity=galilean_velocity, + warpx_psatd_update_with_rho=True, + warpx_current_correction=current_correction, + divE_cleaning=divE_cleaning, + warpx_psatd_J_in_time=J_in_time + ) + +# Diagnostics +diag_field_list = ['B', 'E', 'J', 'rho'] +diag_particle_list = ['weighting','position','momentum'] +coarse_btd_end = int((L_plasma_bulk+0.001+stage_spacing*(N_stage-1))*100000) +stage_end_snapshots=[f'{int((L_plasma_bulk+stage_spacing*ii)*100000)}:{int((L_plasma_bulk+stage_spacing*ii)*100000+50)}:5' for ii in range(1)] +btd_particle_diag = picmi.LabFrameParticleDiagnostic( + name='lab_particle_diags', + species=beams, + grid=grid, + num_snapshots=25*N_stage, + #warpx_intervals=', '.join([f':{coarse_btd_end}:1000']+stage_end_snapshots), + warpx_intervals=', '.join(['0:0']+stage_end_snapshots), + dt_snapshots=0.00001/c, + data_list=diag_particle_list, + write_dir='lab_particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +btd_field_diag = picmi.LabFrameFieldDiagnostic( + name='lab_field_diags', + grid=grid, + num_snapshots=25*N_stage, + dt_snapshots=stage_spacing/25/c, + data_list=diag_field_list, + warpx_lower_bound=[-128.e-6, 0.e-6, -180.e-6], + warpx_upper_bound=[128.e-6, 0.e-6, 0.], + write_dir='lab_field_diags', + warpx_format='openpmd', + warpx_openpmd_backend='bp') + +field_diag = picmi.FieldDiagnostic( + name='field_diags', + data_list=diag_field_list, + grid=grid, + period=100, + write_dir='field_diags', + lower_bound=[-128.e-6, 0.e-6, -180.e-6], + upper_bound=[128.e-6, 0.e-6, 0.], + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +particle_diag = picmi.ParticleDiagnostic( + name='particle_diags', + species=beams, + period=100, + write_dir='particle_diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5') + +beamrel_red_diag = picmi.ReducedDiagnostic( + diag_type='BeamRelevant', + name='beamrel', + species=beam, + period=1) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + warpx_numprocs=num_procs, + warpx_compute_max_step_from_btd=True, + verbose=2, + particle_shape='cubic', + gamma_boost=gamma_boost, + warpx_charge_deposition_algo='standard', + warpx_current_deposition_algo='direct', + warpx_field_gathering_algo='momentum-conserving', + warpx_particle_pusher_algo='vay', + warpx_amrex_the_arena_is_managed=False, + warpx_amrex_use_gpu_aware_mpi=True, + warpx_do_multi_J=do_multiJ, + warpx_do_multi_J_n_depositions=do_multi_J_n_depositions, + warpx_grid_type=grid_type, + # default: 2 for staggered grids, 8 for hybrid grids + warpx_field_centering_order=[16,16,16], + # only for hybrid grids, default: 8 + warpx_current_centering_order=[16,16,16] + ) + +for species in species_list: + if dim=='rz': + n_macroparticle_per_cell=[2,4,2] + else: + n_macroparticle_per_cell=[2,2,2] + sim.add_species( + species, + layout=picmi.GriddedLayout(grid=grid, + n_macroparticle_per_cell=n_macroparticle_per_cell) + ) + +for i_stage in range(N_stage): + sim.add_species_through_plane( + species=beams[i_stage], + layout=picmi.PseudoRandomLayout(grid=grid, n_macroparticles=N_beam_particles), + injection_plane_position=0., + injection_plane_normal_vector=[0.,0.,1.]) + +for i_stage in range(1): + # Add laser + (laser, laser_antenna) = lasers[i_stage] + sim.add_laser(laser, injection_method=laser_antenna) + +# Add diagnostics +sim.add_diagnostic(btd_particle_diag) +#sim.add_diagnostic(btd_field_diag) +#sim.add_diagnostic(field_diag) +#sim.add_diagnostic(particle_diag) + +# Add reduced diagnostic +sim.add_diagnostic(beamrel_red_diag) + +sim.write_input_file(f'inputs_training_{N_stage}_stages') + +# Advance simulation until last time step +sim.step() diff --git a/Docs/source/usage/workflows/ml_materials/train.py b/Docs/source/usage/workflows/ml_materials/train.py new file mode 100644 index 00000000000..dc62819061c --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/train.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +import os +import time + +import neural_network_classes as mynn +import torch +import torch.nn.functional as F +import torch.optim as optim + +############# set model parameters ################# + +stage_i = 0 +species = f'beam_stage_{stage_i}' +source_index = 0 +target_index = 4 +survivor_select_index = 4 + +data_dim = 6 +n_in = data_dim +n_out = data_dim + +learning_rate = 0.0001 +n_epochs = 10 +batch_size = 1200 + +loss_fun = F.mse_loss + +n_hidden_nodes = 20 +n_hidden_layers = 3 +activation_type = 'ReLU' + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') +#################### load dataset ################ +dataset_filename = f'dataset_{species}.pt' +dataset_file = 'datasets/' + dataset_filename + +print(f"trying to load dataset+test-train split in {dataset_file}") + +dataset_with_indices = torch.load(dataset_file) +train_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['train_indices']) +test_data = torch.utils.data.dataset.Subset(dataset_with_indices['dataset'], dataset_with_indices['test_indices']) +source_data = dataset_with_indices['dataset'] +source_means = dataset_with_indices['source_means'] +source_stds = dataset_with_indices['source_stds'] +target_means = dataset_with_indices['target_means'] +target_stds = dataset_with_indices['target_stds'] +print("able to load data and test/train split") + +###### move data to device (GPU) if available ######## +source_device = train_data.dataset.tensors[0].to(device) # equivalently, test_data.tensors[0].to(device) +target_device = train_data.dataset.tensors[1].to(device) +full_dataset_device = torch.utils.data.TensorDataset(source_device.float(), target_device.float()) + +train_data_device = torch.utils.data.dataset.Subset(full_dataset_device, train_data.indices) +test_data_device = torch.utils.data.dataset.Subset(full_dataset_device, test_data.indices) + +train_loader_device = torch.utils.data.DataLoader(train_data_device, batch_size=batch_size, shuffle=True) +test_loader_device = torch.utils.data.DataLoader(test_data_device, batch_size=batch_size, shuffle=True) + +test_source_device = test_data_device.dataset.tensors[0] +test_target_device = test_data_device.dataset.tensors[1] + +training_set_size = len(train_data_device.indices) +testing_set_size = len(test_data_device.indices) + +###### create model ########### + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act=activation_type + ) + +training_time = 0 +train_loss_list = [] +test_loss_list = [] + +model.to(device=device); + +########## train and test functions #### +# Manual: Train function START +def train(model, optimizer, train_loader, loss_fun): + model.train() + total_loss = 0. + for batch_idx, (data, target) in enumerate(train_loader): + #evaluate network with data + output = model(data) + #compute loss + # sum the differences squared, take mean afterward + loss = loss_fun(output, target,reduction='sum') + #backpropagation: step optimizer and reset gradients + loss.backward() + optimizer.step() + optimizer.zero_grad() + total_loss += loss.item() + return total_loss +# Manual: Train function END + +def test(model, test_loader, loss_fun): + model.eval() + total_loss = 0. + with torch.no_grad(): + for batch_idx, (data, target) in enumerate(test_loader): + output = model(data) + total_loss += loss_fun(output, target, reduction='sum').item() + return total_loss + +# Manual: Test function START +def test_dataset(model, test_source, test_target, loss_fun): + model.eval() + with torch.no_grad(): + output = model(test_source) + return loss_fun(output, test_target, reduction='sum').item() +# Manual: Test function END + +######## training loop ######## + +optimizer = optim.Adam(model.parameters(), lr=learning_rate) + +do_print = True + +t3 = time.time() +# Manual: Training loop START +for epoch in range(n_epochs): + if do_print: + t1 = time.time() + ave_train_loss = train(model, optimizer, train_loader_device, loss_fun) / data_dim / training_set_size + ave_test_loss = test_dataset(model, test_source_device, test_target_device, loss_fun) / data_dim / training_set_size + train_loss_list.append(ave_train_loss) + test_loss_list.append(ave_test_loss) + + if do_print: + t2 = time.time() + print('Train Epoch: {:04d} \tTrain Loss: {:.6f} \tTest Loss: {:.6f}, this epoch: {:.3f} s'.format( + epoch + 1, ave_train_loss, ave_test_loss, t2-t1)) +# Manual: Training loop END +t4 = time.time() +print(f'total training time: {t4-t3:.3f}s') + +######### save model ######### + +os.makedirs('models', exist_ok=True) + +# Manual: Save model START +model.to(device='cpu') +torch.save({ + 'n_hidden_layers':n_hidden_layers, + 'n_hidden_nodes':n_hidden_nodes, + 'activation':activation_type, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'train_loss_list': train_loss_list, + 'test_loss_list': test_loss_list, + 'training_time': training_time, + }, f'models/{species}_model.pt') +# Manual: Save model END diff --git a/Docs/source/usage/workflows/ml_materials/visualize.py b/Docs/source/usage/workflows/ml_materials/visualize.py new file mode 100644 index 00000000000..e6da1029b67 --- /dev/null +++ b/Docs/source/usage/workflows/ml_materials/visualize.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Ryan Sandberg +# License: BSD-3-Clause-LBNL +# +from matplotlib import pyplot as plt +import neural_network_classes as mynn +import numpy as np +import torch +import torch.nn.functional as F + +c = 2.998e8 + + +# open model file +stage_i = 0 +species = f'beam_stage_{stage_i}' +model_data = torch.load(f'models/{species}_model.pt',map_location=torch.device('cpu')) +data_dim = 6 +n_in = data_dim +n_out = data_dim +n_hidden_layers = model_data['n_hidden_layers'] +n_hidden_nodes = model_data['n_hidden_nodes'] +activation_type = model_data['activation'] +train_loss_list = model_data['train_loss_list'] +test_loss_list = model_data['test_loss_list'] +training_time = model_data['training_time'] +loss_fun = F.mse_loss + + +n_epochs = len(train_loss_list) +train_counter = np.arange(n_epochs)+1 +test_counter = train_counter + +do_log_plot = False +fig, ax = plt.subplots() +if do_log_plot: + ax.semilogy(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.semilogy(test_counter, test_loss_list, color='green',label='testing loss') +else: + ax.plot(train_counter, train_loss_list, '.-',color='blue',label='training loss') + ax.plot(test_counter, test_loss_list, color='green',label='testing loss') +ax.set_xlabel('number of epochs seen') +ax.set_ylabel(' loss') +ax.legend() +fig_dir = 'figures/' +ax.set_title(f'final test error = {test_loss_list[-1]:.3e} ') +ax.grid() +plt.tight_layout() +plt.savefig(f'{species}_training_testing_error.png') + + +######### plot phase space comparison ####### +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print(f'device={device}') + +model = mynn.OneActNN(n_in = n_in, + n_out = n_out, + n_hidden_nodes=n_hidden_nodes, + n_hidden_layers = n_hidden_layers, + act = activation_type + ) +model.load_state_dict(model_data['model_state_dict']) +model.to(device=device); + +###### load model data ############### +dataset_filename = f'dataset_{species}.pt' +dataset_dir = 'datasets/' +model_input_data = torch.load(dataset_dir + dataset_filename) +dataset = model_input_data['dataset'] +train_indices = model_input_data['train_indices'] +test_indices = model_input_data['test_indices'] +source_means = model_input_data['source_means'] +source_stds = model_input_data['source_stds'] +target_means = model_input_data['target_means'] +target_stds = model_input_data['target_stds'] +source_time, target_time = model_input_data['times'] + + +source = dataset.tensors[0] +test_source = source[test_indices] +test_source_device = test_source.to(device) +with torch.no_grad(): + evaluation_device = model(test_source_device.float()) +eval_cpu = evaluation_device.to('cpu') + +target = dataset.tensors[1] +test_target = target[test_indices] + +target_si = test_target * target_stds + target_means +eval_cpu_si = eval_cpu * target_stds + target_means +target_mu = np.copy(target_si) +eval_cpu_mu = np.copy(eval_cpu_si) +target_mu[:,2] -= c*target_time +eval_cpu_mu[:,2] -= c*target_time +target_mu[:,:3] *= 1e6 +eval_cpu_mu[:,:3] *= 1e6 + + + +loss_tensor = torch.sum(loss_fun(eval_cpu, + test_target, + reduction='none'), + axis=1)/6 +loss_array = loss_tensor.detach().numpy() + +tinds = np.nonzero(loss_array > 0.0)[0] +skip = 10 + +plt.figure() +fig, axT = plt.subplots(3,3) +axes_label = {0:r'x [$\mu$m]', 1:r'y [$\mu$m]', 2:r'z - %.2f cm [$\mu$m]'%(c*target_time),3:r'$p_x$',4:r'$p_y$',5:r'$p_z$'} +xy_inds = [(0,1),(2,0),(2,1)] +def set_axes(ax, indx, indy): + ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') + ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=0.02, label='surrogate',cmap='YlOrRd') + ax.set_xlabel(axes_label[indx]) + ax.set_ylabel(axes_label[indy]) + # return + + +for ii in range(3): + ax = axT[0,ii] + indx,indy = xy_inds[ii] + set_axes(ax,indx,indy) + +for ii in range(2): + indx,indy = xy_inds[ii] + ax = axT[1,ii] + set_axes(ax,indx+3,indy+3) + +for ii in range(3): + ax = axT[2,ii] + indx = ii + indy = ii+3 + set_axes(ax, indx, indy) + + +ax = axT[1,2] +indx = 5 +indy = 4 +ax.scatter(target_mu[::skip,indx],target_mu[::skip,indy],s=8,c='k', label='simulation') +evalplt = ax.scatter(eval_cpu_mu[::skip,indx],eval_cpu_mu[::skip,indy],marker='*',c=loss_array[::skip],s=2, label='surrogate',cmap='YlOrRd') +ax.set_xlabel(axes_label[indx]) +ax.set_ylabel(axes_label[indy]) + +cb = plt.colorbar(evalplt, ax=ax) +cb.set_label('MSE loss') + +fig.suptitle(f'stage {stage_i} prediction') + +plt.tight_layout() + +plt.savefig(f'{species}_model_evaluation.png') diff --git a/Docs/source/usage/workflows/python_extend.rst b/Docs/source/usage/workflows/python_extend.rst new file mode 100644 index 00000000000..d3049508da6 --- /dev/null +++ b/Docs/source/usage/workflows/python_extend.rst @@ -0,0 +1,281 @@ +.. _usage-python-extend: + +Extend a Simulation with Python +=============================== + +When running WarpX directly :ref:`from Python ` it is possible to interact with the simulation. + +For instance, with the :py:meth:`~pywarpx.picmi.Simulation.step` method of the simulation class, one could run ``sim.step(nsteps=1)`` in a loop: + +.. code-block:: python3 + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + steps = 1000 + for _ in range(steps): + sim.step(nsteps=1) + + # do something custom with the sim object + +As a more flexible alternative, one can install `callback functions `__, which will execute a given Python function at a +specific location in the WarpX simulation loop. + +.. automodule:: pywarpx.callbacks + :members: installcallback, uninstallcallback, isinstalled + + +pyAMReX +------- + +Many of the following classes are provided through `pyAMReX `__. +After the simulation is initialized, the pyAMReX module can be accessed via + +.. code-block:: python + + from pywarpx import picmi, libwarpx + + # ... simulation definition ... + + # equivalent to + # import amrex.space3d as amr + # for a 3D simulation + amr = libwarpx.amr # picks the right 1d, 2d or 3d variant + + +Full details for pyAMReX APIs are `documented here `__. +Important APIs include: + +* `amr.ParallelDescriptor `__: MPI-parallel rank information +* `amr.MultiFab `__: MPI-parallel field data +* `amr.ParticleContainer_* `__: MPI-parallel particle data for a particle species + + +Data Access +----------- + +While the simulation is running, callbacks can have read and write access the WarpX simulation data *in situ*. + +An important object in the ``pywarpx.picmi`` module for data access is ``Simulation.extension.warpx``, which is available only during the simulation run. +This object is the Python equivalent to the C++ ``WarpX`` simulation class. + +.. py:class:: WarpX + + .. py:method:: getistep(lev: int) + + Get the current step on mesh-refinement level ``lev``. + + .. py:method:: gett_new(lev: int) + + Get the current physical time on mesh-refinement level ``lev``. + + .. py:method:: getdt(lev: int) + + Get the current physical time step size on mesh-refinement level ``lev``. + + .. py:method:: multifab(multifab_name: str) + + Return MultiFabs by name, e.g., ``"Efield_aux[x][level=0]"``, ``"Efield_cp[x][level=0]"``, ... + + The physical fields in WarpX have the following naming: + + - ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level + - ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. + - ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher). + + .. py:method:: multi_particle_container + + .. py:method:: get_particle_boundary_buffer + + .. py:method:: set_potential_on_eb(potential: str) + + The embedded boundary (EB) conditions can be modified when using the electrostatic solver. + This set the EB potential string and updates the function parser. + + .. py:method:: evolve(numsteps=-1) + + Evolve the simulation the specified number of steps. + + .. autofunction:: pywarpx.picmi.Simulation.extension.finalize + +.. py::def:: pywarpx.picmi.Simulation.extension.get_instance + + Return a reference to the WarpX object. + + +The :py:class:`WarpX` also provides read and write access to field ``MultiFab`` and ``ParticleContainer`` data, shown in the following examples. + +Fields +^^^^^^ + +This example accesses the :math:`E_x(x,y,z)` field at level 0 after every time step and calculate the largest value in it. + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + + @callfromafterstep + def set_E_x(): + warpx = sim.extension.warpx + + # data access + E_x_mf = warpx.multifab(f"Efield_fp[x][level=0]") + + # compute + # iterate over mesh-refinement levels + for lev in range(warpx.finest_level + 1): + # grow (aka guard/ghost/halo) regions + ngv = E_x_mf.n_grow_vect + + # get every local block of the field + for mfi in E_x_mf: + # global index space box, including guards + bx = mfi.tilebox().grow(ngv) + print(bx) # note: global index space of this block + + # numpy representation: non-copying view, including the + # guard/ghost region; .to_cupy() for GPU! + E_x_np = E_x_mf.array(mfi).to_numpy() + + # notes on indexing in E_x_np: + # - numpy uses locally zero-based indexing + # - layout is F_CONTIGUOUS by default, just like AMReX + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For efficiency, use numpy array operation for speed on CPUs. + # For GPUs use .to_cupy() above and compute with cupy or numba. + E_x_np[()] = 42.0 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``E_x``, please see the `pyAMReX documentation `__. + + +High-Level Field Wrapper +"""""""""""""""""""""""" + +.. note:: + + TODO + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + + +Particles +^^^^^^^^^ + +.. code-block:: python3 + + from pywarpx import picmi + from pywarpx.callbacks import callfromafterstep + + # Preparation: set up the simulation + # sim = picmi.Simulation(...) + # ... + + @callfromafterstep + def my_after_step_callback(): + warpx = sim.extension.warpx + Config = sim.extension.Config + + # data access + multi_pc = warpx.multi_particle_container() + pc = multi_pc.get_particle_container_from_name("electrons") + + # compute + # iterate over mesh-refinement levels + for lvl in range(pc.finest_level + 1): + # get every local chunk of particles + for pti in pc.iterator(pc, level=lvl): + # compile-time and runtime attributes in SoA format + soa = pti.soa().to_cupy() if Config.have_gpu else \ + pti.soa().to_numpy() + + # notes: + # Only the next lines are the "HOT LOOP" of the computation. + # For speed, use array operation. + + # write to all particles in the chunk + # note: careful, if you change particle positions, you might need to + # redistribute particles before continuing the simulation step + soa.real[0][()] = 0.30 # x + soa.real[1][()] = 0.35 # y + soa.real[2][()] = 0.40 # z + + # all other attributes: weight, momentum x, y, z, ... + for soa_real in soa.real[3:]: + soa_real[()] = 42.0 + + # by default empty unless ionization or QED physics is used + # or other runtime attributes were added manually + for soa_int in soa.int: + soa_int[()] = 12 + + + sim.step(nsteps=100) + +For further details on how to `access GPU data `__ or compute on ``electrons``, please see the `pyAMReX documentation `__. + + +High-Level Particle Wrapper +""""""""""""""""""""""""""" + +.. note:: + + TODO: What are the benefits of using the high-level wrapper? + TODO: What are the limitations (e.g., in memory usage or compute scalability) of using the high-level wrapper? + +Particles can be added to the simulation at specific positions and with specific attribute values: + +.. code-block:: python + + from pywarpx import particle_containers, picmi + + # ... + + electron_wrapper = particle_containers.ParticleContainerWrapper("electrons") + + +.. autoclass:: pywarpx.particle_containers.ParticleContainerWrapper + :members: + +The ``get_particle_real_arrays()``, ``get_particle_int_arrays()`` and +``get_particle_idcpu_arrays()`` functions are called +by several utility functions of the form ``get_particle_{comp_name}`` where +``comp_name`` is one of ``x``, ``y``, ``z``, ``r``, ``theta``, ``id``, ``cpu``, +``weight``, ``ux``, ``uy`` or ``uz``. + + +Diagnostics +----------- + +Various diagnostics are also accessible from Python. +This includes getting the deposited or total charge density from a given species as well as accessing the scraped particle buffer. +See the example in ``Examples/Tests/ParticleBoundaryScrape`` for a reference on how to interact with scraped particle data. + + +.. autoclass:: pywarpx.particle_containers.ParticleBoundaryBufferWrapper + :members: + + +Modify Solvers +-------------- + +From Python, one can also replace numerical solvers in the PIC loop or add new physical processes into the time step loop. +Examples: + +* :ref:`Capacitive Discharge `: replaces the Poisson solver of an electrostatic simulation (default: MLMG) with a python function that uses `superLU `__ to directly solve the Poisson equation. diff --git a/Docs/spack.yaml b/Docs/spack.yaml index 9f4cc71b184..f17c36e5d97 100644 --- a/Docs/spack.yaml +++ b/Docs/spack.yaml @@ -19,10 +19,13 @@ spack: - doxygen - graphviz - python - - py-sphinx + - py-openpmd-viewer - py-breathe - - py-recommonmark + - py-pybtex - py-pygments + - py-recommonmark + - py-sphinx - py-sphinx-copybutton - py-sphinx-design - py-sphinx-rtd-theme + - py-yt diff --git a/Examples/Physics_applications/beam-beam_collision/README.rst b/Examples/Physics_applications/beam-beam_collision/README.rst new file mode 100644 index 00000000000..559a81277db --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/README.rst @@ -0,0 +1,70 @@ +.. _examples-beam-beam_collision: + +Beam-beam collision +==================== + +This example shows how to simulate the collision between two ultra-relativistic particle beams. +This is representative of what happens at the interaction point of a linear collider. +We consider a right-propagating electron bunch colliding against a left-propagating positron bunch. + +We turn on the Quantum Synchrotron QED module for photon emission (also known as beamstrahlung in the collider community) and +the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). + +To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. + + +This example is based on the following paper :cite:t:`ex-Yakimenko2019`. + + +Run +--- + +The PICMI input file is not available for this example yet. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: inputs + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs``. + + +Visualize +--------- + +The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). + +We compare different results: +* (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; +* (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; +* (black) literature results from :cite:t:`ex-Yakimenko2019`. + +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 128`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: + +.. code-block:: ini + + qed_qs.lookup_table_mode = generate + qed_bw.lookup_table_mode = generate + qed_qs.tab_dndt_chi_min=1e-3 + qed_qs.tab_dndt_chi_max=2e3 + qed_qs.tab_dndt_how_many=512 + qed_qs.tab_em_chi_min=1e-3 + qed_qs.tab_em_chi_max=2e3 + qed_qs.tab_em_chi_how_many=512 + qed_qs.tab_em_frac_how_many=512 + qed_qs.tab_em_frac_min=1e-12 + qed_qs.save_table_in=my_qs_table.txt + qed_bw.tab_dndt_chi_min=1e-2 + qed_bw.tab_dndt_chi_max=2e3 + qed_bw.tab_dndt_how_many=512 + qed_bw.tab_pair_chi_min=1e-2 + qed_bw.tab_pair_chi_max=2e3 + qed_bw.tab_pair_chi_how_many=512 + qed_bw.tab_pair_frac_how_many=512 + qed_bw.save_table_in=my_bw_table.txt + +.. figure:: https://user-images.githubusercontent.com/17280419/291749626-aa61fff2-e6d2-45a3-80ee-84b2851ea0bf.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDMwMzQzNTEsIm5iZiI6MTcwMzAzNDA1MSwicGF0aCI6Ii8xNzI4MDQxOS8yOTE3NDk2MjYtYWE2MWZmZjItZTZkMi00NWEzLTgwZWUtODRiMjg1MWVhMGJmLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFJV05KWUFYNENTVkVINTNBJTJGMjAyMzEyMjAlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjMxMjIwVDAxMDA1MVomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPWFiYzY2MGQyYzIyZGIzYzUxOWI3MzNjZTk5ZDM1YzgyNmY4ZDYxOGRlZjAyZTIwNTAyMTc3NTgwN2Q0YjEwNGMmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JmFjdG9yX2lkPTAma2V5X2lkPTAmcmVwb19pZD0wIn0.I96LQpjqmFXirPDVnBlFQIkCuenR6IuOSY0OIIQvtCo + :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + :width: 100% + + Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. diff --git a/Examples/Physics_applications/beam-beam_collision/inputs b/Examples/Physics_applications/beam-beam_collision/inputs new file mode 100644 index 00000000000..488e997f895 --- /dev/null +++ b/Examples/Physics_applications/beam-beam_collision/inputs @@ -0,0 +1,231 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.beam_energy = 125.*GeV +my_constants.beam_uz = beam_energy/(mc2) +my_constants.beam_charge = 0.14*nano +my_constants.sigmax = 10*nano +my_constants.sigmay = 10*nano +my_constants.sigmaz = 10*nano +my_constants.beam_uth = 0.1/100.*beam_uz +my_constants.n0 = beam_charge / (q_e * sigmax * sigmay * sigmaz * (2.*pi)**(3./2.)) +my_constants.omegab = sqrt(n0 * q_e**2 / (epsilon0*m_e)) +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = -0.5*Lz+3.2*sigmaz + +# BOX +my_constants.Lx = 100.0*clight/omegab +my_constants.Ly = 100.0*clight/omegab +my_constants.Lz = 180.0*clight/omegab + +# for a full scale simulation use: nx, ny, nz = 512, 512, 1024 +my_constants.nx = 64 +my_constants.ny = 64 +my_constants.nz = 128 + + +# TIME +my_constants.T = 0.7*Lz/clight +my_constants.dt = sigmaz/clight/10. + +# DIAGS +my_constants.every_red = 1. +warpx.used_inputs_file = warpx_used_inputs.txt + +################################# +####### GENERAL PARAMETERS ###### +################################# +stop_time = T +amr.n_cell = nx ny nz +amr.max_grid_size = 128 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +warpx.do_electrostatic = relativistic +warpx.const_dt = dt +warpx.grid_type = collocated +algo.particle_shape = 3 +algo.load_balance_intervals=100 +algo.particle_pusher = vay + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 beam2 pho1 pho2 ele1 pos1 ele2 pos2 +particles.photon_species = pho1 pho2 + +beam1.species_type = electron +beam1.injection_style = NUniformPerCell +beam1.num_particles_per_cell_each_dim = 1 1 1 +beam1.profile = parse_density_function +beam1.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z-muz)**2/(2*sigmaz**2))" +beam1.density_min = n0 / 1e3 +beam1.momentum_distribution_type = gaussian +beam1.uz_m = beam_uz +beam1.uy_m = 0.0 +beam1.ux_m = 0.0 +beam1.ux_th = beam_uth +beam1.uy_th = beam_uth +beam1.uz_th = beam_uth +beam1.initialize_self_fields = 1 +beam1.self_fields_required_precision = 5e-10 +beam1.self_fields_max_iters = 10000 +beam1.do_qed_quantum_sync = 1 +beam1.qed_quantum_sync_phot_product_species = pho1 +beam1.do_classical_radiation_reaction = 0 + +beam2.species_type = positron +beam2.injection_style = NUniformPerCell +beam2.num_particles_per_cell_each_dim = 1 1 1 +beam2.profile = parse_density_function +beam2.density_function(x,y,z) = "n0 * exp(-(x-mux)**2/(2*sigmax**2)) * exp(-(y-muy)**2/(2*sigmay**2)) * exp(-(z+muz)**2/(2*sigmaz**2))" +beam2.density_min = n0 / 1e3 +beam2.momentum_distribution_type = gaussian +beam2.uz_m = -beam_uz +beam2.uy_m = 0.0 +beam2.ux_m = 0.0 +beam2.ux_th = beam_uth +beam2.uy_th = beam_uth +beam2.uz_th = beam_uth +beam2.initialize_self_fields = 1 +beam2.self_fields_required_precision = 5e-10 +beam2.self_fields_max_iters = 10000 +beam2.do_qed_quantum_sync = 1 +beam2.qed_quantum_sync_phot_product_species = pho2 +beam2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +ele1.species_type = electron +ele1.injection_style = none +ele1.self_fields_required_precision = 1e-11 +ele1.self_fields_max_iters = 10000 +ele1.do_qed_quantum_sync = 1 +ele1.qed_quantum_sync_phot_product_species = pho1 +ele1.do_classical_radiation_reaction = 0 + +pos1.species_type = positron +pos1.injection_style = none +pos1.self_fields_required_precision = 1e-11 +pos1.self_fields_max_iters = 10000 +pos1.do_qed_quantum_sync = 1 +pos1.qed_quantum_sync_phot_product_species = pho1 +pos1.do_classical_radiation_reaction = 0 + +ele2.species_type = electron +ele2.injection_style = none +ele2.self_fields_required_precision = 1e-11 +ele2.self_fields_max_iters = 10000 +ele2.do_qed_quantum_sync = 1 +ele2.qed_quantum_sync_phot_product_species = pho2 +ele2.do_classical_radiation_reaction = 0 + +pos2.species_type = positron +pos2.injection_style = none +pos2.self_fields_required_precision = 1e-11 +pos2.self_fields_max_iters = 10000 +pos2.do_qed_quantum_sync = 1 +pos2.qed_quantum_sync_phot_product_species = pho2 +pos2.do_classical_radiation_reaction = 0 + +pho1.species_type = photon +pho1.injection_style = none +pho1.do_qed_breit_wheeler = 1 +pho1.qed_breit_wheeler_ele_product_species = ele1 +pho1.qed_breit_wheeler_pos_product_species = pos1 + +pho2.species_type = photon +pho2.injection_style = none +pho2.do_qed_breit_wheeler = 1 +pho2.qed_breit_wheeler_ele_product_species = ele2 +pho2.qed_breit_wheeler_pos_product_species = pos2 + +################################# +############# QED ############### +################################# +qed_qs.photon_creation_energy_threshold = 0. + +qed_qs.lookup_table_mode = builtin +qed_qs.chi_min = 1.e-3 + +qed_bw.lookup_table_mode = builtin +qed_bw.chi_min = 1.e-2 + +# for accurate results use the generated tables with +# the following parameters +# note: must compile with -DWarpX_QED_TABLE_GEN=ON +#qed_qs.lookup_table_mode = generate +#qed_bw.lookup_table_mode = generate +#qed_qs.tab_dndt_chi_min=1e-3 +#qed_qs.tab_dndt_chi_max=2e3 +#qed_qs.tab_dndt_how_many=512 +#qed_qs.tab_em_chi_min=1e-3 +#qed_qs.tab_em_chi_max=2e3 +#qed_qs.tab_em_chi_how_many=512 +#qed_qs.tab_em_frac_how_many=512 +#qed_qs.tab_em_frac_min=1e-12 +#qed_qs.save_table_in=my_qs_table.txt +#qed_bw.tab_dndt_chi_min=1e-2 +#qed_bw.tab_dndt_chi_max=2e3 +#qed_bw.tab_dndt_how_many=512 +#qed_bw.tab_pair_chi_min=1e-2 +#qed_bw.tab_pair_chi_max=2e3 +#qed_bw.tab_pair_chi_how_many=512 +#qed_bw.tab_pair_frac_how_many=512 +#qed_bw.save_table_in=my_bw_table.txt + +warpx.do_qed_schwinger = 0. + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 + +diag1.intervals = 0 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 +diag1.format = openpmd +diag1.dump_last_timestep = 1 +diag1.species = pho1 pho2 ele1 pos1 ele2 pos2 beam1 beam2 + +# REDUCED +warpx.reduced_diags_names = ParticleNumber ColliderRelevant_beam1_beam2 + +ColliderRelevant_beam1_beam2.type = ColliderRelevant +ColliderRelevant_beam1_beam2.intervals = every_red +ColliderRelevant_beam1_beam2.species = beam1 beam2 + +ParticleNumber.type = ParticleNumber +ParticleNumber.intervals = every_red diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 0c6f9828543..01406b7a0fb 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# --- Copyright 2021 Modern Electron +# --- Copyright 2021 Modern Electron (DSMC test added in 2023 by TAE Technologies) # --- Monte-Carlo Collision script to reproduce the benchmark tests from # --- Turner et al. (2013) - https://doi.org/10.1063/1.4775084 @@ -38,7 +38,7 @@ def __init__(self, grid, **kwargs): required_precision=1, **kwargs ) - def initialize_inputs(self): + def solver_initialize_inputs(self): """Grab geometrical quantities from the grid. The boundary potentials are also obtained from the grid using 'warpx_potential_zmin' for the left_voltage and 'warpx_potential_zmax' for the right_voltage. @@ -46,9 +46,7 @@ def initialize_inputs(self): WarpX parser. """ # grab the boundary potentials from the grid object - self.right_voltage = ( - self.grid.potential_zmax.replace('sin', 'np.sin').replace('pi', 'np.pi') - ) + self.right_voltage = self.grid.potential_zmax # set WarpX boundary potentials to None since we will handle it # ourselves in this solver @@ -59,7 +57,7 @@ def initialize_inputs(self): self.grid.potential_zmin = None self.grid.potential_zmax = None - super(PoissonSolver1D, self).initialize_inputs() + super(PoissonSolver1D, self).solver_initialize_inputs() self.nz = self.grid.number_of_cells[0] self.dz = (self.grid.upper_bound[0] - self.grid.lower_bound[0]) / self.nz @@ -108,8 +106,12 @@ def solve(self): calculating phi from rho.""" left_voltage = 0.0 - t = self.sim.extension.warpx.gett_new(0) - right_voltage = eval(self.right_voltage) + right_voltage = eval( + self.right_voltage, { + 't': self.sim.extension.warpx.gett_new(0), + 'sin': np.sin, 'pi': np.pi + } + ) # Construct b vector rho = -self.rho_data / constants.ep0 @@ -158,11 +160,12 @@ class CapacitiveDischargeExample(object): # Time (in seconds) between diagnostic evaluations diag_interval = 32 / freq - def __init__(self, n=0, test=False, pythonsolver=False): + def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): """Get input parameters for the specific case (n) desired.""" self.n = n self.test = test self.pythonsolver = pythonsolver + self.dsmc = dsmc # Case specific input parameters self.voltage = f"{self.voltage[n]}*sin(2*pi*{self.freq:.5e}*t)" @@ -184,6 +187,9 @@ def __init__(self, n=0, test=False, pythonsolver=False): else: self.mcc_subcycling_steps = None + if self.dsmc: + self.rng = np.random.default_rng(23094290) + self.ion_density_array = np.zeros(self.nz + 1) self.setup_run() @@ -236,13 +242,26 @@ def setup_run(self): rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, ) ) + if self.dsmc: + self.neutrals = picmi.Species( + particle_type='He', name='neutrals', + charge=0, mass=self.m_ion, + warpx_reflection_model_zlo=1.0, + warpx_reflection_model_zhi=1.0, + warpx_do_resampling=True, + warpx_resampling_trigger_max_avg_ppc=int(self.seed_nppc*1.5), + initial_distribution=picmi.UniformDistribution( + density=self.gas_density, + rms_velocity=[np.sqrt(constants.kb * self.gas_temp / self.m_ion)]*3, + ) + ) ####################################################################### # Collision initialization # ####################################################################### cross_sec_direc = '../../../../warpx-data/MCC_cross_sections/He/' - mcc_electrons = picmi.MCCCollisions( + electron_colls = picmi.MCCCollisions( name='coll_elec', species=self.electrons, background_density=self.gas_density, @@ -269,24 +288,26 @@ def setup_run(self): } ) - mcc_ions = picmi.MCCCollisions( - name='coll_ion', - species=self.ions, - background_density=self.gas_density, - background_temperature=self.gas_temp, - ndt=self.mcc_subcycling_steps, - scattering_processes={ - 'elastic' : { - 'cross_section' : cross_sec_direc+'ion_scattering.dat' - }, - 'back' : { - 'cross_section' : cross_sec_direc+'ion_back_scatter.dat' - }, - # 'charge_exchange' : { - # 'cross_section' : cross_sec_direc+'charge_exchange.dat' - # } - } - ) + ion_scattering_processes={ + 'elastic': {'cross_section': cross_sec_direc+'ion_scattering.dat'}, + 'back': {'cross_section': cross_sec_direc+'ion_back_scatter.dat'}, + # 'charge_exchange': {'cross_section': cross_sec_direc+'charge_exchange.dat'} + } + if self.dsmc: + ion_colls = picmi.DSMCCollisions( + name='coll_ion', + species=[self.ions, self.neutrals], + ndt=5, scattering_processes=ion_scattering_processes + ) + else: + ion_colls = picmi.MCCCollisions( + name='coll_ion', + species=self.ions, + background_density=self.gas_density, + background_temperature=self.gas_temp, + ndt=self.mcc_subcycling_steps, + scattering_processes=ion_scattering_processes + ) ####################################################################### # Initialize simulation # @@ -296,8 +317,7 @@ def setup_run(self): solver=self.solver, time_step_size=self.dt, max_steps=self.max_steps, - warpx_collisions=[mcc_electrons, mcc_ions], - warpx_load_balance_intervals=self.max_steps//5000, + warpx_collisions=[electron_colls, ion_colls], verbose=self.test ) self.solver.sim = self.sim @@ -314,18 +334,36 @@ def setup_run(self): n_macroparticle_per_cell=[self.seed_nppc], grid=self.grid ) ) + if self.dsmc: + self.sim.add_species( + self.neutrals, + layout = picmi.GriddedLayout( + n_macroparticle_per_cell=[self.seed_nppc//2], grid=self.grid + ) + ) + self.solver.sim_ext = self.sim.extension + + if self.dsmc: + # Periodically reset neutral density to starting temperature + callbacks.installbeforecollisions(self.rethermalize_neutrals) ####################################################################### # Add diagnostics for the CI test to be happy # ####################################################################### - if self.pythonsolver: - file_prefix = 'Python_background_mcc_1d_plt' + if self.dsmc: + file_prefix = 'Python_dsmc_1d_plt' else: - file_prefix = 'Python_background_mcc_1d_tridiag_plt' - + if self.pythonsolver: + file_prefix = 'Python_background_mcc_1d_plt' + else: + file_prefix = 'Python_background_mcc_1d_tridiag_plt' + + species = [self.electrons, self.ions] + if self.dsmc: + species.append(self.neutrals) particle_diag = picmi.ParticleDiagnostic( - species=[self.electrons, self.ions], + species=species, name='diag1', period=0, write_dir='.', @@ -342,6 +380,30 @@ def setup_run(self): self.sim.add_diagnostic(particle_diag) self.sim.add_diagnostic(field_diag) + def rethermalize_neutrals(self): + # When using DSMC the neutral temperature will change due to collisions + # with the ions. This is not captured in the original MCC test. + # Re-thermalize the neutrals every 1000 steps + step = self.sim.extension.warpx.getistep(lev=0) + if step % 1000 != 10: + return + + if not hasattr(self, 'neutral_cont'): + self.neutral_cont = particle_containers.ParticleContainerWrapper( + self.neutrals.name + ) + + ux_arrays = self.neutral_cont.uxp + uy_arrays = self.neutral_cont.uyp + uz_arrays = self.neutral_cont.uzp + + vel_std = np.sqrt(constants.kb * self.gas_temp / self.m_ion) + for ii in range(len(ux_arrays)): + nps = len(ux_arrays[ii]) + ux_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uy_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + uz_arrays[ii][:] = vel_std * self.rng.normal(size=nps) + def _get_rho_ions(self): # deposit the ion density in rho_fp he_ions_wrapper = particle_containers.ParticleContainerWrapper('he_ions') @@ -359,6 +421,10 @@ def run_sim(self): self.sim.step(self.diag_steps) + if self.pythonsolver: + # confirm that the external solver was run + assert hasattr(self.solver, 'phi') + if libwarpx.amr.ParallelDescriptor.MyProc() == 0: np.save(f'ion_density_case_{self.n+1}.npy', self.ion_density_array) @@ -389,11 +455,17 @@ def run_sim(self): '--pythonsolver', help='toggle whether to use the Python level solver', action='store_true' ) +parser.add_argument( + '--dsmc', help='toggle whether to use DSMC for ions in place of MCC', + action='store_true' +) args, left = parser.parse_known_args() sys.argv = sys.argv[:1]+left if args.n < 1 or args.n > 4: raise AttributeError('Test number must be an integer from 1 to 4.') -run = CapacitiveDischargeExample(n=args.n-1, test=args.test, pythonsolver=args.pythonsolver) +run = CapacitiveDischargeExample( + n=args.n-1, test=args.test, pythonsolver=args.pythonsolver, dsmc=args.dsmc +) run.run_sim() diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 71e1070ef2f..65baabba605 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -75,7 +75,7 @@ def __init__(self, grid, **kwargs): self.phi_wrapper = None self.time_sum = 0.0 - def initialize_inputs(self): + def solver_initialize_inputs(self): """Grab geometrical quantities from the grid. """ self.right_voltage = self.grid.potential_xmax @@ -89,7 +89,7 @@ def initialize_inputs(self): self.grid.potential_zmin = None self.grid.potential_zmax = None - super(PoissonSolverPseudo1D, self).initialize_inputs() + super(PoissonSolverPseudo1D, self).solver_initialize_inputs() self.nx = self.grid.number_of_cells[0] self.nz = self.grid.number_of_cells[1] @@ -359,3 +359,6 @@ def solve(self): ########################## sim.step(max_steps) + +# confirm that the external solver was run +assert hasattr(solver, 'phi') diff --git a/Examples/Physics_applications/capacitive_discharge/README.md b/Examples/Physics_applications/capacitive_discharge/README.md deleted file mode 100644 index f5653b2831a..00000000000 --- a/Examples/Physics_applications/capacitive_discharge/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The examples in this directory are based on the benchmark cases from Turner et -al. (Phys. Plasmas 20, 013507, 2013). -See the 'Examples' section in the documentation for previously computed -comparisons between WarpX and the literature results. -The 1D PICMI input file can be used to reproduce the results from Turner et al. -for a given case, N, by executing: - `python3 PICMI_inputs_1d.py -n N` diff --git a/Examples/Physics_applications/capacitive_discharge/README.rst b/Examples/Physics_applications/capacitive_discharge/README.rst new file mode 100644 index 00000000000..708b4528cd5 --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/README.rst @@ -0,0 +1,54 @@ +.. _examples-capacitive-discharge: + +Capacitive Discharge +==================== + +The examples in this directory are based on the benchmark cases from Turner et al. (Phys. Plasmas 20, 013507, 2013) :cite:p:`ex-Turner2013`. + +The Monte-Carlo collision (MCC) model can be used to simulate electron and ion collisions with a neutral background gas. +In particular this can be used to study capacitive discharges between parallel plates. +The implementation has been tested against the benchmark results from :cite:t:`ex-Turner2013`. + +.. note:: + + This example needs `additional calibration data for cross sections `__. + Download this data alongside your inputs file and update the paths in the inputs file: + + .. code-block:: bash + + git clone https://github.com/ECP-WarpX/warpx-data.git + + +Run +--- + +The 1D PICMI input file can be used to reproduce the results from Turner et al. for a given case, ``N`` from 1 to 4, by executing ``python3 PICMI_inputs_1d.py -n N``, e.g., + +.. code-block:: bash + + python3 PICMI_inputs_1d.py -n 1 + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. literalinclude:: PICMI_inputs_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py``. + + +Analyze +------- + +Once the simulation completes an output file ``avg_ion_density.npy`` will be created which can be compared to the literature results as in the plot below. +Running case ``1`` on four CPU processors takes roughly 20 minutes to complete. + + +Visualize +--------- + +The figure below shows a comparison of the ion density as calculated in WarpX (in June 2022 with `PR #3118 `_) compared to the literature results (which can be found `in the supplementary materials of Turner et al. `__). + +.. figure:: https://user-images.githubusercontent.com/40245517/171573007-f7d733c7-c0de-490c-9ed6-ff4c02154358.png + :alt: MCC benchmark against :cite:t:`ex-Turner2013`. + :width: 80% + + MCC benchmark against :cite:t:`ex-Turner2013`. diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py new file mode 100755 index 00000000000..a7a76be46ad --- /dev/null +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# 2023 TAE Technologies + +import os +import sys + +import numpy as np + +sys.path.append('../../../../warpx/Regression/Checksum/') + +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] + +my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) + +ref_density = np.array([ + 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, + 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, + 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, + 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, + 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, + 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, + 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, + 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, + 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, + 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, + 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, + 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, + 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, + 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, + 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, + 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, + 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, + 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, + 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, + 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, + 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, + 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, + 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, + 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, + 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, + 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, + 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, + 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, + 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, + 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, + 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, + 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, + 1.27720853e+14 +]) + +density_data = np.load( 'ion_density_case_1.npy' ) +print(repr(density_data)) +assert np.allclose(density_data, ref_density) diff --git a/Examples/Physics_applications/laser_acceleration/README.md b/Examples/Physics_applications/laser_acceleration/README.md deleted file mode 100644 index 01a87fdce8c..00000000000 --- a/Examples/Physics_applications/laser_acceleration/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Examples of laser acceleration simulations. - -Examples are provided using the executable or Python-driven version of WarpX. - -## Using the executable version: - - inputs.2d: 2d simulation of LWFA in the laboratory frame. - - inputs.3d: 3d simulation of LWFA in the laboratory frame. - - inputs.2d.boost: 2d simulation of LWFA in a boosted frame. This script also - uses rigid injection and a parser for the plasma density. - -## Using the python-driven version: - - laser_acceleration_PICMI.py diff --git a/Examples/Physics_applications/laser_acceleration/README.rst b/Examples/Physics_applications/laser_acceleration/README.rst new file mode 100644 index 00000000000..0ba3b5382f2 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/README.rst @@ -0,0 +1,87 @@ +.. _examples-lwfa: + +Laser-Wakefield Acceleration of Electrons +========================================= + +This example shows how to model a laser-wakefield accelerator (LWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. + +Laser-wakefield acceleration is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (laser diffraction, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round laser driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. + +For LWFA scenarios with long propagation lengths, use the :ref:`boosted frame method `. +An example can be seen in the :ref:`PWFA example `. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_3d.py`` or + * WarpX **executable** using an input file: ``warpx.3d inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_3d``. + + .. tab-item:: RZ + + This example can be run **either** as: + + * **Python** script: ``python3 PICMI_inputs_rz.py`` or + * WarpX **executable** using an input file: ``warpx.rz inputs_3d max_step=400`` + + .. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/inputs_rz``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +You can run the following script to visualize the beam evolution over time: + +.. dropdown:: Script ``plot_3d.py`` + + .. literalinclude:: plot_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_acceleration/plot_3d.py diags/diag1000400/``. + +.. figure:: https://user-images.githubusercontent.com/1353258/287800852-f994a020-4ecc-4987-bffc-2cb7df6144a9.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTEiLCJleHAiOjE3MDE3MTYyNjksIm5iZiI6MTcwMTcxNTk2OSwicGF0aCI6Ii8xMzUzMjU4LzI4NzgwMDg1Mi1mOTk0YTAyMC00ZWNjLTQ5ODctYmZmYy0yY2I3ZGY2MTQ0YTkucG5nP1gtQW16LUFsZ29yaXRobT1BV1M0LUhNQUMtU0hBMjU2JlgtQW16LUNyZWRlbnRpYWw9QUtJQUlXTkpZQVg0Q1NWRUg1M0ElMkYyMDIzMTIwNCUyRnVzLWVhc3QtMSUyRnMzJTJGYXdzNF9yZXF1ZXN0JlgtQW16LURhdGU9MjAyMzEyMDRUMTg1MjQ5WiZYLUFtei1FeHBpcmVzPTMwMCZYLUFtei1TaWduYXR1cmU9NDkyNWJkMTg2NWM3ZjcwZjVkMjlmNDE1NmRjNWEyZWM5MzgxMWJhZTVjMGMxNjdkZDg1Zjk0NmQ1NGEwMjNiMiZYLUFtei1TaWduZWRIZWFkZXJzPWhvc3QmYWN0b3JfaWQ9MCZrZXlfaWQ9MCZyZXBvX2lkPTAifQ.C_NQceQcqiCDzBoSzIjm3c8QdTLNDdtjJmkQjkhW4c8 + :alt: (top) Electric field of the laser pulse and (bottom) absolute density. + + (top) Electric field of the laser pulse and (bottom) absolute density. diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py index a376ed872a9..a33b82ebc02 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py @@ -90,7 +90,7 @@ def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): # Remove the ions rho_th = rho_th - e*n0 -# Dicate which region to compare solutions over +# Dictate which region to compare solutions over # (Currently this is the full domain) min_i = 0 max_i = 10240 diff --git a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py index 4f4ae2a812a..30301996921 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py @@ -90,7 +90,7 @@ def odefcn(phi, xi, kp, a0, c, tau, xi_0, lambda_laser): # Remove the ions rho_th = rho_th - e*n0 -# Dicate which region to compare solutions over (cuttoff 0's from BTD extra) +# Dictate which region to compare solutions over (cuttoff 0's from BTD extra) min_i = 200 max_i = 4864 diff --git a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py index e7d26d2cb30..a66c838fe9d 100755 --- a/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py +++ b/Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py @@ -43,7 +43,7 @@ n_move = 192 # ref ratio = 2 1 -# Refined only transversly. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions +# Refined only transversely. Longitudinal spacing between particles in each stream is the same in both coarse and fine regions rr_longitudinal = 1 np_expected = (n_coarse + n_fine*rr_longitudinal)*(n_0 + n_move) diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index 27ecc00117b..bdcfd7676a4 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -77,6 +77,7 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho +diag1.format = openpmd # Reduced Diagnostics warpx.reduced_diags_names = FP diff --git a/Examples/Physics_applications/laser_acceleration/plot_3d.py b/Examples/Physics_applications/laser_acceleration/plot_3d.py new file mode 100755 index 00000000000..00222ff43c8 --- /dev/null +++ b/Examples/Physics_applications/laser_acceleration/plot_3d.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl +# License: BSD-3-Clause-LBNL +# +# This is a script plots the wakefield of an LWFA simulation. + +import sys + +import matplotlib.pyplot as plt +import yt + +yt.funcs.mylog.setLevel(50) + + +def plot_lwfa(): + # this will be the name of the plot file + fn = sys.argv[1] + + # Read the file + ds = yt.load(fn) + + # plot the laser field and absolute density + fields = ["Ey", "rho"] + normal = "y" + sl = yt.SlicePlot(ds, normal=normal, fields=fields) + for field in fields: + sl.set_log(field, False) + + sl.set_figure_size((4, 8)) + fig = sl.export_to_mpl_figure(nrows_ncols=(2, 1)) + fig.tight_layout() + plt.show() + +if __name__ == "__main__": + plot_lwfa() diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py new file mode 100755 index 00000000000..844501992c3 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +from pywarpx import picmi + +# Physical constants +c = picmi.constants.c +q_e = picmi.constants.q_e + +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 +# time-scale with highly kinetic dynamics +stop_time = 0.2e-12 + +# proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length +# (>=6x V100) +# --> choose larger `max_grid_size` and `blocking_factor` for 1 to 8 grids per GPU accordingly +#nx = 7488 +#nz = 14720 + +# Number of cells +nx = 384 +nz = 512 + +# Domain decomposition (deactivate `warpx_numprocs` in `picmi.Simulation` for this to take effect) +max_grid_size = 64 +blocking_factor = 32 + +# Physical domain +xmin = -7.5e-06 +xmax = 7.5e-06 +zmin = -5.0e-06 +zmax = 25.0e-06 + +# Create grid +grid = picmi.Cartesian2DGrid( + number_of_cells=[nx, nz], + lower_bound=[xmin, zmin], + upper_bound=[xmax, zmax], + lower_boundary_conditions=['open', 'open'], + upper_boundary_conditions=['open', 'open'], + lower_boundary_conditions_particles=['absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing'], + warpx_max_grid_size=max_grid_size, + warpx_blocking_factor=blocking_factor) + +# Particles: plasma parameters +# critical plasma density +nc = 1.742e27 # [m^-3] 1.11485e21 * 1.e6 / 0.8**2 +# number density: "fully ionized" electron density as reference +# [material 1] cryogenic H2 +n0 = 30.0 # [n_c] +# [material 2] liquid crystal +# n0 = 192 +# [material 3] PMMA +# n0 = 230 +# [material 4] Copper (ion density: 8.49e28/m^3; times ionization level) +# n0 = 1400 +plasma_density = n0 * nc +preplasma_L = 0.05e-6 # [m] scale length (>0) +preplasma_Lcut = 2.0e-6 # [m] hard cutoff from surface +plasma_r0 = 2.5e-6 # [m] radius or half-thickness +plasma_eps_z = 0.05e-6 # [m] small offset in z to make zmin, zmax interval larger than 2*(r0 + Lcut) +plasma_creation_limit_z = plasma_r0 + preplasma_Lcut + plasma_eps_z # [m] upper limit in z for particle creation + +plasma_xmin = None +plasma_ymin = None +plasma_zmin = -plasma_creation_limit_z +plasma_xmax = None +plasma_ymax = None +plasma_zmax = plasma_creation_limit_z + +density_expression_str = f'{plasma_density}*((abs(z)<={plasma_r0}) + (abs(z)<{plasma_r0}+{preplasma_Lcut}) * (abs(z)>{plasma_r0}) * exp(-(abs(z)-{plasma_r0})/{preplasma_L}))' + +slab_with_ramp_dist_hydrogen = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax] +) + +# thermal velocity spread for electrons in gamma*beta +ux_th = .01 +uz_th = .01 + +slab_with_ramp_dist_electrons = picmi.AnalyticDistribution( + density_expression=density_expression_str, + lower_bound=[plasma_xmin, plasma_ymin, plasma_zmin], + upper_bound=[plasma_xmax, plasma_ymax, plasma_zmax], + # if `momentum_expressions` and `momentum_spread_expressions` are unset, + # a Gaussian momentum distribution is assumed given that `rms_velocity` has any non-zero elements + rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s +) + +# TODO: add additional attributes orig_x and orig_z +electrons = picmi.Species( + particle_type='electron', + name='electrons', + initial_distribution=slab_with_ramp_dist_electrons, +) + +# TODO: add additional attributes orig_x and orig_z +hydrogen = picmi.Species( + particle_type='proton', + name='hydrogen', + initial_distribution=slab_with_ramp_dist_hydrogen +) + +# Laser +# e_max = a0 * 3.211e12 / lambda_0[mu] +# a0 = 16, lambda_0 = 0.8mu -> e_max = 64.22 TV/m +e_max = 64.22e12 +position_z = -4.0e-06 +profile_t_peak = 50.e-15 +profile_focal_distance = 4.0e-06 +laser = picmi.GaussianLaser( + wavelength=0.8e-06, + waist=4.e-06, + duration=30.e-15, + focal_position=[0, 0, profile_focal_distance + position_z], + centroid_position=[0, 0, position_z - c * profile_t_peak], + propagation_direction=[0, 0, 1], + polarization_direction=[1, 0, 0], + E0=e_max, + fill_in=False) +laser_antenna = picmi.LaserAntenna( + position=[0., 0., position_z], + normal_vector=[0, 0, 1]) + +# Electromagnetic solver +solver = picmi.ElectromagneticSolver( + grid=grid, + method='Yee', + cfl=0.999, + divE_cleaning=0, + #warpx_pml_ncell=10 +) + +# Diagnostics +particle_diag = picmi.ParticleDiagnostic( + name='Python_LaserIonAcc2d_plt', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + # demonstration of a spatial and momentum filter + warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' +) +# reduce resolution of output fields +coarsening_ratio = [4, 4] +ncell_field = [] +for (ncell_comp, cr) in zip([nx,nz], coarsening_ratio): + ncell_field.append(int(ncell_comp/cr)) +field_diag = picmi.FieldDiagnostic( + name='Python_LaserIonAcc2d_plt', + grid=grid, + period=100, + number_of_cells=ncell_field, + data_list=['B', 'E', 'J', 'rho', 'rho_electrons', 'rho_hydrogen'], + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5' +) + +particle_fw_diag = picmi.ParticleDiagnostic( + name='openPMDfw', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + warpx_plot_filter_function='(uz>=0) * (x<1.0e-6) * (x>-1.0e-6)' +) + +particle_bw_diag = picmi.ParticleDiagnostic( + name='openPMDbw', + period=100, + write_dir='./diags', + warpx_format='openpmd', + warpx_openpmd_backend='h5', + warpx_plot_filter_function='(uz<0)' +) + +# histograms with 2.0 degree acceptance angle in fw direction +# 2 deg * pi / 180 : 0.03490658503 rad +# half-angle +/- : 0.017453292515 rad +histuH_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histuH', + period=100, + species=hydrogen, + bin_number=1000, + bin_min=0.0, + bin_max=0.474, # 100 MeV protons + histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', + filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') + +histue_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histue', + period=100, + species=electrons, + bin_number=1000, + bin_min=0.0, + bin_max=197.0, # 100 MeV electrons + histogram_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)', + filter_function='u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)') + +# just a test entry to make sure that the histogram filter is purely optional: +# this one just records uz of all hydrogen ions, independent of their pointing +histuzAll_rdiag = picmi.ReducedDiagnostic( + diag_type='ParticleHistogram', + name='histuzAll', + period=100, + species=hydrogen, + bin_number=1000, + bin_min=-0.474, + bin_max=0.474, + histogram_function='uz') + +field_probe_z_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_Z', + period=100, + integrate=0, + probe_geometry='Line', + x_probe=0.0, + z_probe=-5.0e-6, + x1_probe=0.0, + z1_probe=25.0e-6, + resolution=3712) + +field_probe_scat_point_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_ScatPoint', + period=1, + integrate=0, + probe_geometry='Point', + x_probe=0.0, + z_probe=15.0e-6) + +field_probe_scat_line_rdiag = picmi.ReducedDiagnostic( + diag_type='FieldProbe', + name='FieldProbe_ScatLine', + period=100, + integrate=1, + probe_geometry='Line', + x_probe=-2.5e-6, + z_probe=15.0e-6, + x1_probe=2.5e-6, + z1_probe=15e-6, + resolution=201) + +load_balance_costs_rdiag = picmi.ReducedDiagnostic( + diag_type='LoadBalanceCosts', + name='LBC', + period=100) + +# Set up simulation +sim = picmi.Simulation( + solver=solver, + max_time=stop_time, # need to remove `max_step` to run this far + verbose=1, + particle_shape='cubic', + warpx_numprocs=[1, 2], # deactivate `numprocs` for dynamic load balancing + warpx_use_filter=1, + warpx_load_balance_intervals=100, + warpx_load_balance_costs_update='heuristic' +) + +# Add plasma electrons +sim.add_species( + electrons, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add hydrogen ions +sim.add_species( + hydrogen, + layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[2,2]) + # for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c + #layout=picmi.GriddedLayout(grid=grid, n_macroparticle_per_cell=[4,8]) +) + +# Add laser +sim.add_laser( + laser, + injection_method=laser_antenna) + +# Add full diagnostics +sim.add_diagnostic(particle_diag) +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_fw_diag) +sim.add_diagnostic(particle_bw_diag) +# Add reduced diagnostics +sim.add_diagnostic(histuH_rdiag) +sim.add_diagnostic(histue_rdiag) +sim.add_diagnostic(histuzAll_rdiag) +sim.add_diagnostic(field_probe_z_rdiag) +sim.add_diagnostic(field_probe_scat_point_rdiag) +sim.add_diagnostic(field_probe_scat_line_rdiag) +sim.add_diagnostic(load_balance_costs_rdiag) +# TODO: make ParticleHistogram2D available + +# Write input file that can be used to run with the compiled version +sim.write_input_file(file_name='inputs_2d_picmi') + +# Initialize inputs and WarpX instance +sim.initialize_inputs() +sim.initialize_warpx() + +# Advance simulation until last time step +sim.step(max_step) diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst new file mode 100644 index 00000000000..18a427aa1c7 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -0,0 +1,110 @@ +.. _examples-laser-ion: + +Laser-Ion Acceleration with a Planar Target +=========================================== + +This example shows how to model laser-ion acceleration with planar targets of solid density :cite:p:`ex-Wilks2001,ex-Bulanov2008,ex-Macchi2013`. +The acceleration mechanism in this scenario depends on target parameters. + +Although laser-ion acceleration requires full 3D modeling for adequate description of the acceleration dynamics, especially the acceleration field lengths and decay times, this example models a 2D example. +2D modeling can often hint at a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + The resolution of this 2D case is extremely low by default. + This includes spatial and temporal resolution, but also the number of macro-particles per cell representing the target density for proper phase space sampling. + You will need a computing cluster for adequate resolution of the target density, see comments in the input file. + +.. warning:: + + It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. + The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. + Not setting these parameters can quickly lead to memory overflows. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or +* WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py``. + + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/inputs_2d``. + +Analyze +------- + +.. _fig-tnsa-ps-electrons-pinhole: + +.. figure:: https://user-images.githubusercontent.com/5416860/295003882-c755fd47-4bb3-4439-9319-c48214cbaafd.png + :alt: Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + :width: 100% + + Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + +.. _fig-tnsa-ps-protons-pinhole: + +.. figure:: https://user-images.githubusercontent.com/5416860/295003988-dea3dfb7-0d55-4616-b32d-061fb429f9ac.png + :alt: Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + :width: 100% + + Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + +Time-resolved phase electron space analysis as in :numref:`fig-tnsa-ps-electrons-pinhole` gives information about, e.g., how laser energy is locally converted into electron kinetic energy. +Later in time, ion phase spaces like :numref:`fig-tnsa-ps-protons-pinhole` can reveal where accelerated ion populations originate. + +.. dropdown:: Script ``analysis_histogram_2D.py`` + + .. literalinclude:: analysis_histogram_2D.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/analysis_histogram_2D.py``. + +Visualize +--------- + +.. note:: + + The following images for densities and electromagnetic fields were created with a run on 64 NVidia A100 GPUs featuring a total number of cells of ``nx = 8192`` and ``nz = 16384``, as well as 64 particles per cell per species. + +.. _fig-tnsa-densities: + +.. figure:: https://user-images.githubusercontent.com/5416860/296338802-8059c39c-0be8-4e4d-b41b-f976b626bd7f.png + :alt: Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + :width: 80% + + Particle densities for electrons (top), protons (middle), and electrons again in logarithmic scale (bottom). + +Particle density output illustrates the evolution of the target in time and space. +Logarithmic scales can help to identify where the target becomes transparent for the laser pulse (bottom panel in :numref:`fig-tnsa-densities` ). + +.. _fig-tnsa-fields: + +.. figure:: https://user-images.githubusercontent.com/5416860/296338609-a49eee7f-6793-4b55-92f1-0b887e6437ab.png + :alt: Electromagnetic field visualization for E_x (top), B_y (middle), and E_z (bottom). + :width: 80% + + Electromagnetic field visualization for :math:`E_x` (top), :math:`B_y` (middle), and :math:`E_z` (bottom). + +Electromagnetic field output shows where the laser field is strongest at a given point in time, and where accelerating fields build up :numref:`fig-tnsa-fields`. + +.. dropdown:: Script ``plot_2d.py`` + + .. literalinclude:: plot_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/laser_ion/plot_2d.py``. diff --git a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py index 175d8eed568..a262a2373e5 100644 --- a/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py +++ b/Examples/Physics_applications/laser_ion/analysis_histogram_2D.py @@ -6,6 +6,7 @@ import matplotlib.colors as colors import matplotlib.pyplot as plt +import numpy as np from openpmd_viewer import OpenPMDTimeSeries parser = argparse.ArgumentParser(description='Process a 2D histogram name and an integer.') @@ -18,30 +19,43 @@ ts = OpenPMDTimeSeries(path) it = ts.iterations -data, info = ts.get_field(field = "data", iteration = 0, plot = True) +data, info = ts.get_field(field="data", iteration=0, plot=True) print('The available iterations of the simulation are:', it) -print('The axis of the histogram are (0: ordinate ; 1: abscissa):', info.axes) +print('The axes of the histogram are (0: ordinate ; 1: abscissa):', info.axes) print('The data shape is:', data.shape) # Add the simulation time to the title once this information # is available in the "info" FieldMetaInformation object. if args.iter == 'All' : - for i in it : + for it_idx, i in enumerate(it): plt.figure() - data, info = ts.get_field(field = "data", iteration = i, plot = False) - plt.imshow(data, aspect="auto", norm=colors.LogNorm(), extent=info.imshow_extent) - plt.title(args.hist2D + " (iteration %d)" % i) - plt.xlabel(info.axes[1] + ' (m)') - plt.ylabel(info.axes[0] + ' (m.s-1)') + data, info = ts.get_field(field="data", iteration=i, plot=False) + abscissa_name = info.axes[1] # This might be 'z' or something else + abscissa_values = getattr(info, abscissa_name, None) + ordinate_name = info.axes[0] # This might be 'z' or something else + ordinate_values = getattr(info, ordinate_name, None) + + plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") + plt.xlabel(info.axes[1]+r' ($\mu$m)') + plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') plt.colorbar() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.pdf') + plt.tight_layout() + plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') else : i = int(args.iter) + it_idx = np.where(i == it)[0][0] plt.figure() - data, info = ts.get_field(field = "data", iteration = i, plot = False) - plt.imshow(data, aspect="auto", norm=colors.LogNorm(), extent=info.imshow_extent) - plt.title(args.hist2D + " (iteration %d)" % i) - plt.xlabel(info.axes[1] + ' (m)') - plt.ylabel(info.axes[0] + ' (m.s-1)') + data, info = ts.get_field(field="data", iteration=i, plot=False) + abscissa_name = info.axes[1] # This might be 'z' or something else + abscissa_values = getattr(info, abscissa_name, None) + ordinate_name = info.axes[0] # This might be 'z' or something else + ordinate_values = getattr(info, ordinate_name, None) + + plt.pcolormesh(abscissa_values/1e-6, ordinate_values, data, norm=colors.LogNorm(), rasterized=True) + plt.title(args.hist2D + f" Time: {ts.t[it_idx]:.2e} s (Iteration: {i:d})") + plt.xlabel(info.axes[1]+r' ($\mu$m)') + plt.ylabel(info.axes[0]+r' ($m_\mathrm{species} c$)') plt.colorbar() - plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.pdf') + plt.tight_layout() + plt.savefig('Histogram_2D_' + args.hist2D + '_iteration_' + str(i) + '.png') diff --git a/Examples/Physics_applications/laser_ion/inputs b/Examples/Physics_applications/laser_ion/inputs_2d similarity index 92% rename from Examples/Physics_applications/laser_ion/inputs rename to Examples/Physics_applications/laser_ion/inputs_2d index 8fe5acf5f46..5ad8334e9ef 100644 --- a/Examples/Physics_applications/laser_ion/inputs +++ b/Examples/Physics_applications/laser_ion/inputs_2d @@ -2,6 +2,9 @@ # Domain, Resolution & Numerics # +# We only run 100 steps for tests +# Disable `max_step` below to run until the physical `stop_time`. +max_step = 100 # time-scale with highly kinetic dynamics stop_time = 0.2e-12 # [s] # time-scale for converged ion energy @@ -9,12 +12,12 @@ stop_time = 0.2e-12 # [s] # - ions will start to leave the box #stop_time = 1.0e-12 # [s] -# quick tests at ultra-low res. (CI) -#amr.n_cell = 384 512 +# quick tests at ultra-low res. (for CI, and local computer) +amr.n_cell = 384 512 # proper resolution for 10 n_c excl. acc. length # (>=1x V100) -amr.n_cell = 2688 3712 +#amr.n_cell = 2688 3712 # proper resolution for 30 n_c (dx<=3.33nm) incl. acc. length # (>=6x V100) @@ -94,6 +97,8 @@ particles.species_names = electrons hydrogen hydrogen.species_type = hydrogen hydrogen.injection_style = NUniformPerCell hydrogen.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#hydrogen.num_particles_per_cell_each_dim = 4 8 hydrogen.momentum_distribution_type = at_rest # minimum and maximum z position between which particles are initialized # --> should be set for dense targets limit memory consumption during initialization @@ -107,6 +112,8 @@ hydrogen.attribute.orig_z(x,y,z,ux,uy,uz,t) = "z" electrons.species_type = electron electrons.injection_style = NUniformPerCell electrons.num_particles_per_cell_each_dim = 2 2 +# for more realistic simulations, try to avoid that macro-particles represent more than 1 n_c +#electrons.num_particles_per_cell_each_dim = 4 8 electrons.momentum_distribution_type = "gaussian" electrons.ux_th = .01 electrons.uz_th = .01 @@ -200,22 +207,19 @@ diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen # reduce resolution of output fields diag1.coarsening_ratio = 4 4 -diag1.electrons.variables = w ux uy uz -diag1.hydrogen.variables = w ux uz orig_x orig_z # demonstration of a spatial and momentum filter diag1.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) diag1.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) +diag1.format = openpmd +diag1.openpmd_backend = h5 openPMDfw.intervals = 100 openPMDfw.diag_type = Full openPMDfw.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho rho_electrons rho_hydrogen # reduce resolution of output fields openPMDfw.coarsening_ratio = 4 4 -openPMDfw.electrons.variables = w ux uy uz -openPMDfw.hydrogen.variables = w ux uy uz orig_x orig_z openPMDfw.format = openpmd openPMDfw.openpmd_backend = h5 -openPMDfw.species = electrons hydrogen # demonstration of a spatial and momentum filter openPMDfw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) openPMDfw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz>=0) * (x<1.0e-6) * (x>-1.0e-6) @@ -225,11 +229,8 @@ openPMDbw.diag_type = Full openPMDbw.fields_to_plot = rho_hydrogen # reduce resolution of output fields openPMDbw.coarsening_ratio = 4 4 -openPMDbw.electrons.variables = w ux uy uz -openPMDbw.hydrogen.variables = w ux uy uz orig_x orig_z openPMDbw.format = openpmd openPMDbw.openpmd_backend = h5 -openPMDbw.species = electrons hydrogen # demonstration of a momentum filter openPMDbw.electrons.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) openPMDbw.hydrogen.plot_filter_function(t,x,y,z,ux,uy,uz) = (uz<0) @@ -246,21 +247,19 @@ warpx.reduced_diags_names = histuH histue histuzAll FieldProbe histuH.type = ParticleHistogram histuH.intervals = 100 -histuH.path = "./" histuH.species = hydrogen histuH.bin_number = 1000 histuH.bin_min = 0.0 -histuH.bin_max = 35.0 +histuH.bin_max = 0.474 # 100 MeV protons histuH.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" histuH.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" histue.type = ParticleHistogram histue.intervals = 100 -histue.path = "./" histue.species = electrons histue.bin_number = 1000 histue.bin_min = 0.0 -histue.bin_max = 0.1 +histue.bin_max = 197 # 100 MeV electrons histue.histogram_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, sqrt(u2), 0.0)" histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(acos(uz / sqrt(u2))) <= 0.017453, 0)" @@ -268,11 +267,10 @@ histue.filter_function(t,x,y,z,ux,uy,uz) = "u2=ux*ux+uy*uy+uz*uz; if(u2>0, abs(a # this one just records uz of all hydrogen ions, independent of their pointing histuzAll.type = ParticleHistogram histuzAll.intervals = 100 -histuzAll.path = "./" histuzAll.species = hydrogen histuzAll.bin_number = 1000 -histuzAll.bin_min = -35.0 -histuzAll.bin_max = 35.0 +histuzAll.bin_min = -0.474 +histuzAll.bin_max = 0.474 histuzAll.histogram_function(t,x,y,z,ux,uy,uz) = "uz" FieldProbe_Z.type = FieldProbe @@ -313,8 +311,8 @@ PhaseSpaceIons.bin_number_abs = 1000 PhaseSpaceIons.bin_number_ord = 1000 PhaseSpaceIons.bin_min_abs = -5.e-6 PhaseSpaceIons.bin_max_abs = 25.e-6 -PhaseSpaceIons.bin_min_ord = -20.e-6 -PhaseSpaceIons.bin_max_ord = 20.e-6 +PhaseSpaceIons.bin_min_ord = -0.474 +PhaseSpaceIons.bin_max_ord = 0.474 PhaseSpaceIons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" PhaseSpaceIons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" PhaseSpaceIons.value_function(t,x,y,z,ux,uy,uz,w) = "w" @@ -327,8 +325,8 @@ PhaseSpaceElectrons.bin_number_abs = 1000 PhaseSpaceElectrons.bin_number_ord = 1000 PhaseSpaceElectrons.bin_min_abs = -5.e-6 PhaseSpaceElectrons.bin_max_abs = 25.e-6 -PhaseSpaceElectrons.bin_min_ord = -20 -PhaseSpaceElectrons.bin_max_ord = 20 +PhaseSpaceElectrons.bin_min_ord = -197 +PhaseSpaceElectrons.bin_max_ord = 197 PhaseSpaceElectrons.histogram_function_abs(t,x,y,z,ux,uy,uz,w) = "z" PhaseSpaceElectrons.histogram_function_ord(t,x,y,z,ux,uy,uz,w) = "uz" PhaseSpaceElectrons.value_function(t,x,y,z,ux,uy,uz,w) = "w" diff --git a/Examples/Physics_applications/laser_ion/plot_2d.py b/Examples/Physics_applications/laser_ion/plot_2d.py new file mode 100644 index 00000000000..e85782a7d23 --- /dev/null +++ b/Examples/Physics_applications/laser_ion/plot_2d.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Marco Garten +# License: BSD-3-Clause-LBNL +# +# This script plots the densities and fields of a 2D laser-ion acceleration simulation. + + +import argparse +import os +import re + +from matplotlib.colors import TwoSlopeNorm +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import pandas as pd +import scipy.constants as sc + +plt.rcParams.update({'font.size':16}) + +def create_analysis_dir(directory): + if not os.path.exists(directory): + os.makedirs(directory) + + +def visualize_density_iteration(ts, iteration, out_dir): + """ + Visualize densities and fields of a single iteration. + + :param ts: OpenPMDTimeSeries + :param iteration: Output iteration (simulation timestep) + :param out_dir: Directory for PNG output + :return: + """ + # Physics parameters + lambda_L = 800e-9 # Laser wavelength in meters + omega_L = 2 * np.pi * sc.c / lambda_L # Laser frequency in seconds + n_c = sc.m_e * sc.epsilon_0 * omega_L**2 / sc.elementary_charge**2 # Critical plasma density in meters^(-3) + micron = 1e-6 + + # Simulation parameters + n_e0 = 30 + n_max = 2 * n_e0 + nr = 1 # Number to decrease resolution + + # Data fetching + it = iteration + ii = np.where(ts.iterations == it)[0][0] + + time = ts.t[ii] + rho_e, rho_e_info = ts.get_field(field="rho_electrons", iteration=it) + rho_d, rho_d_info = ts.get_field(field="rho_hydrogen", iteration=it) + + # Rescale to critical density + rho_e = rho_e / (sc.elementary_charge * n_c) + rho_d = rho_d / (sc.elementary_charge * n_c) + + # Axes setup + fig, axs = plt.subplots(3, 1, figsize=(5, 8)) + xax, zax = rho_e_info.x, rho_e_info.z + + # Plotting + # Electron density + im0 = axs[0].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, -rho_e.T[::nr, ::nr], + vmin=0, vmax=n_max, cmap="Reds", rasterized=True) + plt.colorbar(im0, ax=axs[0], label=r"$n_\mathrm{\,e}\ (n_\mathrm{c})$") + + # Hydrogen density + im1 = axs[1].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, rho_d.T[::nr, ::nr], + vmin=0, vmax=n_max, cmap="Blues", rasterized=True) + plt.colorbar(im1, ax=axs[1], label=r"$n_\mathrm{\,H}\ (n_\mathrm{c})$") + + # Masked electron density + divnorm = TwoSlopeNorm(vmin=-7., vcenter=0., vmax=2) + masked_data = np.ma.masked_where(rho_e.T == 0, rho_e.T) + my_cmap = plt.cm.PiYG_r.copy() + my_cmap.set_bad(color='black') + im2 = axs[2].pcolormesh(zax[::nr]/micron, xax[::nr]/micron, np.log(-masked_data[::nr, ::nr]), + norm=divnorm, cmap=my_cmap, rasterized=True) + plt.colorbar(im2, ax=axs[2], ticks=[-6, -3, 0, 1, 2], extend='both', + label=r"$\log n_\mathrm{\,e}\ (n_\mathrm{c})$") + + # Axis labels and title + for ax in axs: + ax.set_aspect(1.0) + ax.set_ylabel(r"$x$ ($\mu$m)") + for ax in axs[:-1]: + ax.set_xticklabels([]) + axs[2].set_xlabel(r"$z$ ($\mu$m)") + fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + + plt.tight_layout() + + plt.savefig(f"{out_dir}/densities_{it:06d}.png") + +def visualize_field_iteration(ts, iteration, out_dir): + + # Additional parameters + nr = 1 # Number to decrease resolution + micron = 1e-6 + + # Data fetching + it = iteration + ii = np.where(ts.iterations == it)[0][0] + time = ts.t[ii] + + Ex, Ex_info = ts.get_field(field="E", coord="x", iteration=it) + Exmax = np.max(np.abs([np.min(Ex),np.max(Ex)])) + By, By_info = ts.get_field(field="B", coord="y", iteration=it) + Bymax = np.max(np.abs([np.min(By),np.max(By)])) + Ez, Ez_info = ts.get_field(field="E", coord="z", iteration=it) + Ezmax = np.max(np.abs([np.min(Ez),np.max(Ez)])) + + # Axes setup + fig,axs = plt.subplots(3, 1, figsize=(5, 8)) + xax, zax = Ex_info.x, Ex_info.z + + # Plotting + im0 = axs[0].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,Ex.T[::nr,::nr], + vmin=-Exmax, vmax=Exmax, + cmap="RdBu", rasterized=True) + + plt.colorbar(im0,ax=axs[00], label=r"$E_x$ (V/m)") + + im1 = axs[1].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,By.T[::nr,::nr], + vmin=-Bymax, vmax=Bymax, + cmap="RdBu", rasterized=True) + plt.colorbar(im1,ax=axs[1], label=r"$B_y$ (T)") + + im2 = axs[2].pcolormesh( + zax[::nr]/micron,xax[::nr]/micron,Ez.T[::nr,::nr], + vmin=-Ezmax, vmax=Ezmax, + cmap="RdBu", rasterized=True) + plt.colorbar(im2,ax=axs[2],label=r"$E_z$ (V/m)") + + # Axis labels and title + for ax in axs: + ax.set_aspect(1.0) + ax.set_ylabel(r"$x$ ($\mu$m)") + for ax in axs[:-1]: + ax.set_xticklabels([]) + axs[2].set_xlabel(r"$z$ ($\mu$m)") + fig.suptitle(f"Iteration: {it}, Time: {time/1e-15:.1f} fs") + + plt.tight_layout() + + plt.savefig(f"{out_dir}/fields_{it:06d}.png") + +def visualize_particle_histogram_iteration(diag_name="histuH", species="hydrogen", iteration=1000, out_dir="./analysis"): + + it = iteration + + if species == "hydrogen": + # proton rest energy in eV + mc2 = sc.m_p/sc.electron_volt * sc.c**2 + elif species == "electron": + mc2 = sc.m_e/sc.electron_volt * sc.c**2 + else: + raise NotImplementedError("The only implemented presets for this analysis script are `electron` or `hydrogen`.") + + fs = 1.e-15 + MeV = 1.e6 + + df = pd.read_csv(f"./diags/reducedfiles/{diag_name}.txt",delimiter=r'\s+') + # the columns look like this: + # #[0]step() [1]time(s) [2]bin1=0.000220() [3]bin2=0.000660() [4]bin3=0.001100() + + # matches words, strings surrounded by " ' ", dots, minus signs and e for scientific notation in numbers + nested_list = [re.findall(r"[\w'\.]+",col) for col in df.columns] + + index = pd.MultiIndex.from_tuples(nested_list, names=('column#', 'name', 'bin value')) + + df.columns = (index) + + steps = df.values[:, 0].astype(int) + ii = np.where(steps == it)[0][0] + time = df.values[:, 1] + data = df.values[:, 2:] + edge_vals = np.array([float(row[2]) for row in df.columns[2:]]) + edges_MeV = (np.sqrt(edge_vals**2 + 1)-1) * mc2 / MeV + + time_fs = time / fs + + fig,ax = plt.subplots(1,1) + + ax.plot(edges_MeV, data[ii, :]) + ax.set_yscale("log") + ax.set_ylabel(r"d$N$/d$\mathcal{E}$ (arb. u.)") + ax.set_xlabel(r"$\mathcal{E}$ (MeV)") + + fig.suptitle(f"{species} - Iteration: {it}, Time: {time_fs[ii]:.1f} fs") + + plt.tight_layout() + plt.savefig(f"./{out_dir}/{diag_name}_{it:06d}.png") + + +if __name__ == "__main__": + + # Argument parsing + parser = argparse.ArgumentParser(description='Visualize Laser-Ion Accelerator Densities and Fields') + parser.add_argument('-d', '--diag_dir', type=str, default='./diags/diag1', help='Directory containing density and field diagnostics') + parser.add_argument('-i', '--iteration', type=int, default=None, help='Specific iteration to visualize') + parser.add_argument('-hn', '--histogram_name', type=str, default='histuH', help='Name of histogram diagnostic to visualize') + parser.add_argument('-hs', '--histogram_species', type=str, default='hydrogen', help='Particle species in the visualized histogram diagnostic') + args = parser.parse_args() + + # Create analysis directory + analysis_dir = 'analysis' + create_analysis_dir(analysis_dir) + + # Loading the time series + ts = OpenPMDTimeSeries(args.diag_dir) + + if args.iteration is not None: + visualize_density_iteration(ts, args.iteration, analysis_dir) + visualize_field_iteration(ts, args.iteration, analysis_dir) + visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, args.iteration, analysis_dir) + else: + for it in ts.iterations: + visualize_density_iteration(ts, it, analysis_dir) + visualize_field_iteration(ts, it, analysis_dir) + visualize_particle_histogram_iteration(args.histogram_name, args.histogram_species, it, analysis_dir) diff --git a/Examples/Physics_applications/plasma_acceleration/README.rst b/Examples/Physics_applications/plasma_acceleration/README.rst new file mode 100644 index 00000000000..d5775e93aa8 --- /dev/null +++ b/Examples/Physics_applications/plasma_acceleration/README.rst @@ -0,0 +1,63 @@ +.. _examples-pwfa: + +Beam-Driven Wakefield Acceleration of Electrons +=============================================== + +This example shows how to model a beam-driven plasma-wakefield accelerator (PWFA) :cite:p:`ex-TajimaDawson1982,ex-Esarey1996`. + +PWFA is best performed in 3D or quasi-cylindrical (RZ) geometry, in order to correctly capture some of the key physics (structure of the space-charge fields, beamloading, shape of the accelerating bubble in the blowout regime, etc.). +For physical situations that have close-to-cylindrical symmetry, simulations in RZ geometry capture the relevant physics at a fraction of the computational cost of a 3D simulation. +On the other hand, for physical situation with strong asymmetries (e.g., non-round driver, strong hosing of the accelerated beam, etc.), only 3D simulations are suitable. + +Additionally, to speed up computation, this example uses the :ref:`boosted frame method ` to effectively model long acceleration lengths. + +Alternatively, an other common approximation for PWFAs is quasi-static modeling, e.g., if effects such as self-injection can be ignored. +In the Beam, Plasma & Accelerator Simulation Toolkit (BLAST), `HiPACE++ `__ provides such methods. + +.. note:: + + TODO: The Python (PICMI) input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_plasma_acceleration.py`` or +* WarpX **executable** using an input file: ``warpx.3d inputs_3d_boost`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should use the boosted frame method, like the ``inputs_3d_boost`` file. + + .. literalinclude:: PICMI_inputs_plasma_acceleration.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/PICMI_inputs_plasma_acceleration.py``. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_3d_boost + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_acceleration/inputs_3d_boost``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/plasma_mirror/README.rst b/Examples/Physics_applications/plasma_mirror/README.rst new file mode 100644 index 00000000000..8741db09699 --- /dev/null +++ b/Examples/Physics_applications/plasma_mirror/README.rst @@ -0,0 +1,54 @@ +.. _examples-plasma-mirror: + +Plasma-Mirror +============= + +This example shows how to model a plasma mirror, using a planar target of solid density :cite:p:`ex-Dromey2004,ex-Roedel2010`. + +Although laser-solid interaction modeling requires full 3D modeling for adequate description of the dynamics at play, this example models a 2D example. +2D modeling provide a qualitative overview of the dynamics, but mostly saves computational costs since the plasma frequency (and Debye length) of the surface plasma determines the resolution need in laser-solid interaction modeling. + +.. note:: + + TODO: The Python (PICMI) input file needs to be created. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: (*TODO*) or +* WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Physics_applications/plasma_mirror/inputs_2d``. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py new file mode 100644 index 00000000000..08ebe23f828 --- /dev/null +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# +# --- Input file for spacecraft charging testing in RZ. +# --- This input defines a conducting sphere (spacecraft) immersed in a thermal +# --- plasma with the same given initial conditions as in the article: +# --- (*) J. Deca, G. Lapenta, R. Marchand, S. Markidis; +# --- Spacecraft charging analysis with the implicit particle-in-cell code iPic3D. +# --- Part III. A. pages 3-4 +# --- Phys. Plasmas 1 October 2013; 20 (10): 102902. https://doi.org/10.1063/1.4826951. +# --- The conducting sphere starts with an initial potential of 1V and will interact with +# --- the surrounding plasma, initially static. The charging of the spacecraft - by accumulation +# --- of electrons - leads to a decrease of the potential on the surface over the time +# --- until reaching an equilibrium floating potential of ~144.5 V (*). + +from mpi4py import MPI as mpi +import numpy as np +import scipy.constants as scc + +from pywarpx import picmi +from pywarpx.callbacks import installafterEsolve, installafterInitEsolve +from pywarpx.fields import ExWrapper, EzWrapper, PhiFPWrapper, RhoFPWrapper +from pywarpx.particle_containers import ParticleBoundaryBufferWrapper + + +# Utilities +class SpaceChargeFieldCorrector(object): + """ + Class used by the callback functions to calculate the + correct charge on the spacecraft at each initialisation. + """ + def __init__(self): + self.saved_first_iteration_fields = False + self.spacecraft_potential = 1. # Initial voltage: 1V + self.spacecraft_capacitance = None + + def correct_space_charge_fields(self, q=None): + """ + Function that will be called at each iteration, + after each electrostatic solve in WarpX + """ + assert self.saved_first_iteration_fields + + # Compute the charge that WarpX thinks there is on the spacecraft + # from phi and rho after the Poisson solver + q_v = compute_virtual_charge_on_spacecraft() + if q is None: + q = compute_actual_charge_on_spacecraft() + + # Correct fields so as to recover the actual charge + Er = ExWrapper(include_ghosts=True)[:,:] + Er[...] = Er[...]+(q - q_v)*self.normalized_Er[...] + Ez = EzWrapper(include_ghosts=True)[:,:] + Ez[...] += (q - q_v)*self.normalized_Ez[...] + phi = PhiFPWrapper(include_ghosts=True)[:,:] + phi[...] += (q - q_v)*self.normalized_phi[...] + self.spacecraft_potential += (q - q_v)*self.spacecraft_capacitance + sim.extension.warpx.set_potential_on_eb( "%f" %self.spacecraft_potential ) + print('Setting potential to %f' %self.spacecraft_potential) + + # Confirm that the charge on the spacecraft is now correct + compute_virtual_charge_on_spacecraft() + + def save_normalized_vacuum_Efields(self,): + # Compute the charge that WarpX thinks there is on the spacecraft + # from phi and rho after the Poisson solver + q_v = compute_virtual_charge_on_spacecraft() + self.spacecraft_capacitance = 1./q_v # the potential was set to 1V + + # Check that this iteration corresponded to a vacuum solve + rho = RhoFPWrapper(include_ghosts=False) + + # In principle, we should check that `rho` is exactly 0 + # However, due to machine precision errors when adding the charge + # of ions and electrons, this can be slightly different than 0 + assert np.all( abs(rho[...]) < 1.e-11 ) + + # Record fields + Er = ExWrapper(include_ghosts=True)[:,:] + self.normalized_Er = Er[...] /q_v + Ez = EzWrapper(include_ghosts=True)[:,:] + self.normalized_Ez = Ez[...] /q_v + phi = PhiFPWrapper(include_ghosts=True)[:,:] + self.normalized_phi = phi[...] /q_v + + self.saved_first_iteration_fields = True + self.correct_space_charge_fields(q=0) + + +def compute_virtual_charge_on_spacecraft(): + """ + Given that we asked WarpX to solve the Poisson + equation with phi=1 on the spacecraft and phi=0 + on the boundary of the domain, compute the charge + that WarpX thinks there should be on the spacecraft. + """ + # Get global array for the whole domain (across MPI ranks) + phi = PhiFPWrapper(include_ghosts=False)[:,:] + rho = RhoFPWrapper(include_ghosts=False)[:,:] + + # Check that this codes correspond to the global size of the box + assert phi.shape == (nr+1, nz+1) + assert rho.shape == (nr+1, nz+1) + + dr, dz = sim.extension.warpx.Geom(lev=0).data().CellSize() + + # Compute integral of grad phi over surfaces of the domain + r = np.linspace(rmin, rmax, len(phi), endpoint=False) + (rmax - rmin) / (2 * len(phi)) #shift of the r points because the derivaties are calculated in the middle + face_z0 = 2 * np.pi * 1./dz * ( (phi[:,0]-phi[:,1]) * r ).sum() * dr #here I am assuming that phi is a numpy array that can handle elementwise mult + face_zend = 2 * np.pi * 1./dz * ( (phi[:,-1]-phi[:,-2]) * r ).sum() * dr + face_rend = 2 * np.pi * 1./dr*((phi[-1,:]-phi[-2,:]) * rmax).sum() * dz + grad_phi_integral = face_z0 + face_zend + face_rend + + # Compute integral of rho over volume of the domain + # (i.e. total charge of the plasma particles) + rho_integral = 0.0 + for k in range(1, nz-1): + for i in range(1, nr-1): + rho_integral += rho[i,k] * r[i] * dr * dz + + # Due to an oddity in WarpX (which will probably be solved later) + # we need to multiply `rho` by `-epsilon_0` to get the correct charge + rho_integral *= 2 * np.pi * -scc.epsilon_0 #does this oddity still exist? + + # Compute charge of the spacecraft, based on Gauss theorem + q_spacecraft = - rho_integral - scc.epsilon_0 * grad_phi_integral + print('Virtual charge on the spacecraft: %e' %q_spacecraft) + return q_spacecraft + + +def compute_actual_charge_on_spacecraft(): + """ + Compute the actual charge on the spacecraft, + by counting how many electrons and protons + were collected by the WarpX embedded boundary (EB) + """ + charge = {'electrons': -scc.e, 'protons': scc.e} + q_spacecraft = 0 + particle_buffer = ParticleBoundaryBufferWrapper() + for species in charge.keys(): + weights = particle_buffer.get_particle_boundary_buffer(species, 'eb', 'w', 0) + sum_weights_over_tiles = sum([w.sum() for w in weights]) + + # Reduce across all MPI ranks + ntot = float(mpi.COMM_WORLD.allreduce(sum_weights_over_tiles, op=mpi.SUM)) + print('Total number of %s collected on spacecraft: %e'%(species, ntot)) + q_spacecraft += ntot * charge[species] + + print('Actual charge on the spacecraft: %e' %q_spacecraft) + return q_spacecraft + + +########################## +# numerics parameters +########################## + +dt=1.27e-8 + +# --- Nb time steps +max_steps = 1000 +diagnostic_interval = 10 + +# --- grid +nr = 40 +nz= 80 + +rmin = 0.0 +rmax = 3 +zmin = -3 +zmax = 3 + +number_per_cell =5 +number_per_cell_each_dim = [10,1, 1] + + +########################## +# physics components +########################## + +n = 7.0e9 #plasma density #particles/m^3 +Te = 85 #Electron temp in eV +Ti = 0.05 * Te #Ion temp in eV +qe = picmi.constants.q_e #elementary charge +m_e = picmi.constants.m_e #electron mass +m_i = 1836.0 * m_e #mass of ion +v_eth = (qe * Te / m_e) ** 0.5 +v_pth = (qe * Ti / m_i) ** 0.5 + +# nothing to change in the distribution function? +e_dist = picmi.UniformDistribution(density = n, rms_velocity=[v_eth, v_eth, v_eth] ) +e_dist2 = picmi.UniformFluxDistribution( + flux=n*v_eth/(2*np.pi)**.5, # Flux for Gaussian with vmean=0 + surface_flux_position=3, + flux_direction=-1, flux_normal_axis='r', + gaussian_flux_momentum_distribution=True, + rms_velocity=[v_eth, v_eth, v_eth] ) +electrons = picmi.Species(particle_type='electron', + name='electrons', + initial_distribution=[e_dist,e_dist2], + warpx_save_particles_at_eb=1) + +p_dist = picmi.UniformDistribution(density = n, rms_velocity=[v_pth, v_pth, v_pth] ) +p_dist2 = picmi.UniformFluxDistribution( + flux=n*v_pth/(2*np.pi)**.5, # Flux for Gaussian with vmean=0 + surface_flux_position=3, + flux_direction=-1, flux_normal_axis='r', + gaussian_flux_momentum_distribution=True, + rms_velocity=[v_pth, v_pth, v_pth] ) +protons = picmi.Species(particle_type='proton', + name='protons', + initial_distribution=[p_dist,p_dist2], + warpx_save_particles_at_eb=1) + + +########################## +# numerics components +########################## + +grid = picmi.CylindricalGrid( + number_of_cells = [nr, nz], + n_azimuthal_modes = 1, + lower_bound = [rmin, zmin], + upper_bound = [rmax, zmax], + lower_boundary_conditions = ['none', 'dirichlet'], + upper_boundary_conditions = ['dirichlet', 'dirichlet'], + lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + upper_boundary_conditions_particles = ['absorbing', 'reflecting'] +) + +solver = picmi.ElectrostaticSolver( + grid=grid, method='Multigrid', + warpx_absolute_tolerance=1e-7 +) + +embedded_boundary = picmi.EmbeddedBoundary( + implicit_function="-(x**2+y**2+z**2-radius**2)", + potential=1., # arbitrary value ; this will be corrected by a callback function + radius = 0.3277 +) + + +########################## +# diagnostics +########################## + +field_diag = picmi.FieldDiagnostic( + name = 'diag1', + grid = grid, + period = diagnostic_interval, + data_list = ['Er', 'Ez', 'phi', 'rho', + 'rho_electrons', 'rho_protons'], + warpx_format = 'openpmd', + write_dir = '.', + warpx_file_prefix = 'spacecraft_charging_plt' +) + +part_diag = picmi.ParticleDiagnostic(name = 'diag1', + period = diagnostic_interval, + species = [electrons, protons], + warpx_format = 'openpmd', + write_dir = '.', + warpx_file_prefix = 'spacecraft_charging_plt' +) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation( + solver = solver, + time_step_size = dt, + max_steps = max_steps, + warpx_embedded_boundary=embedded_boundary, + warpx_amrex_the_arena_is_managed=1, + warpx_random_seed=1 +) + +layout1=picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, + grid=grid) +layout2=picmi.PseudoRandomLayout(n_macroparticles_per_cell=number_per_cell, + grid=grid) +sim.add_species(electrons, + layout = [layout1,layout2]) + +sim.add_species(protons, + layout = [layout1,layout2]) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(part_diag) + +########################## +# simulation run +########################## + +spc = SpaceChargeFieldCorrector() + +installafterInitEsolve( spc.save_normalized_vacuum_Efields ) +installafterEsolve( spc.correct_space_charge_fields ) + +sim.step(max_steps) diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py new file mode 100755 index 00000000000..d8e8ac86af8 --- /dev/null +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python + +""" +This script tests the potential-time profile on the surface of +a conducting sphere (spacecraft) immersed in an initially static +thermal plasma. The potential on the spacecraft decreases over +the time to reach an equilibrium floating potential. + +An input Python file PICMI_inputs_rz.py is used. + +The test will check the curve fitting parameters v0 and tau defined +by the following exponential function: phi(t)=v0(1-exp(-t/tau)) +""" + +import os +import sys + +import matplotlib.pyplot as plt +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.optimize import curve_fit +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') + +ts = OpenPMDTimeSeries('./spacecraft_charging_plt') +dt = 1.27e-8 +t=[] +phi=[] +it = ts.iterations + +for i in it: + phi_i = ts.get_field('phi',iteration=i,plot=False) + # Find the minimum value among all grids for this iteration + phi_min = np.min(phi_i[0]) + phi.append(phi_min) + t.append(dt*i) + + +def func(x, v0, tau): + + return v0 * (1-np.exp(-np.array(x) / tau)) + + + +popt, pcov = curve_fit(func, t, phi) + +plt.plot(t,phi, label='modelisation') +plt.plot(t, func(t, *popt), 'r-',label='fit: v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) +plt.legend() +plt.savefig('min_phi_analysis.png') + +print('fit parameters between the min(phi) curve over the time and the function v0(1-exp(-t/tau)):') +print('v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) + + +tolerance_v0=0.01 +tolerance_tau=0.01 +print("tolerance for v0 = "+ str(tolerance_v0 *100) + '%') +print("tolerance for tau = "+ str(tolerance_tau*100) + '%') + +mean_v0=-151.347 +mean_tau=0.000004351 + +diff_v0=np.abs((popt[0]-mean_v0)/mean_v0) +diff_tau=np.abs((popt[1]-mean_tau)/mean_tau) + +print("percentage error for v0 = "+ str(diff_v0 *100) + '%') +print("percentage error for tau = "+ str(diff_tau*100) + '%') + +assert (diff_v0 < tolerance_v0) and (diff_tau < tolerance_tau), 'Test spacecraft_charging did not pass' diff --git a/Examples/Physics_applications/uniform_plasma/README.rst b/Examples/Physics_applications/uniform_plasma/README.rst new file mode 100644 index 00000000000..50d132712c6 --- /dev/null +++ b/Examples/Physics_applications/uniform_plasma/README.rst @@ -0,0 +1,65 @@ +.. _examples-uniform-plasma: + +Uniform Plasma +============== + +This example evolves a uniformly distributed, hot plasma over time. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_3d`` file. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created following the ``inputs_2d`` file. + + .. tab-item:: Executable: Input File + + This example can be run **either** as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``usage/examples/lwfa/inputs_2d``. + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py new file mode 100755 index 00000000000..0f00010a505 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright 2023 David Grote +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from +# the script `inputs_1d`. This simulates a 1D periodic plasma using the implicit solver. +import os +import re +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +if re.match('SemiImplicitPicard_1d', fn): + tolerance_rel = 2.5e-5 +elif re.match('ImplicitPicard_1d', fn): + # This case should have near machine precision conservation of energy + tolerance_rel = 1.e-14 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel}") + +assert( max_delta_E < tolerance_rel ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_2d.py new file mode 100755 index 00000000000..fa3299925a8 --- /dev/null +++ b/Examples/Tests/Implicit/analysis_vandb_2d.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_vandb_2d`. +# This simulates a 2D periodic plasma using the implicit solver +# with the Villasenor deposition using shape factor 2. +import os +import sys + +import numpy as np +from scipy.constants import e, epsilon_0 +import yt + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +field_energy = np.loadtxt('diags/reducedfiles/field_energy.txt', skiprows=1) +particle_energy = np.loadtxt('diags/reducedfiles/particle_energy.txt', skiprows=1) + +total_energy = field_energy[:,2] + particle_energy[:,2] + +delta_E = (total_energy - total_energy[0])/total_energy[0] +max_delta_E = np.abs(delta_E).max() + +# This case should have near machine precision conservation of energy +tolerance_rel_energy = 2.e-14 +tolerance_rel_charge = 2.e-15 + +print(f"max change in energy: {max_delta_E}") +print(f"tolerance: {tolerance_rel_energy}") + +assert( max_delta_E < tolerance_rel_energy ) + +# check for machine precision conservation of charge density +n0 = 1.e30 + +pltdir = sys.argv[1] +ds = yt.load(pltdir) +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) + +divE = data['boxlib', 'divE'].value +rho = data['boxlib', 'rho'].value + +# compute local error in Gauss's law +drho = (rho - epsilon_0*divE)/e/n0 + +# compute RMS on in error on the grid +nX = drho.shape[0] +nZ = drho.shape[1] +drho2_avg = (drho**2).sum()/(nX*nZ) +drho_rms = np.sqrt(drho2_avg) + +print(f"rms error in charge conservation: {drho_rms}") +print(f"tolerance: {tolerance_rel_charge}") + +assert( drho_rms < tolerance_rel_charge ) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d new file mode 100644 index 00000000000..50d28a2db75 --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = implicit_picard +algo.max_picard_iterations = 31 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit new file mode 100644 index 00000000000..2271a0bb1bc --- /dev/null +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -0,0 +1,81 @@ +################################# +############ CONSTANTS ############# +################################# + +my_constants.n0 = 1.e30 # plasma densirty, m^-3 +my_constants.nz = 40 # number of grid cells +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.nppcz = 100 # number of particles/cell in z +my_constants.dt = 0.1/wpe # time step size, s + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_step = 100 +amr.n_cell = nz +amr.max_level = 0 + +geometry.dims = 1 +geometry.prob_lo = 0.0 +geometry.prob_hi = 10.*de0 +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic + +################################# +############ NUMERICS ########### +################################# + +warpx.const_dt = dt +algo.evolve_scheme = semi_implicit_picard +algo.max_picard_iterations = 5 +algo.picard_iteration_tolerance = 0. +algo.current_deposition = esirkepov +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ PLASMA ############# +################################# + +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 100 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_2d new file mode 100644 index 00000000000..2dc57323efe --- /dev/null +++ b/Examples/Tests/Implicit/inputs_vandb_2d @@ -0,0 +1,92 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.n0 = 1.e30 # m^-3 +my_constants.Ti = 100. # eV +my_constants.Te = 100. # eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) +my_constants.de0 = clight/wpe +my_constants.nppcz = 10 # number of particles/cell in z +my_constants.dt = 0.1/wpe # s + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 20 +amr.n_cell = 40 40 +amr.max_grid_size = 8 +amr.blocking_factor = 8 +amr.max_level = 0 +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 # physical domain +geometry.prob_hi = 10.0*de0 10.0*de0 + +################################# +####### Boundary condition ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = dt +#warpx.cfl = 0.5656 +warpx.use_filter = 0 + +algo.maxwell_solver = Yee +algo.evolve_scheme = "implicit_picard" +algo.require_picard_convergence = 0 +algo.max_picard_iterations = 25 +algo.picard_iteration_tolerance = 0.0 #1.0e-12 +algo.particle_pusher = "boris" +#algo.particle_pusher = "higuera" + +algo.particle_shape = 2 +#algo.current_deposition = "direct" +#algo.current_deposition = "esirkepov" +algo.current_deposition = "villasenor" + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = nppcz nppcz +electrons.profile = constant +electrons.density = 1.e30 # number per m^3 +electrons.momentum_distribution_type = "gaussian" +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight + +protons.charge = q_e +protons.mass = m_p +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = nppcz nppcz +protons.profile = constant +protons.density = 1.e30 # number per m^3 +protons.momentum_distribution_type = "gaussian" +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 20 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE +diag1.electrons.variables = w ux uy uz +diag1.protons.variables = w ux uy uz + +warpx.reduced_diags_names = particle_energy field_energy +particle_energy.type = ParticleEnergy +particle_energy.intervals = 1 +field_energy.type = FieldEnergy +field_energy.intervals = 1 diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz index 050118d036f..2e22ca299ea 100644 --- a/Examples/Tests/LoadExternalField/inputs_rz +++ b/Examples/Tests/LoadExternalField/inputs_rz @@ -46,7 +46,7 @@ algo.particle_shape = 1 ################################# ############ PLASMA ############# ################################# -particles.species_names = proton #electron +particles.species_names = proton proton.injection_style = "SingleParticle" proton.single_particle_pos = 0.0 0.2 2.5 proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 @@ -55,14 +55,11 @@ proton.do_not_deposit = 1 proton.mass = m_p proton.charge = q_e -#electron.injection_style = "SingleParticle" -#electron.single_particle_pos = 0.0 0.2 2.5 -#electron.single_particle_u = 0.0 0.0 0.0 -#electron.single_particle_weight = 1.0 -#electron.mass = 1.0 -#electron.charge = -q_e*1.0e-20 - # Diagnostics -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 chk diag1.intervals = 300 diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/boosted_diags/analysis.py b/Examples/Tests/boosted_diags/analysis.py index c6c089f9807..c0b03f4a20b 100755 --- a/Examples/Tests/boosted_diags/analysis.py +++ b/Examples/Tests/boosted_diags/analysis.py @@ -21,6 +21,7 @@ import numpy as np import openpmd_api as io +from openpmd_viewer import OpenPMDTimeSeries import yt yt.funcs.mylog.setLevel(0) @@ -48,9 +49,13 @@ Ez_openpmd = ds_openpmd.meshes['E']['z'].load_chunk() Ez_openpmd = Ez_openpmd.transpose() series.flush() - # Compare arrays to check consistency between new BTD formats (plotfile and openPMD) assert(np.allclose(Ez_plotfile, Ez_openpmd, rtol=rtol, atol=atol)) +# Check that particle random sub-selection has been applied +ts = OpenPMDTimeSeries('./diags/diag2/') +w, = ts.get_particle(['w'], species='beam', iteration=3) +assert (400 < len(w)) & (len(w) < 600) + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/boosted_diags/inputs_3d b/Examples/Tests/boosted_diags/inputs_3d index ba98558be47..1b6b3448f26 100644 --- a/Examples/Tests/boosted_diags/inputs_3d +++ b/Examples/Tests/boosted_diags/inputs_3d @@ -122,3 +122,4 @@ diag2.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho diag2.format = openpmd diag2.buffer_size = 32 diag2.openpmd_backend = h5 +diag2.beam.random_fraction = 0.5 diff --git a/Examples/Tests/energy_conserving_thermal_plasma/analysis.py b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py new file mode 100755 index 00000000000..43e9b6d9822 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/analysis.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# Copyright 2024, Remi Lehe +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the conservation of energy for a thermal plasma with periodic boundary. +# Here, we use energy-converving gather and an electrostatic solver. The energy +# is not expected to be exactly conserved, but it is expected to be better conserved +# than other gathering scheme. This tests checks that the energy does not increase by +# more than 0.3% over the duration of the simulatoin. + +import os +import sys + +import numpy as np + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get energy as a function of time, from reduced diagnostics +EFdata = np.genfromtxt('./diags/reducedfiles/EF.txt') # Field energy +EPdata = np.genfromtxt('./diags/reducedfiles/EP.txt') # Particle energy +field_energy = EFdata[:,2] +particle_energy = EPdata[:,2] +E = field_energy + particle_energy +print(abs(E-E[0])/E[0]) +# Check that the energy is conserved to 0.3% +assert np.all( abs(E-E[0])/E[0] < 0.003 ) + +# Checksum test +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic b/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic new file mode 100644 index 00000000000..dab1b77db50 --- /dev/null +++ b/Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic @@ -0,0 +1,83 @@ +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 500 +amr.n_cell = 8 8 +amr.max_level = 0 +geometry.dims = 2 + +warpx.do_electrostatic = labframe +algo.field_gathering = energy-conserving +algo.particle_shape = 2 +warpx.use_filter = 0 + +################################# +############ CONSTANTS ############# +################################# +my_constants.n0 = 1.e30 # plasma density, m^-3 +my_constants.Ti = 100. # ion temperature, eV +my_constants.Te = 100. # electron temperature, eV +my_constants.wpe = q_e*sqrt(n0/(m_e*epsilon0)) # electron plasma frequency, radians/s +my_constants.de0 = clight/wpe # skin depth, m +my_constants.dt = ( 0.2 )/wpe # time step size, s +warpx.const_dt = dt + +################################# +####### GENERAL PARAMETERS ###### +################################# +geometry.dims = 2 +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 10.*de0 10.*de0 +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 + +################################# +###### BOUNDARY CONDITIONS ###### +################################# +boundary.field_lo = periodic periodic +boundary.field_hi = periodic periodic +boundary.particle_lo = periodic periodic +boundary.particle_hi = periodic periodic + +################################# +############ PLASMA ############# +################################# +particles.species_names = electrons protons + +electrons.species_type = electron +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 2 2 +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = sqrt(Te*q_e/m_e)/clight +electrons.uy_th = sqrt(Te*q_e/m_e)/clight +electrons.uz_th = sqrt(Te*q_e/m_e)/clight +electrons.xmin = 0 +electrons.xmax = 10.*de0 +electrons.zmin = 0 +electrons.zmax =10.*de0 + +protons.species_type = proton +protons.injection_style = "NUniformPerCell" +protons.num_particles_per_cell_each_dim = 2 2 +protons.profile = constant +protons.density = n0 +protons.momentum_distribution_type = gaussian +protons.ux_th = sqrt(Ti*q_e/m_p)/clight +protons.uy_th = sqrt(Ti*q_e/m_p)/clight +protons.uz_th = sqrt(Ti*q_e/m_p)/clight +protons.xmin = 0 +protons.xmax = 10.*de0 +protons.zmin = 0 +protons.zmax =10.*de0 + +diagnostics.diags_names = diag1 +diag1.intervals = 500 +diag1.diag_type = Full + +warpx.reduced_diags_names = EP EF +EP.type = ParticleEnergy +EP.intervals = 100 +EF.type = FieldEnergy +EF.intervals =100 diff --git a/Examples/Tests/field_probe/analysis_field_probe.py b/Examples/Tests/field_probe/analysis_field_probe.py index e038bfd0f48..e167942d77c 100755 --- a/Examples/Tests/field_probe/analysis_field_probe.py +++ b/Examples/Tests/field_probe/analysis_field_probe.py @@ -9,9 +9,9 @@ """ This script tests the accuracy of the FieldProbe diagnostic by observing a plane wave undergoing single slit diffraction. The input file inputs_2d is used. This -file defines the simulation box, laser pulse, embeded boundary with single slit, +file defines the simulation box, laser pulse, embedded boundary with single slit, and line of detector points. The plane wave initializes near the negative Z end -of the simulation box. The wave interacts with the embeded boundary at Z=0. The +of the simulation box. The wave interacts with the embedded boundary at Z=0. The wave undergoes diffraction at the slit. The electromagnetic flux is calculated at the line detector which is placed perpendicular to Z beyond the slit. This test will check if the detected EM flux matches expected values, @@ -39,7 +39,7 @@ def I_envelope (x, lam = 0.2e-6, a = 0.3e-6, D = 1.7e-6): arg = np.pi * a / lam * np.sin(np.arctan(x / D)) return np.sinc( arg / np.pi )**2 -# Count non-outlyer values away from simulation boundaries +# Count non-outlier values away from simulation boundaries counter = np.arange(60, 140, 2) # Count average error from expected values diff --git a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py index 0998de25a7b..470cbfe6065 100755 --- a/Examples/Tests/flux_injection/analysis_flux_injection_3d.py +++ b/Examples/Tests/flux_injection/analysis_flux_injection_3d.py @@ -17,7 +17,7 @@ After the particles are emitted with flux injection, this script produces histograms of the velocity distribution and compares it with the expected -velocity distibution (Gaussian or Gaussian-flux depending on the direction +velocity distribution (Gaussian or Gaussian-flux depending on the direction of space) """ import os diff --git a/Examples/Tests/gaussian_beam/README.rst b/Examples/Tests/gaussian_beam/README.rst new file mode 100644 index 00000000000..bfca2bb2398 --- /dev/null +++ b/Examples/Tests/gaussian_beam/README.rst @@ -0,0 +1,47 @@ +.. _examples-gaussian-beam: + +Gaussian Beam +============= + +This example initializes a Gaussian beam distribution. + + +Run +--- + +This example can be run **either** as: + +* **Python** script: ``python3 PICMI_inputs_gaussian_beam.py`` or +* WarpX **executable** using an input file: (*TODO*) + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Python: Script + + .. literalinclude:: PICMI_inputs_gaussian_beam.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/gaussian_beam/PICMI_inputs_gaussian_beam.py``. + + .. tab-item:: Executable: Input File + + .. note:: + + TODO: This input file should be created following the ``PICMI_inputs_gaussian_beam.py`` file. + + +Analyze +------- + +.. note:: + + This section is TODO. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/gaussian_beam/analysis_focusing_beam.py b/Examples/Tests/gaussian_beam/analysis_focusing_beam.py new file mode 100755 index 00000000000..4a5fa3b927b --- /dev/null +++ b/Examples/Tests/gaussian_beam/analysis_focusing_beam.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Arianna Formenti +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + + +import os +import sys + +import numpy as np +from scipy.constants import c, eV, m_e, micro, nano + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') + +import checksumAPI +from openpmd_viewer import OpenPMDTimeSeries + +GeV=1e9*eV +energy = 125.*GeV +gamma = energy/(m_e*c**2) +sigmax = 516.0*nano +sigmay = 7.7*nano +sigmaz = 300.*micro +nz = 256 +Lz = 20*sigmaz +gridz = np.linspace(-0.5*Lz, 0.5*Lz, nz) +tol = gridz[1] - gridz[0] +emitx = 50*micro +emity = 20*nano +focal_distance = 4*sigmaz + +def s(z, sigma0, emit): + '''The theoretical size of a focusing beam (in the absence of space charge), + at position z, given its emittance and size at focus.''' + return np.sqrt(sigma0**2 + emit**2 * (z - focal_distance)**2 / sigma0**2) + +filename = sys.argv[1] + +ts = OpenPMDTimeSeries('./diags/openpmd/') + +x, y, z, w, = ts.get_particle( ['x', 'y', 'z', 'w'], species='beam1', iteration=0, plot=False) + +imin = np.argmin(np.sqrt((gridz+0.8*focal_distance)**2)) +imax = np.argmin(np.sqrt((gridz-0.8*focal_distance)**2)) + +sx, sy = [], [] +# Compute the size of the beam in each z slice +subgrid = gridz[imin:imax] +for d in subgrid: + i = np.sqrt((z - d)**2) < tol + if (np.sum(i)!=0): + mux = np.average(x[i], weights=w[i]) + muy = np.average(y[i], weights=w[i]) + sx.append(np.sqrt(np.average((x[i]-mux)**2, weights=w[i]))) + sy.append(np.sqrt(np.average((y[i]-muy)**2, weights=w[i]))) + +# Theoretical prediction for the size of the beam in each z slice +sx_theory = s(subgrid, sigmax, emitx/gamma) +sy_theory = s(subgrid, sigmay, emity/gamma) + +assert(np.allclose(sx, sx_theory, rtol=0.051, atol=0)) +assert(np.allclose(sy, sy_theory, rtol=0.038, atol=0)) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/gaussian_beam/inputs_focusing_beam b/Examples/Tests/gaussian_beam/inputs_focusing_beam new file mode 100644 index 00000000000..08e978671fc --- /dev/null +++ b/Examples/Tests/gaussian_beam/inputs_focusing_beam @@ -0,0 +1,120 @@ +################################# +########## MY CONSTANTS ######### +################################# +my_constants.mc2 = m_e*clight*clight +my_constants.nano = 1.0e-9 +my_constants.micro = 1.e-6 +my_constants.GeV = q_e*1.e9 + +# BEAMS +my_constants.energy = 125.*GeV +my_constants.gamma = energy/mc2 +my_constants.npart = 2.e10 +my_constants.nmacropart = 1000000 +my_constants.charge = q_e * npart + +my_constants.sigmax = 516.0*nano +my_constants.sigmay = 7.7*nano +my_constants.sigmaz = 300.*micro + +my_constants.mux = 0.0 +my_constants.muy = 0.0 +my_constants.muz = 0.0 + +my_constants.ux = 0.0 +my_constants.uy = 0.0 +my_constants.uz = gamma + +my_constants.emitx = 50*micro +my_constants.emity = 20*nano +my_constants.emitz = 0. + +my_constants.dux = emitx / sigmax +my_constants.duy = emity / sigmay +my_constants.duz = emitz / sigmaz + +my_constants.focal_distance = 4*sigmaz + +# BOX +my_constants.Lx = 20*sigmax +my_constants.Ly = 20*sigmay +my_constants.Lz = 20*sigmaz +my_constants.nx = 256 +my_constants.ny = 256 +my_constants.nz = 256 + + + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 0 +amr.n_cell = nx ny nz +amr.max_grid_size = 256 +amr.blocking_factor = 2 +amr.max_level = 0 +geometry.dims = 3 +geometry.prob_lo = -0.5*Lx -0.5*Ly -0.5*Lz +geometry.prob_hi = 0.5*Lx 0.5*Ly 0.5*Lz + +################################# +######## BOUNDARY CONDITION ##### +################################# +boundary.field_lo = PEC PEC PEC +boundary.field_hi = PEC PEC PEC +boundary.particle_lo = Absorbing Absorbing Absorbing +boundary.particle_hi = Absorbing Absorbing Absorbing + +################################# +############ NUMERICS ########### +################################# +algo.particle_shape = 3 + +################################# +########### PARTICLES ########### +################################# +particles.species_names = beam1 + +beam1.species_type = electron +beam1.injection_style = gaussian_beam +beam1.x_rms = sigmax +beam1.y_rms = sigmay +beam1.z_rms = sigmaz +beam1.x_m = muy +beam1.y_m = mux +beam1.z_m = muz +beam1.focal_distance = focal_distance +beam1.npart = nmacropart +beam1.q_tot = -charge + +beam1.momentum_distribution_type = gaussian +beam1.ux_m = ux +beam1.uy_m = uy +beam1.uz_m = uz +beam1.ux_th = dux +beam1.uy_th = duy +beam1.uz_th = duz + +################################# +######### DIAGNOSTICS ########### +################################# +# FULL +diagnostics.diags_names = diag1 openpmd + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.write_species = 1 +diag1.species = beam1 +diag1.fields_to_plot = rho_beam1 +diag1.format = plotfile +diag1.dump_last_timestep = 1 + + +openpmd.intervals = 1 +openpmd.diag_type = Full +openpmd.write_species = 1 +openpmd.species = beam1 +openpmd.beam1.variables = w x y z +openpmd.fields_to_plot = none +openpmd.format = openpmd +openpmd.dump_last_timestep = 1 diff --git a/Examples/Tests/ion_stopping/analysis_ion_stopping.py b/Examples/Tests/ion_stopping/analysis_ion_stopping.py index dd9ee0f8744..b090014c0cf 100755 --- a/Examples/Tests/ion_stopping/analysis_ion_stopping.py +++ b/Examples/Tests/ion_stopping/analysis_ion_stopping.py @@ -73,7 +73,7 @@ def stopping_from_ions(dt, ni, Ti, mi, Zi, Zb, ion_mass, ion_energy): ion_energy = (f1)**(2./3.)/e return ion_energy -# Fetch background parameters and inital particle data +# Fetch background parameters and initial particle data ds0 = yt.load(f'{prefix}{len(last_it)*"0"}') ad0 = ds0.all_data() diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index 86d24181a75..a076361bf50 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -88,7 +88,7 @@ name = 'diag1', period = 10000, species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') field_diag = picmi.FieldDiagnostic( diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index e5d61fc0c0a..95732b03e36 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -93,5 +93,13 @@ assert( error_rel < tolerance_rel ) +# Check that the user runtime component (if it exists) worked as expected +try: + orig_z = ad['electrons', 'particle_orig_z'].v + assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print('particle_orig_z has reasonable values') +except yt.utilities.exceptions.YTFieldNotFound: + pass # Some of the tested script to not have the quantity orig_z + test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_rt b/Examples/Tests/ionization/inputs_2d_rt index 130eb1cc46a..f7035c567ac 100644 --- a/Examples/Tests/ionization/inputs_2d_rt +++ b/Examples/Tests/ionization/inputs_2d_rt @@ -36,6 +36,8 @@ ions.physical_element = N electrons.mass = m_e electrons.charge = -q_e electrons.injection_style = none +electrons.addRealAttributes = orig_z +electrons.attribute.orig_z(x,y,z,ux,uy,uz,t) = z lasers.names = laser1 laser1.profile = Gaussian diff --git a/Examples/Tests/langmuir/README.md b/Examples/Tests/langmuir/README.md deleted file mode 100644 index febf280d490..00000000000 --- a/Examples/Tests/langmuir/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Examples of Langmuir oscillations in a uniform plasma in 1D, 2D, 3D, and RZ - -In each case, a uniform plasma is setup with a sinusoidal perturbation in the -electron momentum along each axis. The plasma is followed for a short period -of time, long enough so that E fields develop. The resulting fields can be -compared to the analytic solutions. - -# Input files (for C++ version) - - inputs_1d - inputs_2d - inputs_3d - inputs_rz - -# Input files (for Python version) - - PICMI_inputs_2d.py - PICMI_inputs_3d.py - PICMI_inputs_rz.py - -# Analysis scripts to check the results - - analysis_1d.py - analysis_2d.py - analysis_3d.py - analysis_rz.py diff --git a/Examples/Tests/langmuir/README.rst b/Examples/Tests/langmuir/README.rst new file mode 100644 index 00000000000..60c0018744c --- /dev/null +++ b/Examples/Tests/langmuir/README.rst @@ -0,0 +1,146 @@ +.. _examples-langmuir: + +Langmuir Waves +============== + +These are examples of Plasma oscillations (`Langmuir waves `__) in a uniform plasma in 1D, 2D, 3D, and RZ. + +In each case, a uniform plasma is setup with a sinusoidal perturbation in the electron momentum along each axis. +The plasma is followed for a short period of time, long enough so that E fields develop. +The resulting fields can be compared to the analytic solutions. + + +Run +--- + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: 3D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_3d.py``. + + .. literalinclude:: PICMI_inputs_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_3d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.3d inputs_3d`` + + .. literalinclude:: inputs_3d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_3d``. + + .. tab-item:: 2D + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_2d.py``. + + .. literalinclude:: PICMI_inputs_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_2d.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.2d inputs_2d`` + + .. literalinclude:: inputs_2d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_2d``. + + + .. tab-item:: RZ + + .. tab-set:: + + .. tab-item:: Python: Script + + This example can be run as a **Python** script: ``python3 PICMI_inputs_rz.py``. + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/PICMI_inputs_rz.py``. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.rz inputs_rz`` + + .. literalinclude:: inputs_rz + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_rz``. + + + .. tab-item:: 1D + + .. tab-set:: + + .. tab-item:: Python: Script + + .. note:: + + TODO: This input file should be created, like the ``inputs_1d`` file. + + .. tab-item:: Executable: Input File + + This example can be run as WarpX **executable** using an input file: ``warpx.1d inputs_1d`` + + .. literalinclude:: inputs_1d + :language: ini + :caption: You can copy this file from ``Examples/Tests/langmuir/inputs_1d``. + + +Analyze +------- + +We run the following script to analyze correctness: + +.. tab-set:: + + .. tab-item:: 3D + + .. dropdown:: Script ``analysis_3d.py`` + + .. literalinclude:: analysis_3d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_3d.py``. + + .. tab-item:: 2D + + .. dropdown:: Script ``analysis_2d.py`` + + .. literalinclude:: analysis_2d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_2d.py``. + + .. tab-item:: RZ + + .. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_rz.py``. + + .. tab-item:: 1D + + .. dropdown:: Script ``analysis_1d.py`` + + .. literalinclude:: analysis_1d.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/langmuir/analysis_1d.py``. + + +Visualize +--------- + +.. note:: + + This section is TODO. diff --git a/Examples/Tests/langmuir/analysis_2d.py b/Examples/Tests/langmuir/analysis_2d.py index e137d50229d..b3327703b82 100755 --- a/Examples/Tests/langmuir/analysis_2d.py +++ b/Examples/Tests/langmuir/analysis_2d.py @@ -38,6 +38,9 @@ # Parse test name and check if Vay current deposition (algo.current_deposition=vay) is used vay_deposition = True if re.search( 'Vay_deposition', fn ) else False +# Parse test name and check if particle_shape = 4 is used +particle_shape_4 = True if re.search('particle_shape_4', fn) else False + # Parameters (these parameters must match the parameters in `inputs.multi.rt`) epsilon = 0.01 n = 4.e24 @@ -114,7 +117,11 @@ def get_theoretical_field( field, t ): fig.tight_layout() fig.savefig('Langmuir_multi_2d_analysis.png', dpi = 200) -tolerance_rel = 0.05 +if particle_shape_4: +# lower fidelity, due to smoothing + tolerance_rel = 0.07 +else: + tolerance_rel = 0.05 print("error_rel : " + str(error_rel)) print("tolerance_rel: " + str(tolerance_rel)) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index 8550b8767c6..eec2ba4fffb 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -65,11 +65,11 @@ keV_to_Joule = scc.e*1e3 MeV_to_Joule = scc.e*1e6 barn_to_square_meter = 1.e-28 -m_p = scc.m_p # Proton mass +m_p = 1.00782503223*scc.m_u # Proton mass m_b = 11.00930536*scc.m_u # Boron 11 mass m_reduced = m_p*m_b/(m_p+m_b) -m_a = 4.002602*scc.m_u # Alpha mass -m_be = 7.94748*m_p # Beryllium 8 mass +m_a = 4.00260325413*scc.m_u # Alpha mass +m_be = 7.94748*scc.m_p # Beryllium 8 mass Z_boron = 5. Z_proton = 1. E_Gamow = (Z_boron*Z_proton*np.pi*scc.fine_structure)**2*2.*m_reduced*scc.c**2 @@ -395,7 +395,7 @@ def p_sq_boron_frame_to_E_COM_frame(p_proton_sq): # Use invariant E**2 - p**2c**2 of 4-momentum norm to compute energy in center of mass frame E_com = np.sqrt(E_lab**2 - p_proton_sq*scc.c**2) # Corresponding kinetic energy - E_com_kin = E_com - (m_b+scc.m_p)*scc.c**2 + E_com_kin = E_com - (m_b+m_p)*scc.c**2 return E_com_kin*(p_proton_sq>0.) def p_sq_to_kinetic_energy(p_sq, m): @@ -521,11 +521,11 @@ def check_initial_energy2(data): # Loop over all slices (i.e. cells in the z direction) for slice_number in range(1, size_z): - ## For simplicity, all the calculations in this functino are done nonrelativistically + ## For simplicity, all the calculations in this function are done nonrelativistically ## Proton kinetic energy in the lab frame before fusion E_proton_nonrelativistic = Energy_step*slice_number**2 ## Corresponding square norm of proton momentum - p_proton_sq = 2.*scc.m_p*E_proton_nonrelativistic + p_proton_sq = 2.*m_p*E_proton_nonrelativistic ## Kinetic energy in the lab frame after ## proton + boron 11 -> alpha + beryllium 8 E_after_fusion = E_proton_nonrelativistic + E_fusion diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 3a8d2002a40..0943eee1d22 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -74,18 +74,22 @@ neutron_1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -deuterium_2.species_type = deuterium -deuterium_2.injection_style = "NRandomPerCell" -deuterium_2.num_particles_per_cell = 1000 -deuterium_2.profile = "parse_density_function" ## A tenth of the macroparticles in each cell is made of immobile high-density background deuteriums. ## The other nine tenths are made of fast low-density beam deuteriums. -deuterium_2.density_function(x,y,z) = if(y - floor(y) < 0.1, 10.*background_dens, 10./9.*beam_dens) -deuterium_2.momentum_distribution_type = "parse_momentum_function" -deuterium_2.momentum_function_ux(x,y,z) = 0. -deuterium_2.momentum_function_uy(x,y,z) = 0. -deuterium_2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight))" +deuterium_2.species_type = deuterium +deuterium_2.injection_sources = high_density low_density +deuterium_2.profile = constant +deuterium_2.high_density.injection_style = "NRandomPerCell" +deuterium_2.high_density.num_particles_per_cell = 100 +deuterium_2.high_density.density = 10.*background_dens +deuterium_2.high_density.momentum_distribution_type = at_rest +deuterium_2.low_density.injection_style = "NRandomPerCell" +deuterium_2.low_density.num_particles_per_cell = 900 +deuterium_2.low_density.density = 10./9.*beam_dens +deuterium_2.low_density.momentum_distribution_type = "parse_momentum_function" +deuterium_2.low_density.momentum_function_ux(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uy(x,y,z) = 0. +deuterium_2.low_density.momentum_function_uz(x,y,z) = sqrt(2*m_deuterium*Energy_step*(floor(z)**2))/(m_deuterium*clight) deuterium_2.do_not_push = 1 deuterium_2.do_not_deposit = 1 diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d index 36414552c11..30988b1ca2f 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_2d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 80000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 8000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(x - floor(x) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 4800 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 800 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 800 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d index 46744013986..421e39d2765 100644 --- a/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d +++ b/Examples/Tests/nuclear_fusion/inputs_proton_boron_3d @@ -33,11 +33,12 @@ particles.species_names = proton1 boron1 alpha1 proton2 boron2 alpha2 proton3 bo proton4 boron4 alpha4 proton5 boron5 alpha5 my_constants.m_b11 = 11.00930536*m_u # Boron 11 mass -my_constants.m_reduced = m_p*m_b11/(m_p+m_b11) +my_constants.m_h = 1.00782503223*m_u # Hydrogen 1 mass +my_constants.m_reduced = m_h*m_b11/(m_h+m_b11) my_constants.keV_to_J = 1.e3*q_e my_constants.Energy_step = 22. * keV_to_J -proton1.species_type = proton +proton1.species_type = hydrogen1 proton1.injection_style = "NRandomPerCell" proton1.num_particles_per_cell = 10000 proton1.profile = constant @@ -46,7 +47,7 @@ proton1.momentum_distribution_type = "parse_momentum_function" proton1.momentum_function_ux(x,y,z) = 0. proton1.momentum_function_uy(x,y,z) = 0. ## Thanks to the floor, all particles in the same cell have the exact same momentum -proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_p*clight) +proton1.momentum_function_uz(x,y,z) = sqrt(2*m_reduced*Energy_step*(floor(z)**2))/(m_h*clight) proton1.do_not_push = 1 proton1.do_not_deposit = 1 @@ -63,14 +64,14 @@ boron1.momentum_function_uz(x,y,z) = -sqrt(2*m_reduced*Energy_step*(floor(z)**2) boron1.do_not_push = 1 boron1.do_not_deposit = 1 -alpha1.species_type = helium +alpha1.species_type = helium4 alpha1.do_not_push = 1 alpha1.do_not_deposit = 1 my_constants.background_dens = 1.e26 my_constants.beam_dens = 1.e20 -proton2.species_type = proton +proton2.species_type = hydrogen1 proton2.injection_style = "NRandomPerCell" proton2.num_particles_per_cell = 1000 proton2.profile = "parse_density_function" @@ -81,7 +82,7 @@ proton2.momentum_distribution_type = "parse_momentum_function" proton2.momentum_function_ux(x,y,z) = 0. proton2.momentum_function_uy(x,y,z) = 0. proton2.momentum_function_uz(x,y,z) = "if(y - floor(y) < 0.1, - 0., sqrt(2*m_p*Energy_step*(floor(z)**2))/(m_p*clight))" + 0., sqrt(2*m_h*Energy_step*(floor(z)**2))/(m_h*clight))" proton2.do_not_push = 1 proton2.do_not_deposit = 1 @@ -94,19 +95,19 @@ boron2.momentum_distribution_type = "constant" boron2.do_not_push = 1 boron2.do_not_deposit = 1 -alpha2.species_type = helium +alpha2.species_type = helium4 alpha2.do_not_push = 1 alpha2.do_not_deposit = 1 my_constants.temperature = 44. * keV_to_J -proton3.species_type = proton +proton3.species_type = hydrogen1 proton3.injection_style = "NRandomPerCell" proton3.num_particles_per_cell = 600 proton3.profile = constant proton3.density = 1.e28 proton3.momentum_distribution_type = "maxwell_boltzmann" -proton3.theta = temperature/(m_p*clight**2) +proton3.theta = temperature/(m_h*clight**2) proton3.do_not_push = 1 proton3.do_not_deposit = 1 @@ -120,19 +121,19 @@ boron3.theta = temperature/(m_b11*clight**2) boron3.do_not_push = 1 boron3.do_not_deposit = 1 -alpha3.species_type = helium +alpha3.species_type = helium4 alpha3.do_not_push = 1 alpha3.do_not_deposit = 1 my_constants.proton4_energy = 550*keV_to_J -proton4.species_type = proton +proton4.species_type = hydrogen1 proton4.injection_style = "NRandomPerCell" proton4.num_particles_per_cell = 100 proton4.profile = "constant" proton4.density = 1.e35 proton4.momentum_distribution_type = "constant" -proton4.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton4.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton4.do_not_push = 1 proton4.do_not_deposit = 1 @@ -145,17 +146,17 @@ boron4.momentum_distribution_type = "constant" boron4.do_not_push = 1 boron4.do_not_deposit = 1 -alpha4.species_type = helium +alpha4.species_type = helium4 alpha4.do_not_push = 1 alpha4.do_not_deposit = 1 -proton5.species_type = proton +proton5.species_type = hydrogen1 proton5.injection_style = "NRandomPerCell" proton5.num_particles_per_cell = 100 proton5.profile = "constant" proton5.density = 1.e35 proton5.momentum_distribution_type = "constant" -proton5.uz = sqrt(2*m_p*proton4_energy)/(m_p*clight) +proton5.uz = sqrt(2*m_h*proton4_energy)/(m_h*clight) proton5.do_not_push = 1 proton5.do_not_deposit = 1 @@ -168,7 +169,7 @@ boron5.momentum_distribution_type = "constant" boron5.do_not_push = 1 boron5.do_not_deposit = 1 -alpha5.species_type = helium +alpha5.species_type = helium4 alpha5.do_not_push = 1 alpha5.do_not_deposit = 1 diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 9363bcebd18..4f3cd731323 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -58,7 +58,7 @@ class EMModes(object): # Plasma resistivity - used to dampen the mode excitation eta = [[1e-7, 1e-7], [1e-7, 1e-5], [1e-7, 1e-4]] # Number of substeps used to update B - substeps = [[250, 500], [250, 750], [250, 1000]] + substeps = 20 def __init__(self, test, dim, B_dir, verbose): """Get input parameters for the specific case desired.""" @@ -149,7 +149,6 @@ def get_simulation_parameters(self): self.NPPC = self.NPPC[self.dim-1] self.eta = self.eta[self.dim-1][idx] - self.substeps = self.substeps[self.dim-1][idx] def get_plasma_quantities(self): """Calculate various plasma parameters based on the simulation input.""" diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 21d8cafe750..81ba65e5b28 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -53,7 +53,7 @@ class CylindricalNormalModes(object): # Plasma resistivity - used to dampen the mode excitation eta = 5e-4 # Number of substeps used to update B - substeps = 250 + substeps = 20 def __init__(self, test, verbose): """Get input parameters for the specific case desired.""" diff --git a/Examples/Tests/ohm_solver_EM_modes/README.rst b/Examples/Tests/ohm_solver_EM_modes/README.rst new file mode 100644 index 00000000000..034ee5815f0 --- /dev/null +++ b/Examples/Tests/ohm_solver_EM_modes/README.rst @@ -0,0 +1,119 @@ +.. _examples-ohm-solver-em-modes: + +Ohm solver: Electromagnetic modes +================================= + +In this example a simulation is seeded with a thermal plasma while an initial magnetic field is applied in either the +:math:`z` or :math:`x` direction. The simulation is progressed for a large number of steps and the resulting fields are +Fourier analyzed for Alfvén mode excitations. + +Run +--- + +The same input script can be used for 1d, 2d or 3d Cartesian simulations as well +as replicating either the parallel propagating or ion-Bernstein modes as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Parallel propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir z + + .. tab-item:: Perpendicular propagating waves + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --bdir {x/y} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and compares the calculated spectrum +to the theoretical dispersions. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis.py``. + +Right and left circularly polarized electromagnetic waves are supported through the cyclotron motion of the ions, except +in a region of thermal resonances as indicated on the plot below. + +.. figure:: https://user-images.githubusercontent.com/40245517/216207688-9c39374a-9e69-45b8-a588-35b087b83d27.png + :alt: Parallel EM modes in thermal ion plasma + :width: 70% + + Calculated Alvén waves spectrum with the theoretical dispersions overlaid. + +Perpendicularly propagating modes are also supported, commonly referred to as ion-Bernstein modes. + +.. figure:: https://user-images.githubusercontent.com/40245517/231217944-7d12b8d4-af4b-44f8-a1b9-a2b59ce3a1c2.png + :alt: Perpendicular modes in thermal ion plasma + :width: 50% + + Calculated ion Bernstein waves spectrum with the theoretical dispersion overlaid. + +Ohm solver: Cylindrical normal modes +==================================== + +A RZ-geometry example case for normal modes propagating along an applied magnetic +field in a cylinder is also available. The analytical solution for these modes +are described in :cite:t:`ex-Stix1992` Chapter 6, Sec. 2. + +Run +--- + +The following script initializes a thermal plasma in a metallic cylinder with +periodic boundaries at the cylinder ends. + +.. dropdown:: Script ``PICMI_inputs_rz.py`` + + .. literalinclude:: PICMI_inputs_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py``. + +The example can be executed using: + +.. code-block:: bash + + python3 PICMI_inputs_rz.py + +Analyze +------- + +After the simulation completes the following script can be used to analyze the +field evolution and extract the normal mode dispersion relation. It performs a +standard Fourier transform along the cylinder axis and a Hankel transform in the +radial direction. + +.. dropdown:: Script ``analysis_rz.py`` + + .. literalinclude:: analysis_rz.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_EM_modes/analysis_rz.py``. + +The following figure was produced with the above analysis script, showing excellent +agreement between the calculated and theoretical dispersion relations. + +.. figure:: https://user-images.githubusercontent.com/40245517/259251824-33e78375-81d8-410d-a147-3fa0498c66be.png + :alt: Normal EM modes in a metallic cylinder + :width: 90% + + Cylindrical normal mode dispersion comparing the calculated spectrum with the + theoretical one. diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index 9eb747dd6e8..9004c24a05c 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -154,7 +154,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.41633903, 19.39353485, 101.08693342, 11.09248295]) + amps, np.array([61.50945919, 19.74831134, 101.01820349, 10.8974811]) ) if sim.test: diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index bd28f46b087..d545195a9e5 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -2,7 +2,7 @@ # # --- Test script for the kinetic-fluid hybrid model in WarpX wherein ions are # --- treated as kinetic particles and electrons as an isothermal, inertialess -# --- background fluid. The script simulates ion Landau damping as descibed +# --- background fluid. The script simulates ion Landau damping as described # --- in section 4.5 of Munoz et al. (2018). import argparse @@ -56,7 +56,7 @@ class IonLandauDamping(object): # Plasma resistivity - used to dampen the mode excitation eta = 1e-7 # Number of substeps used to update B - substeps = 100 + substeps = 10 def __init__(self, test, dim, m, T_ratio, verbose): @@ -222,7 +222,7 @@ def setup_run(self): period=100, write_dir='.', species=[self.ions], - data_list = ['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], warpx_file_prefix=f'Python_ohms_law_solver_landau_damping_{self.dim}d_plt', ) simulation.add_diagnostic(particle_diag) diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst new file mode 100644 index 00000000000..dd4f94b4edf --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/README.rst @@ -0,0 +1,50 @@ +.. _examples-ohm-solver-ion-landau-damping: + +Ohm solver: Ion Landau Damping +============================== + +Landau damping is a well known process in which electrostatic (acoustic) waves are +damped by transferring energy to particles satisfying a resonance condition. +The process can be simulated by seeding a plasma with a specific acoustic mode +(density perturbation) and tracking the strength of the mode as a function of time. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations and to sweep different +temperature ratios. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --temp_ratio {value} + +Analyze +------- + +The following script extracts the amplitude of the seeded mode as a function +of time and compares it to the theoretical damping rate. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py``. + +The figure below shows a set of such simulations with parameters matching those +described in section 4.5 of :cite:t:`ex-MUNOZ2018`. +The straight lines show the theoretical damping rate for the given temperature ratios. + +.. figure:: https://user-images.githubusercontent.com/40245517/230523935-3c8d63bd-ee69-4639-b111-f06dad5587f6.png + :alt: Ion Landau damping + :width: 70% + + Decay of seeded modes as a function of time for different electron-ion temperature ratios. + The theoretical damping of the given modes are shown in dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 79f268f6e3b..d9a71df6ac4 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -54,7 +54,7 @@ class HybridPICBeamInstability(object): # Plasma resistivity - used to dampen the mode excitation eta = 1e-7 # Number of substeps used to update B - substeps = 400 + substeps = 10 # Beam parameters n_beam = [0.02, 0.1] @@ -95,11 +95,9 @@ def __init__(self, test, dim, resonant, verbose): diag_period = 1 / 4.0 # Output interval (ion cyclotron periods) self.diag_steps = int(diag_period / self.DT) - # if this is a test case run for only 25 cyclotron periods and use - # fewer substeps to speed up the simulation + # if this is a test case run for only 25 cyclotron periods if self.test: self.LT = 25.0 - self.substeps = 100 self.total_steps = int(np.ceil(self.LT / self.DT)) @@ -261,7 +259,7 @@ def setup_run(self): name='diag1', period=1250, species=[self.ions, self.beam_ions], - data_list = ['ux', 'uy', 'uz', 'x', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'z', 'weighting'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_ion_beam_1d_plt', ) @@ -378,7 +376,7 @@ def text_diag(self): self.prev_step = step def energy_diagnostic(self): - """Diangostic to get the total, magnetic and kinetic energies in the + """Diagnostic to get the total, magnetic and kinetic energies in the simulation.""" step = simulation.extension.warpx.getistep(lev=0) - 1 diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/README.rst b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst new file mode 100644 index 00000000000..59469cf4aa9 --- /dev/null +++ b/Examples/Tests/ohm_solver_ion_beam_instability/README.rst @@ -0,0 +1,75 @@ +.. _examples-ohm-solver-ion-beam-instability: + +Ohm solver: Ion Beam R Instability +================================== + +In this example a low density ion beam interacts with a "core" plasma population which induces an instability. +Based on the relative density between the beam and the core plasma a resonant or non-resonant condition can +be accessed. + +Run +--- + +The same input script can be used for 1d, 2d or 3d simulations as well as +replicating either the resonant or non-resonant condition as indicated below. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py``. + +For `MPI-parallel `__ runs, prefix these lines with ``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + +.. tab-set:: + + .. tab-item:: Resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} --resonant + + .. tab-item:: Non-resonant case + + Execute: + + .. code-block:: bash + + python3 PICMI_inputs.py -dim {1/2/3} + +Analyze +------- + +The following script reads the simulation output from the above example, performs +Fourier transforms of the field data and outputs the figures shown below. + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_ion_beam_instability/analysis.py``. + +The figures below show the evolution of the y-component of the magnetic field as the beam and +core plasma interact. + +.. figure:: https://user-images.githubusercontent.com/40245517/217923933-6bdb65cb-7d26-40d8-8687-7dd75274bd48.png + :alt: Resonant ion beam R instability + :width: 70% + +.. figure:: https://user-images.githubusercontent.com/40245517/217925983-b91d6482-69bc-43c1-8c7d-23ebe7c69d49.png + :alt: Non-resonant ion beam R instability + :width: 70% + + Evolution of :math:`B_y` for resonant (top) and non-resonant (bottom) conditions. + +The growth rates of the strongest growing modes for the resonant case are compared +to theory (dashed lines) in the figure below. + +.. figure:: https://github.com/ECP-WarpX/WarpX/assets/40245517/a94bb6e5-30e9-4d8f-9e6b-844dc8f51d17 + :alt: Resonant ion beam R instability growth rates + :width: 50% + + Time series of the mode amplitudes for m = 4, 5, 6 from simulation. The + theoretical growth for these modes are also shown as dashed lines. diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index b541a83283a..6a57b1c1046 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -85,7 +85,7 @@ plt.plot(t_grid, np.abs(field_kt[:, 5] / sim.B0), 'b', label=f'm = 5, $kl_i={k[5]:.2f}$') plt.plot(t_grid, np.abs(field_kt[:, 6] / sim.B0), 'k', label=f'm = 6, $kl_i={k[6]:.2f}$') - # The theoretical growth rates for the 4th, 5th and 6th Foruier modes of + # The theoretical growth rates for the 4th, 5th and 6th Fourier modes of # the By-field was obtained from Fig. 12a of Munoz et al. # Note the rates here are gamma / w_ci gamma4 = 0.1915611861780133 @@ -190,11 +190,11 @@ # assert's fail, the full benchmark should be rerun (same as the test but # without the `--test` argument) and the growth rates (up to saturation) # compared to the theoretical ones to determine if the physics test passes. - # At creation, the full test had the following errors when ran on 8 procs: - # m4_rms_error = 4.476; m5_rms_error = 9.211; m6_rms_error = 3.252 - assert m4_rms_error < 1.55 - assert m5_rms_error < 0.75 - assert m6_rms_error < 0.40 + # At creation, the full test (3d) had the following errors (ran on 1 V100): + # m4_rms_error = 3.329; m5_rms_error = 1.052; m6_rms_error = 2.583 + assert np.isclose(m4_rms_error, 1.515, atol=0.01) + assert np.isclose(m5_rms_error, 0.718, atol=0.01) + assert np.isclose(m6_rms_error, 0.357, atol=0.01) # checksum check import os diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 143f1f3e82e..a7f2e0da12d 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -59,7 +59,7 @@ class ForceFreeSheetReconnection(object): # Plasma resistivity - used to dampen the mode excitation eta = 6e-3 # normalized resistivity # Number of substeps used to update B - substeps = 750 + substeps = 20 def __init__(self, test, verbose): @@ -255,7 +255,7 @@ def setup_run(self): period=self.total_steps, write_dir='.', species=[self.ions], - data_list=['ux', 'uy', 'uz', 'x', 'y', 'weighting'], + data_list=['ux', 'uy', 'uz', 'x', 'z', 'weighting'], warpx_file_prefix='Python_ohms_law_solver_magnetic_reconnection_2d_plt', # warpx_format='openpmd', # warpx_openpmd_backend='h5', diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst new file mode 100644 index 00000000000..943b5bd0248 --- /dev/null +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/README.rst @@ -0,0 +1,45 @@ +.. _examples-ohm-solver-magnetic-reconnection: + +Ohm Solver: Magnetic Reconnection +================================= + +Hybrid-PIC codes are often used to simulate magnetic reconnection in space plasmas. +An example of magnetic reconnection from a force-free sheet is provided, based on +the simulation described in :cite:t:`ex-Le2016`. + +Run +--- + +The following **Python** script configures and launches the simulation. + +.. dropdown:: Script ``PICMI_inputs.py`` + + .. literalinclude:: PICMI_inputs.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py``. + +Running the full simulation should take about 4 hours if executed on 1 V100 GPU. +For `MPI-parallel `__ runs, prefix these lines with +``mpiexec -n 4 ...`` or ``srun -n 4 ...``, depending on the system. + + .. code-block:: bash + + python3 PICMI_inputs.py + +Analyze +------- + +The following script extracts the reconnection rate as a function of time and +animates the evolution of the magnetic field (as shown below). + +.. dropdown:: Script ``analysis.py`` + + .. literalinclude:: analysis.py + :language: python3 + :caption: You can copy this file from ``Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py``. + +.. figure:: https://user-images.githubusercontent.com/40245517/229639784-b5d3b596-3550-4570-8761-8d9a67aa4b3b.gif + :alt: Magnetic reconnection. + :width: 70% + + Magnetic reconnection from a force-free sheet. diff --git a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py index 9ac40818e12..bb1ebd73082 100755 --- a/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py +++ b/Examples/Tests/particle_boundary_process/PICMI_inputs_reflection.py @@ -131,12 +131,12 @@ print("Number of electrons in lower buffer:", n) assert n == 67 -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_hi', 'step_scraped', 0) +scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_hi', 'stepScraped', 0) for arr in scraped_steps: # print(arr) assert all(arr == 4) -scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_lo', 'step_scraped', 0) +scraped_steps = buffer.get_particle_boundary_buffer("electrons", 'z_lo', 'stepScraped', 0) for arr in scraped_steps: # print(arr) assert all(arr == 8) diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index 9871bdac655..e5a9a58f597 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -133,7 +133,7 @@ print(f"Number of electrons in buffer (proc #{my_id}): {n}") assert n == 612 -scraped_steps = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'step_scraped', 0) +scraped_steps = particle_buffer.get_particle_boundary_buffer("electrons", 'eb', 'stepScraped', 0) for arr in scraped_steps: assert all(np.array(arr, copy=False) > 40) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index a4b7d9e134e..572871b8ed5 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -153,10 +153,10 @@ def add_particles(): ########################## assert (elec_wrapper.nps == 270 / (2 - args.unique)) -assert (elec_wrapper.particle_container.get_comp_index('w') == 0) -assert (elec_wrapper.particle_container.get_comp_index('newPid') == 4) +assert (elec_wrapper.particle_container.get_comp_index('w') == 2) +assert (elec_wrapper.particle_container.get_comp_index('newPid') == 6) -new_pid_vals = elec_wrapper.get_particle_arrays('newPid', 0) +new_pid_vals = elec_wrapper.get_particle_real_arrays('newPid', 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 1becd4464e7..5de9879f0f8 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -120,12 +120,12 @@ elec_count = elec_wrapper.nps # check that the runtime attributes have the right indices -assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 4) -assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 5) +assert (elec_wrapper.particle_container.get_comp_index('prev_x') == 6) +assert (elec_wrapper.particle_container.get_comp_index('prev_z') == 7) # sanity check that the prev_z values are reasonable and # that the correct number of values are returned -prev_z_vals = elec_wrapper.get_particle_arrays('prev_z', 0) +prev_z_vals = elec_wrapper.get_particle_real_arrays('prev_z', 0) running_count = 0 for z_vals in prev_z_vals: diff --git a/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py b/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py new file mode 100755 index 00000000000..81a981e70a6 --- /dev/null +++ b/Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 Luca Fedeli, Maxence Thevenet, Remi Lehe +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +""" +This script tests the absorption of particles in the PML. + +The input file inputs_2d/inputs is used: it features a positive and a +negative particle, going in opposite direction and eventually +leaving the box. This script tests that the field in the box +is close to 0 once the particles have left. With regular +PML, this test fails, since the particles leave a spurious +charge, with associated fields, behind them. +""" +import os +import sys + +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +ds = yt.load( filename ) + +# Check that the field is low enough +ad0 = ds.covering_grid(level=0, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array_lev0 = ad0[('mesh','Ex')].to_ndarray() +Ey_array_lev0 = ad0[('mesh','Ey')].to_ndarray() +Ez_array_lev0 = ad0[('mesh','Ez')].to_ndarray() +max_Efield_lev0 = max(Ex_array_lev0.max(), Ey_array_lev0.max(), Ez_array_lev0.max()) +print( "max_Efield level0 = %s" %max_Efield_lev0 ) + +ad1 = ds.covering_grid(level=1, left_edge=ds.domain_left_edge, dims=ds.domain_dimensions) +Ex_array_lev1 = ad1[('mesh','Ex')].to_ndarray() +Ey_array_lev1 = ad1[('mesh','Ey')].to_ndarray() +Ez_array_lev1 = ad1[('mesh','Ez')].to_ndarray() +max_Efield_lev1 = max(Ex_array_lev1.max(), Ey_array_lev1.max(), Ez_array_lev1.max()) +print( "max_Efield level1 = %s" %max_Efield_lev1 ) + +# The field associated with the particle does not have +# the same amplitude in 2d and 3d +if ds.dimensionality == 2: + if ds.max_level == 0: + tolerance_abs = 0.0003 + elif ds.max_level == 1: + tolerance_abs = 0.0006 +elif ds.dimensionality == 3: + if ds.max_level == 0: + tolerance_abs = 10 + elif ds.max_level == 1: + tolerance_abs = 110 +else: + raise ValueError("Unknown dimensionality") + +print("tolerance_abs: " + str(tolerance_abs)) +assert max_Efield_lev0 < tolerance_abs +assert max_Efield_lev1 < tolerance_abs + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particles_in_pml/inputs_mr_2d b/Examples/Tests/particles_in_pml/inputs_mr_2d index a6db42a3897..40e869a5e16 100644 --- a/Examples/Tests/particles_in_pml/inputs_mr_2d +++ b/Examples/Tests/particles_in_pml/inputs_mr_2d @@ -1,16 +1,15 @@ -max_step = 200 +max_step = 300 amr.n_cell = 128 128 amr.blocking_factor = 16 -amr.max_grid_size = 1024 +amr.max_grid_size = 64 amr.max_level = 1 # Geometry geometry.dims = 2 geometry.prob_lo = -32.e-6 -32.e-6 # physical domain geometry.prob_hi = 32.e-6 32.e-6 -warpx.fine_tag_lo = -8.e-6 -8.e-6 # physical domain -warpx.fine_tag_hi = 8.e-6 8.e-6 +warpx.ref_patch_function(x,y,z) = " ((x > -16.e-6)*(x<-10.e-6) + (x > 9.e-6)*(x<16.e-6))*(z>-8.e-6)*(z<8.e-6)" # Boundary condition boundary.field_lo = pml pml pml @@ -38,14 +37,14 @@ particles.species_names = electron proton electron.charge = -q_e electron.mass = m_e electron.injection_style = "singleparticle" -electron.single_particle_pos = 0. 0. 0. +electron.single_particle_pos = -12.e-6 0. 0. electron.single_particle_u = 2. 0. 0. electron.single_particle_weight = 1. proton.charge = q_e -proton.mass = m_p # Very heavy ; should not move +proton.mass = m_p proton.injection_style = "singleparticle" -proton.single_particle_pos = 0. 0. 0. +proton.single_particle_pos = -12.e-6 0. 0. proton.single_particle_u = -2. 0. 0. proton.single_particle_weight = 1. @@ -54,6 +53,6 @@ algo.particle_shape = 1 # Diagnostics diagnostics.diags_names = diag1 -diag1.intervals = 200 +diag1.intervals = 300 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz diff --git a/Examples/Tests/pec/analysis_pec.py b/Examples/Tests/pec/analysis_pec.py index 0414420ef91..841cc06ae22 100755 --- a/Examples/Tests/pec/analysis_pec.py +++ b/Examples/Tests/pec/analysis_pec.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/pec/analysis_pec_mr.py b/Examples/Tests/pec/analysis_pec_mr.py index 1f9e5276c51..e8aab4dcd6f 100755 --- a/Examples/Tests/pec/analysis_pec_mr.py +++ b/Examples/Tests/pec/analysis_pec_mr.py @@ -7,7 +7,7 @@ # License: BSD-3-Clause-LBNL # # This is a script that analyses the simulation results from -# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoindal wave. +# the script `inputs_field_PEC_mr_3d`. This simulates a sinusoidal wave. # The electric field (Ey) is a standing wave due to the PEC boundary condition, # and as a result, the minimum and maximum value after reflection would be two times the value at initialization due to constructive interference. # Additionally, the value of Ey at the boundary must be equal to zero. diff --git a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py index 6114fb05de2..50d222bbf36 100644 --- a/Examples/Tests/plasma_lens/PICMI_inputs_3d.py +++ b/Examples/Tests/plasma_lens/PICMI_inputs_3d.py @@ -33,17 +33,25 @@ # Particles vel_z = 0.5*c -multiparticles_distribution = picmi.ParticleListDistribution(x = [0.05, 0.], - y = [0., 0.04], - z = [0.05, 0.05], - ux = [0., 0.], - uy = [0., 0.], - uz = [vel_z, vel_z], - weight = [1., 1.]) +offset_x_particle = picmi.ParticleListDistribution(x = [0.05], + y = [0.], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) + +offset_y_particle = picmi.ParticleListDistribution(x = [0.], + y = [0.04], + z = [0.05], + ux = [0.], + uy = [0.], + uz = [vel_z], + weight = [1.]) electrons = picmi.Species(particle_type = 'electron', name = 'electrons', - initial_distribution = multiparticles_distribution) + initial_distribution = [offset_x_particle, offset_y_particle]) # Plasma lenses plasma_lenses = picmi.PlasmaLens(period = 0.5, diff --git a/Examples/Tests/plasma_lens/analysis.py b/Examples/Tests/plasma_lens/analysis.py index b410c6eb128..9ce82d903f8 100755 --- a/Examples/Tests/plasma_lens/analysis.py +++ b/Examples/Tests/plasma_lens/analysis.py @@ -92,12 +92,22 @@ def applylens(x0, vx0, vz0, gamma, lens_length, lens_strength): plasma_lens_strengths_B = np.zeros(len(plasma_lens_zstarts)) -x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) -y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) -z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) -ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c -uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c -uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c +try: + # The picmi version + x0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_x')) + y0 = float(ds.parameters.get('electrons.dist1.multiple_particles_pos_y')) + z0 = float(ds.parameters.get('electrons.dist0.multiple_particles_pos_z')) + ux0 = float(ds.parameters.get('electrons.dist0.multiple_particles_ux'))*c + uy0 = float(ds.parameters.get('electrons.dist1.multiple_particles_uy'))*c + uz0 = eval(ds.parameters.get('electrons.dist0.multiple_particles_uz'))*c +except TypeError: + # The inputs version + x0 = float(ds.parameters.get('electrons.multiple_particles_pos_x').split()[0]) + y0 = float(ds.parameters.get('electrons.multiple_particles_pos_y').split()[1]) + z0 = float(ds.parameters.get('electrons.multiple_particles_pos_z').split()[0]) + ux0 = float(ds.parameters.get('electrons.multiple_particles_ux').split()[0])*c + uy0 = float(ds.parameters.get('electrons.multiple_particles_uy').split()[1])*c + uz0 = eval(ds.parameters.get('electrons.multiple_particles_uz').split()[0])*c tt = 0. xx = x0 diff --git a/Examples/Tests/point_of_contact_EB/analysis.py b/Examples/Tests/point_of_contact_EB/analysis.py new file mode 100755 index 00000000000..cb1fc23ee62 --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/analysis.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +""" +This script tests the coordinates of the point of contact of an electron hitting a sphere in 3D. +It compares the numerical results with the analytical solutions. +The sphere is centered on O and has a radius of 0.2 (EB) +The electron is initially at: (-0.25,0,0) and moves with a normalized momentum: (1,0.5,0) +An input file PICMI_inputs_3d.py is used. +""" +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +yt.funcs.mylog.setLevel(0) +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# Open plotfile specified in command line +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename, output_format='openpmd') + +ts_scraping = OpenPMDTimeSeries('./diags/diag2/particles_at_eb/') + +it=ts_scraping.iterations +step_scraped, delta, x, y, z, nx, ny, nz=ts_scraping.get_particle( ['stepScraped','deltaTimeScraped','x','y','z', 'nx', 'ny', 'nz'], species='electron', iteration=it ) +delta_reduced=delta[0]*1e10 + +# Analytical results calculated +x_analytic=-0.1983 +y_analytic=0.02584 +z_analytic=0.0000 +nx_analytic=-0.99 +ny_analytic=0.13 +nz_analytic=0.0 + +#result obtained by analysis of simulations +step_ref=3 +delta_reduced_ref=0.59 + +print('NUMERICAL coordinates of the point of contact:') +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_scraped[0],delta_reduced,x[0], y[0], z[0], nx[0], ny[0], nz[0])) +print('\n') +print('ANALYTICAL coordinates of the point of contact:') +print('step_scraped=%d, time_stamp=%5.4f e-10, x=%5.4f, y=%5.4f, z=%5.4f, nx=%5.4f, ny=%5.4f, nz=%5.4f' % (step_ref, delta_reduced_ref, x_analytic, y_analytic, z_analytic, nx_analytic, ny_analytic, nz_analytic)) + +tolerance=0.001 +tolerance_t=0.01 +tolerance_n=0.01 +print("tolerance = "+ str(tolerance *100) + '%') +print("tolerance for the time = "+ str(tolerance_t *100) + '%') +print("tolerance for the normal components = "+ str(tolerance_n *100) + '%') + +diff_step=np.abs((step_scraped[0]-step_ref)/step_ref) +diff_delta=np.abs((delta_reduced-delta_reduced_ref)/delta_reduced_ref) +diff_x=np.abs((x[0]-x_analytic)/x_analytic) +diff_y=np.abs((y[0]-y_analytic)/y_analytic) +diff_nx=np.abs((nx[0]-nx_analytic)/nx_analytic) +diff_ny=np.abs((ny[0]-ny_analytic)/ny_analytic) + +print("percentage error for x = %5.4f %%" %(diff_x *100)) +print("percentage error for y = %5.4f %%" %(diff_y *100)) +print("percentage error for nx = %5.2f %%" %(diff_nx *100)) +print("percentage error for ny = %5.2f %%" %(diff_ny *100)) +print("nz = %5.2f " %(nz[0])) + +assert (diff_x < tolerance) and (diff_y < tolerance) and (np.abs(z[0]) < 1e-8) and (diff_step < 1e-8) and (diff_delta < tolerance_t) and (diff_nx < tolerance_n) and (diff_ny < tolerance_n) and (np.abs(nz) < 1e-8) , 'Test point_of_contact did not pass' diff --git a/Examples/Tests/point_of_contact_EB/inputs_3d b/Examples/Tests/point_of_contact_EB/inputs_3d new file mode 100644 index 00000000000..005733487e5 --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/inputs_3d @@ -0,0 +1,48 @@ +amr.max_level = 0 + +max_step = 3 + +amr.n_cell = 64 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = 3 +geometry.prob_lo = -0.26 -0.26 -0.26 +geometry.prob_hi = 0.26 0.26 0.26 + +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1e-10 +warpx.eb_implicit_function = "-(x**2+y**2+z**2-0.2**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = -0.25 0 0 +electron.single_particle_u = 1 0.5 0.0 +electron.single_particle_weight = 1 + +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex +diag1.format = openpmd + +diag2.diag_type = BoundaryScraping +diag2.format = openpmd diff --git a/Examples/Tests/point_of_contact_EB/inputs_rz b/Examples/Tests/point_of_contact_EB/inputs_rz new file mode 100644 index 00000000000..a492c1f705a --- /dev/null +++ b/Examples/Tests/point_of_contact_EB/inputs_rz @@ -0,0 +1,48 @@ +amr.max_level = 0 + +max_step = 3 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.26 +geometry.prob_hi = 0.26 0.26 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1e-10 +warpx.eb_implicit_function = "-(x**2 -0.2**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "SingleParticle" +electron.single_particle_pos = -0.25 0 0 +electron.single_particle_u = 1 0.5 0.0 +electron.single_particle_weight = 1 + +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er +diag1.format = openpmd + +diag2.diag_type = BoundaryScraping +diag2.format = openpmd diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index 706dedb6959..32c9f4e5808 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -158,10 +158,10 @@ def add_particles(): ########################## assert electron_wrapper.nps == 90 -assert electron_wrapper.particle_container.get_comp_index("w") == 0 -assert electron_wrapper.particle_container.get_comp_index("newPid") == 4 +assert electron_wrapper.particle_container.get_comp_index("w") == 2 +assert electron_wrapper.particle_container.get_comp_index("newPid") == 6 -new_pid_vals = electron_wrapper.get_particle_arrays("newPid", 0) +new_pid_vals = electron_wrapper.get_particle_real_arrays("newPid", 0) for vals in new_pid_vals: assert np.allclose(vals, 5) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 0390eed580e..7d40bd5edf4 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -53,8 +53,8 @@ def n_remaining_particles( iteration ): w, = ts_full.get_particle(['w'], iteration=iteration) return len(w) def n_scraped_particles( iteration ): - timestamp = ts_scraping.get_particle( ['timestamp'], iteration=ts_scraping.iterations[0] ) - return (timestamp <= iteration).sum() + step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + return (step_scraped <= iteration).sum() n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) n_total = n_remaining[0] diff --git a/Examples/Tests/scraping/analysis_rz_filter.py b/Examples/Tests/scraping/analysis_rz_filter.py new file mode 100755 index 00000000000..dd13b993dc5 --- /dev/null +++ b/Examples/Tests/scraping/analysis_rz_filter.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2023 Kale Weichman +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the particle scraping for the embedded boundary in RZ. +# Particles are initialized between r=0.15 and r=0.2 +# having a negative radial velocity. +# A cylindrical embedded surface is placed at r=0.1. +# Upon reaching the surface, particles should be removed. +# At the end of the simulation, i.e., at time step 37, +# there should be 512 particles left. +# This test checks that plot_filter_fucntion works with the boundary scraping diagnostic +# by making sure that the particles removed from only half the domain (z>0) have been recorded. + +# Possible errors: 0 +# tolerance: 0 +# Possible running time: < 1 s + +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +import yt + +tolerance = 0 + +fn = sys.argv[1] +ds = yt.load( fn ) +ad = ds.all_data() +x = ad['electron', 'particle_position_x'].v + +error = len(x)-512 +print('error = ', error) +print('tolerance = ', tolerance) +assert(error==tolerance) + +# Check that all the removed particles are properly recorded +# by making sure that, at each iteration, the sum of the number of +# remaining particles and scraped particles is equal to half the +# original number of particles +# also check that no particles with z <= 0 have been scraped +ts_full = OpenPMDTimeSeries('./diags/diag2/') +ts_scraping = OpenPMDTimeSeries('./diags/diag3/particles_at_eb') + +def n_remaining_particles( iteration ): + w, = ts_full.get_particle(['w'], iteration=iteration) + return len(w) +def n_scraped_particles( iteration ): + step_scraped = ts_scraping.get_particle( ['stepScraped'], iteration=ts_scraping.iterations[0] ) + return (step_scraped <= iteration).sum() +def n_scraped_z_leq_zero( iteration ): + z_pos, = ts_scraping.get_particle( ['z'], iteration=ts_scraping.iterations[0] ) + return (z_pos <= 0).sum() +n_remaining = np.array([ n_remaining_particles(iteration) for iteration in ts_full.iterations ]) +n_scraped = np.array([ n_scraped_particles(iteration) for iteration in ts_full.iterations ]) +n_z_leq_zero = np.array([ n_scraped_z_leq_zero(iteration) for iteration in ts_full.iterations ]) +n_total = n_remaining[0] +assert np.all( 2*n_scraped+n_remaining == n_total) +assert np.all( n_z_leq_zero == 0) diff --git a/Examples/Tests/scraping/inputs_rz_filter b/Examples/Tests/scraping/inputs_rz_filter new file mode 100644 index 00000000000..0d67fb96f6c --- /dev/null +++ b/Examples/Tests/scraping/inputs_rz_filter @@ -0,0 +1,58 @@ +amr.max_level = 1 +warpx.fine_tag_lo = 0.0 -0.5 +warpx.fine_tag_hi = 0.25 0.0 + +max_step = 37 + +amr.n_cell = 64 64 +amr.blocking_factor = 8 +amr.max_grid_size = 128 + +geometry.dims = RZ +geometry.prob_lo = 0.0 -0.5 +geometry.prob_hi = 0.5 0.5 + +boundary.field_lo = none periodic +boundary.field_hi = pec periodic +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +warpx.const_dt = 1.216119097e-11 +warpx.eb_implicit_function = "-(x**2-0.1**2)" + +# Do not evolve the E and B fields +algo.maxwell_solver = none +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + +particles.species_names = electron +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 4 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x+y*y>0.15*0.15)*(x*x+y*y<0.2*0.2)*1.0e21" +electron.momentum_distribution_type = parse_momentum_function +electron.momentum_function_ux(x,y,z) = "if(x*x+y*y>0.0, -1.0*x/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uy(x,y,z) = "if(x*x+y*y>0.0, -1.0*y/sqrt(x*x+y*y), 0.0)" +electron.momentum_function_uz(x,y,z) = "0" +electron.save_particles_at_eb = 1 + +diagnostics.diags_names = diag1 diag2 diag3 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Er + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Er +diag2.format = openpmd + +diag3.diag_type = BoundaryScraping +diag3.format = openpmd +diag3.electron.plot_filter_function(t,x,y,z,ux,uy,uz) = "z > 0" diff --git a/Examples/Tests/single_particle/analysis_bilinear_filter.py b/Examples/Tests/single_particle/analysis_bilinear_filter.py index 95ee0761943..db7250dc3bb 100755 --- a/Examples/Tests/single_particle/analysis_bilinear_filter.py +++ b/Examples/Tests/single_particle/analysis_bilinear_filter.py @@ -42,7 +42,7 @@ my_order[1] -= 1 my_filter = my_filterx[:,None]*my_filtery -# Apply filter. my_F_filetered is the theoretical value for filtered field +# Apply filter. my_F_filtered is the theoretical value for filtered field my_F_filtered = signal.convolve2d(my_F_nofilter, my_filter, boundary='symm', mode='same') # Get simulation result for F_filtered diff --git a/Examples/analysis_default_openpmd_regression.py b/Examples/analysis_default_openpmd_regression.py new file mode 100755 index 00000000000..3aadc49ac51 --- /dev/null +++ b/Examples/analysis_default_openpmd_regression.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +import os +import re +import sys + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +if re.search( 'single_precision', fn ): + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd', rtol=2.e-6) +else: + checksumAPI.evaluate_checksum(test_name, fn, output_format='openpmd') diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 6beb97c8291..91a34b9d3d6 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -23,7 +23,7 @@ def _localsetattr(self, name, value): object.__setattr__(self, name, value) def add_new_attr(self, name, value): - """Names starting with "_" are make instance attributes. + """Names starting with "_" are made instance attributes. Otherwise the attribute is added to the args list. """ if name.startswith('_'): @@ -31,6 +31,15 @@ def add_new_attr(self, name, value): else: self.argvattrs[name] = value + def add_new_group_attr(self, group, name, value): + """The attribute is added to the args list in the form "group.name" if + group is not an empty string, otherwise as only "name". + """ + if group: + self.argvattrs[f'{group}.{name}'] = value + else: + self.argvattrs[name] = value + def __setattr__(self, name, value): self.add_new_attr(name, value) diff --git a/Python/pywarpx/_libwarpx.py b/Python/pywarpx/_libwarpx.py index fa2044d4240..1eb410e65b8 100755 --- a/Python/pywarpx/_libwarpx.py +++ b/Python/pywarpx/_libwarpx.py @@ -5,8 +5,8 @@ # # NOTE: We will reduce the libwarpx.py level of abstraction eventually! # Please add new functionality directly to pybind11-bound modules -# and call them via sim.extension.libwarpx_so. ... and sim.extension.warpx. -# ... from user code. +# and call them via sim.extension.libwarpx_so. ... and sim.extension.Config and +# sim.extension.warpx. ... from user code. # # Authors: Axel Huebl, Andrew Myers, David Grote, Remi Lehe, Weiqun Zhang # @@ -108,6 +108,8 @@ def load_library(self): from . import warpx_pybind_3d as cxx_3d self.libwarpx_so = cxx_3d self.dim = 3 + + self.Config = self.libwarpx_so.Config except ImportError: raise Exception(f"Dimensionality '{self.geometry_dim}' was not compiled in this Python install. Please recompile with -DWarpX_DIMS={_dims}") diff --git a/Python/pywarpx/callbacks.py b/Python/pywarpx/callbacks.py index b8824d0ef16..8f3c1dc5e2c 100644 --- a/Python/pywarpx/callbacks.py +++ b/Python/pywarpx/callbacks.py @@ -1,66 +1,79 @@ -# Copyright 2017-2022 The WarpX Community +# Copyright 2017-2023 The WarpX Community # # This file is part of WarpX. # -# Authors: David Grote, Roelof Groenewald +# Authors: David Grote, Roelof Groenewald, Axel Huebl # # License: BSD-3-Clause-LBNL -"""call back operations -===================== +"""Callback Locations +------------------ These are the functions which allow installing user created functions so that they are called at various places along the time step. The following three functions allow the user to install, uninstall and verify the different call back types. - - installcallback: Installs a function to be called at that specified time - - uninstallcallback: Uninstalls the function (so it won't be called anymore) - - isinstalled: Checks if the function is installed + +* :py:func:`installcallback`: Installs a function to be called at that specified time +* :py:func:`uninstallcallback`: Uninstalls the function (so it won't be called anymore) +* :py:func:`isinstalled`: Checks if the function is installed These functions all take a callback location name (string) and function or instance method as an argument. Note that if an instance method is used, an extra reference to the method's object is saved. -The install can be done using a decorator, which has the prefix "callfrom". See -example below. - Functions can be called at the following times: - - beforeInitEsolve: before the initial solve for the E fields (i.e. before the PIC loop starts) - - afterinit: immediately after the init is complete - - beforeEsolve: before the solve for E fields - - poissonsolver: In place of the computePhi call but only in an electrostatic simulation - - afterEsolve: after the solve for E fields - - beforedeposition: before the particle deposition (for charge and/or current) - - afterdeposition: after particle deposition (for charge and/or current) - - beforestep: before the time step - - afterstep: after the time step - - afterdiagnostics: after diagnostic output - - oncheckpointsignal: on a checkpoint signal - - onbreaksignal: on a break signal. These callbacks will be the last ones executed before the simulation ends. - - particlescraper: just after the particle boundary conditions are applied - but before lost particles are processed - - particleloader: at the time that the standard particle loader is called - - particleinjection: called when particle injection happens, after the position - advance and before deposition is called, allowing a user - defined particle distribution to be injected each time step - - appliedfields: allows directly specifying any fields to be applied to the particles - during the advance - -To use a decorator, the syntax is as follows. This will install the function -``myplots`` to be called after each step. - -@callfromafterstep -def myplots(): - ppzx() - -This is equivalent to the following: - -def myplots(): - ppzx() -installcallback('afterstep', myplots) +* ``beforeInitEsolve``: before the initial solve for the E fields (i.e. before the PIC loop starts) +* ``afterinit``: immediately after the init is complete +* ``beforeEsolve``: before the solve for E fields +* ``poissonsolver``: In place of the computePhi call but only in an electrostatic simulation +* ``afterEsolve``: after the solve for E fields +* ``beforedeposition``: before the particle deposition (for charge and/or current) +* ``afterdeposition``: after particle deposition (for charge and/or current) +* ``beforestep``: before the time step +* ``afterstep``: after the time step +* ``afterdiagnostics``: after diagnostic output +* ``oncheckpointsignal``: on a checkpoint signal +* ``onbreaksignal``: on a break signal. These callbacks will be the last ones executed before the simulation ends. +* ``particlescraper``: just after the particle boundary conditions are applied + but before lost particles are processed +* ``particleloader``: at the time that the standard particle loader is called +* ``particleinjection``: called when particle injection happens, after the position + advance and before deposition is called, allowing a user + defined particle distribution to be injected each time step + +Example that calls the Python function ``myplots`` after each step: + +.. code-block:: python3 + + from pywarpx.callbacks import installcallback + + def myplots(): + # do something here + + installcallback('afterstep', myplots) + + # run simulation + sim.step(nsteps=100) + +The install can also be done using a `Python decorator `__, which has the prefix ``callfrom``. +To use a decorator, the syntax is as follows. This will install the function ``myplots`` to be called after each step. +The above example is quivalent to the following: + +.. code-block:: python3 + + from pywarpx.callbacks import callfromafterstep + + @callfromafterstep + def myplots(): + # do something here + + # run simulation + sim.step(nsteps=100) """ + import copy import sys import time @@ -81,7 +94,7 @@ class CallbackFunctions(object): original reference. This is good since the user does not need to keep the reference to the object (for example it can be created using a local variable in a function). It may be bad if the user thinks an object was - deleted, but it actually isn't since it had (unkown to the user) + deleted, but it actually isn't since it had (unknown to the user) installed a method in one of the call back lists. """ @@ -282,15 +295,20 @@ def callfuncsinlist(self,*args,**kw): callback_instances[key] = CallbackFunctions(name=key, **val) def installcallback(name, f): - "Adds a function to the list of functions called by this callback" + """Installs a function to be called at that specified time. + + Adds a function to the list of functions called by this callback. + """ callback_instances[name].installfuncinlist(f) def uninstallcallback(name, f): - "Removes the function from the list of functions called by this callback" + """Uninstalls the function (so it won't be called anymore). + + Removes the function from the list of functions called by this callback.""" callback_instances[name].uninstallfuncinlist(f) def isinstalled(name, f): - "Checks if the function is called by this callback" + """Checks if a function is installed for this callback.""" return callback_instances[name].isinstalledfuncinlist(f) def clear_all(): diff --git a/Python/pywarpx/fields.py b/Python/pywarpx/fields.py index 5eeeda50ea7..dc0de9d9491 100644 --- a/Python/pywarpx/fields.py +++ b/Python/pywarpx/fields.py @@ -120,7 +120,7 @@ def shape(self): min_box = self.mf.box_array().minimal_box() shape = list(min_box.size - min_box.small_end) if self.include_ghosts: - nghosts = self.mf.n_grow_vect() + nghosts = self.mf.n_grow_vect shape = [shape[i] + 2*nghosts[i] for i in range(self.dim)] shape.append(self.mf.nComp) return tuple(shape) @@ -159,16 +159,9 @@ def mesh(self, direction): if self.include_ghosts: # The ghost cells are added to the upper and lower end of the global domain. - nghosts = self.mf.n_grow_vect() - ilo = list(ilo) - ihi = list(ihi) - min_box = self.mf.box_array().minimal_box() - imax = min_box.big_end - for i in range(self.dim): - if ilo[i] == 0: - ilo[i] -= nghosts[i] - if ihi[i] == imax[i]: - ihi[i] += nghosts[i] + nghosts = self.mf.n_grow_vect + ilo -= nghosts[idir] + ihi += nghosts[idir] # Cell size in the direction warpx = libwarpx.libwarpx_so.get_instance() @@ -206,7 +199,7 @@ def _get_indices(self, index, missing): def _get_n_ghosts(self): """Return the list of number of ghosts. This includes the component dimension.""" - nghosts = list(self._get_indices(self.mf.n_grow_vect(), 0)) + nghosts = list(self._get_indices(self.mf.n_grow_vect, 0)) # The components always has nghosts = 0 nghosts.append(0) return nghosts @@ -215,7 +208,7 @@ def _get_min_indices(self): """Returns the minimum indices, expanded to length 3""" min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imin = self._get_indices(min_box.small_end, 0) return imin @@ -224,7 +217,7 @@ def _get_max_indices(self): """ min_box = self.mf.box_array().minimal_box() if self.include_ghosts: - min_box.grow(self.mf.n_grow_vect()) + min_box.grow(self.mf.n_grow_vect) imax = self._get_indices(min_box.big_end, 0) return imax @@ -348,7 +341,7 @@ def _get_intersect_slice(self, mfi, starts, stops, icstart, icstop): """ box = mfi.tilebox() if self.include_ghosts: - box.grow(self.mf.n_grow_vect()) + box.grow(self.mf.n_grow_vect) ilo = self._get_indices(box.small_end, 0) ihi = self._get_indices(box.big_end, 0) @@ -416,7 +409,7 @@ def __getitem__(self, index): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) # Gather the data to be included in a list to be sent to other processes starts = [ixstart, iystart, izstart] @@ -506,7 +499,7 @@ def __setitem__(self, index, value): ixstart, ixstop = self._find_start_stop(ii[0], ixmin, ixmax+1, 0) iystart, iystop = self._find_start_stop(ii[1], iymin, iymax+1, 1) izstart, izstop = self._find_start_stop(ii[2], izmin, izmax+1, 2) - icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp(), 3) + icstart, icstop = self._find_start_stop(ic, 0, self.mf.n_comp, 3) if isinstance(value, np.ndarray): # Expand the shape of the input array to match the shape of the global array diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index cece7968079..19b0a1ab6c4 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -117,14 +117,16 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, kwargs[key] = np.full(maxlen, val) # --- The number of built in attributes + # --- The positions + built_in_attrs = libwarpx.dim # --- The three velocities - built_in_attrs = 3 + built_in_attrs += 3 if libwarpx.geometry_dim == 'rz': # --- With RZ, there is also theta built_in_attrs += 1 # --- The number of extra attributes (including the weight) - nattr = self.particle_container.num_real_comps() - built_in_attrs + nattr = self.particle_container.num_real_comps - built_in_attrs attr = np.zeros((maxlen, nattr)) attr[:,0] = w @@ -188,28 +190,25 @@ def add_real_comp(self, pid_name, comm=True): self.particle_container.add_real_comp(pid_name, comm) - def get_particle_structs(self, level, copy_to_host=False): + def get_particle_real_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle struct data - on each tile for this process. The particle data is represented as a structured - array and contains the particle 'x', 'y', 'z', and 'idcpu'. + This returns a list of numpy or cupy arrays containing the particle real array data + on each tile for this process. - Unless copy_to_host is specified, the data for the structs are - not copied, but share the underlying memory buffer with WarpX. The + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The arrays are fully writeable. - Note that cupy does not support structs: - https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: "|V24"). If copied - to host via copy_to_host, we correct for the right numpy AoS type. - Parameters ---------- - level : int + comp_name : str + The component of the array data that will be returned + + level : int The refinement level to reference (default=0) - copy_to_host : bool + copy_to_host : bool For GPU-enabled runs, one can either return the GPU arrays (the default) or force a device-to-host copy. @@ -217,30 +216,29 @@ def get_particle_structs(self, level, copy_to_host=False): ------- List of arrays - The requested particle struct data + The requested particle array data ''' - particle_data = [] + comp_idx = self.particle_container.get_comp_index(comp_name) + + data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): + soa = pti.soa() + idx = soa.get_real_data(comp_idx) if copy_to_host: - particle_data.append(pti.aos().to_numpy(copy=True)) + data_array.append(idx.to_numpy(copy=True)) else: - if libwarpx.amr.Config.have_gpu: - libwarpx.amr.Print( - "get_particle_structs: cupy does not yet support structs. " - "https://github.com/cupy/cupy/issues/2031" - "Did you mean copy_to_host=True?" - ) xp, cupy_status = load_cupy() if cupy_status is not None: libwarpx.amr.Print(cupy_status) - aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy - particle_data.append(aos_arr) - return particle_data + + data_array.append(xp.array(idx, copy=False)) + + return data_array - def get_particle_arrays(self, comp_name, level, copy_to_host=False): + def get_particle_int_arrays(self, comp_name, level, copy_to_host=False): ''' - This returns a list of numpy or cupy arrays containing the particle array data + This returns a list of numpy or cupy arrays containing the particle int array data on each tile for this process. Unless copy_to_host is specified, the data for the arrays are not @@ -266,12 +264,52 @@ def get_particle_arrays(self, comp_name, level, copy_to_host=False): List of arrays The requested particle array data ''' - comp_idx = self.particle_container.get_comp_index(comp_name) + comp_idx = self.particle_container.get_icomp_index(comp_name) + + data_array = [] + for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): + soa = pti.soa() + idx = soa.get_int_data(comp_idx) + if copy_to_host: + data_array.append(idx.to_numpy(copy=True)) + else: + xp, cupy_status = load_cupy() + if cupy_status is not None: + libwarpx.amr.Print(cupy_status) + + data_array.append(xp.array(idx, copy=False)) + + return data_array + + + def get_particle_idcpu_arrays(self, level, copy_to_host=False): + ''' + This returns a list of numpy or cupy arrays containing the particle idcpu data + on each tile for this process. + + Unless copy_to_host is specified, the data for the arrays are not + copied, but share the underlying memory buffer with WarpX. The + arrays are fully writeable. + + Parameters + ---------- + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + List of arrays + The requested particle array data + ''' data_array = [] for pti in libwarpx.libwarpx_so.WarpXParIter(self.particle_container, level): soa = pti.soa() - idx = soa.GetRealData(comp_idx) + idx = soa.get_idcpu_data() if copy_to_host: data_array.append(idx.to_numpy(copy=True)) else: @@ -284,6 +322,31 @@ def get_particle_arrays(self, comp_name, level, copy_to_host=False): return data_array + def get_particle_idcpu(self, level=0, copy_to_host=False): + ''' + Return a list of numpy or cupy arrays containing the particle 'idcpu' + numbers on each tile. + + Parameters + ---------- + + level : int + The refinement level to reference (default=0) + + copy_to_host : bool + For GPU-enabled runs, one can either return the GPU + arrays (the default) or force a device-to-host copy. + + Returns + ------- + + List of arrays + The requested particle idcpu + ''' + return self.get_particle_idcpu_arrays(level, copy_to_host=copy_to_host) + idcpu = property(get_particle_idcpu) + + def get_particle_id(self, level=0, copy_to_host=False): ''' Return a list of numpy or cupy arrays containing the particle 'id' @@ -305,8 +368,8 @@ def get_particle_id(self, level=0, copy_to_host=False): List of arrays The requested particle ids ''' - structs = self.get_particle_structs(level, copy_to_host) - return [libwarpx.amr.unpack_ids(struct['cpuid']) for struct in structs] + idcpu = self.get_particle_idcpu(level, copy_to_host) + return [libwarpx.amr.unpack_ids(tile) for tile in idcpu] def get_particle_cpu(self, level=0, copy_to_host=False): @@ -330,8 +393,8 @@ def get_particle_cpu(self, level=0, copy_to_host=False): List of arrays The requested particle cpus ''' - structs = self.get_particle_structs(level, copy_to_host) - return [libwarpx.amr.unpack_cpus(struct['cpuid']) for struct in structs] + idcpu = self.get_particle_idcpu(level, copy_to_host) + return [libwarpx.amr.unpack_cpus(tile) for tile in idcpu] def get_particle_x(self, level=0, copy_to_host=False): @@ -355,20 +418,7 @@ def get_particle_x(self, level=0, copy_to_host=False): List of arrays The requested particle x position ''' - if copy_to_host: - # Use the numpy version of cosine - xp = np - else: - xp, cupy_status = load_cupy() - - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d' or libwarpx.geometry_dim == '2d': - return [struct['x'] for struct in structs] - elif libwarpx.geometry_dim == 'rz': - theta = self.get_particle_theta(level, copy_to_host) - return [struct['x']*xp.cos(theta) for struct, theta in zip(structs, theta)] - elif libwarpx.geometry_dim == '1d': - raise Exception('get_particle_x: There is no x coordinate with 1D Cartesian') + return self.get_particle_real_arrays('x', level, copy_to_host=copy_to_host) xp = property(get_particle_x) @@ -393,20 +443,7 @@ def get_particle_y(self, level=0, copy_to_host=False): List of arrays The requested particle y position ''' - if copy_to_host: - # Use the numpy version of sine - xp = np - else: - xp, cupy_status = load_cupy() - - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d': - return [struct['y'] for struct in structs] - elif libwarpx.geometry_dim == 'rz': - theta = self.get_particle_theta(level, copy_to_host) - return [struct['x']*xp.sin(theta) for struct, theta in zip(structs, theta)] - elif libwarpx.geometry_dim == '1d' or libwarpx.geometry_dim == '2d': - raise Exception('get_particle_y: There is no y coordinate with 1D or 2D Cartesian') + return self.get_particle_real_arrays('y', level, copy_to_host=copy_to_host) yp = property(get_particle_y) @@ -433,11 +470,12 @@ def get_particle_r(self, level=0, copy_to_host=False): ''' xp, cupy_status = load_cupy() - structs = self.get_particle_structs(level, copy_to_host) if libwarpx.geometry_dim == 'rz': - return [struct['x'] for struct in structs] + return self.get_particle_x(level, copy_to_host) elif libwarpx.geometry_dim == '3d': - return [xp.sqrt(struct['x']**2 + struct['y']**2) for struct in structs] + x = self.get_particle_x(level, copy_to_host) + y = self.get_particle_y(level, copy_to_host) + return xp.sqrt(x**2 + y**2) elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_r: There is no r coordinate with 1D or 2D Cartesian') rp = property(get_particle_r) @@ -467,10 +505,11 @@ def get_particle_theta(self, level=0, copy_to_host=False): xp, cupy_status = load_cupy() if libwarpx.geometry_dim == 'rz': - return self.get_particle_arrays('theta', level, copy_to_host) + return self.get_particle_real_arrays('theta', level, copy_to_host) elif libwarpx.geometry_dim == '3d': - structs = self.get_particle_structs(level, copy_to_host) - return [xp.arctan2(struct['y'], struct['x']) for struct in structs] + x = self.get_particle_x(level, copy_to_host) + y = self.get_particle_y(level, copy_to_host) + return xp.arctan2(y, x) elif libwarpx.geometry_dim == '2d' or libwarpx.geometry_dim == '1d': raise Exception('get_particle_theta: There is no theta coordinate with 1D or 2D Cartesian') thetap = property(get_particle_theta) @@ -497,13 +536,7 @@ def get_particle_z(self, level=0, copy_to_host=False): List of arrays The requested particle z position ''' - structs = self.get_particle_structs(level, copy_to_host) - if libwarpx.geometry_dim == '3d': - return [struct['z'] for struct in structs] - elif libwarpx.geometry_dim == 'rz' or libwarpx.geometry_dim == '2d': - return [struct['y'] for struct in structs] - elif libwarpx.geometry_dim == '1d': - return [struct['x'] for struct in structs] + return self.get_particle_real_arrays('z', level, copy_to_host=copy_to_host) zp = property(get_particle_z) @@ -528,7 +561,7 @@ def get_particle_weight(self, level=0, copy_to_host=False): List of arrays The requested particle weight ''' - return self.get_particle_arrays('w', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('w', level, copy_to_host=copy_to_host) wp = property(get_particle_weight) @@ -553,7 +586,7 @@ def get_particle_ux(self, level=0, copy_to_host=False): List of arrays The requested particle x momentum ''' - return self.get_particle_arrays('ux', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('ux', level, copy_to_host=copy_to_host) uxp = property(get_particle_ux) @@ -578,7 +611,7 @@ def get_particle_uy(self, level=0, copy_to_host=False): List of arrays The requested particle y momentum ''' - return self.get_particle_arrays('uy', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('uy', level, copy_to_host=copy_to_host) uyp = property(get_particle_uy) @@ -604,7 +637,7 @@ def get_particle_uz(self, level=0, copy_to_host=False): The requested particle z momentum ''' - return self.get_particle_arrays('uz', level, copy_to_host=copy_to_host) + return self.get_particle_real_arrays('uz', level, copy_to_host=copy_to_host) uzp = property(get_particle_uz) @@ -652,9 +685,10 @@ def getbz(self): def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): - ''' + """ Deposit this species' charge density in rho_fp in order to access that data via pywarpx.fields.RhoFPWrapper(). + Parameters ---------- species_name : str @@ -665,7 +699,7 @@ def deposit_charge_density(self, level, clear_rho=True, sync_rho=True): If True, zero out rho_fp before deposition. sync_rho : bool If True, perform MPI exchange and properly set boundary cells for rho_fp. - ''' + """ rho_fp = libwarpx.warpx.multifab(f'rho_fp[level={level}]') if rho_fp is None: @@ -719,70 +753,6 @@ def get_particle_boundary_buffer_size(self, species_name, boundary, local=False) ) - def get_particle_boundary_buffer_structs( - self, species_name, boundary, level, copy_to_host=False - ): - ''' - This returns a list of numpy or cupy arrays containing the particle struct data - for a species that has been scraped by a specific simulation boundary. The - particle data is represented as a structured array and contains the - particle 'x', 'y', 'z', and 'idcpu'. - - Unless copy_to_host is specified, the data for the structs are - not copied, but share the underlying memory buffer with WarpX. The - arrays are fully writeable. - - Note that cupy does not support structs: - https://github.com/cupy/cupy/issues/2031 - and will return arrays of binary blobs for the AoS (DP: "|V24"). If copied - to host via copy_to_host, we correct for the right numpy AoS type. - - Parameters - ---------- - - species_name : str - The species name that the data will be returned for - - boundary : str - The boundary from which to get the scraped particle data in the - form x/y/z_hi/lo or eb. - - level : int - The refinement level to reference (default=0) - - copy_to_host : bool - For GPU-enabled runs, one can either return the GPU - arrays (the default) or force a device-to-host copy. - - Returns - ------- - - List of arrays - The requested particle struct data - ''' - particle_container = self.particle_buffer.get_particle_container( - species_name, self._get_boundary_number(boundary) - ) - - particle_data = [] - for pti in libwarpx.libwarpx_so.BoundaryBufferParIter(particle_container, level): - if copy_to_host: - particle_data.append(pti.aos().to_numpy(copy=True)) - else: - if libwarpx.amr.Config.have_gpu: - libwarpx.amr.Print( - "get_particle_structs: cupy does not yet support structs. " - "https://github.com/cupy/cupy/issues/2031" - "Did you mean copy_to_host=True?" - ) - xp, cupy_status = load_cupy() - if cupy_status is not None: - libwarpx.amr.Print(cupy_status) - aos_arr = xp.array(pti.aos(), copy=False) # void blobs for cupy - particle_data.append(aos_arr) - return particle_data - - def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level): ''' This returns a list of numpy or cupy arrays containing the particle array data @@ -802,9 +772,10 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) form x/y/z_hi/lo or eb. comp_name : str - The component of the array data that will be returned. If - "step_scraped" the special attribute holding the timestep at - which a particle was scraped will be returned. + The component of the array data that will be returned. + "x", "y", "z", "ux", "uy", "uz", "w" + "stepScraped","deltaTimeScraped", + if boundary='eb': "nx", "ny", "nz" level : int Which AMR level to retrieve scraped particle data from. @@ -815,18 +786,20 @@ def get_particle_boundary_buffer(self, species_name, boundary, comp_name, level) species_name, self._get_boundary_number(boundary) ) data_array = [] - if comp_name == 'step_scraped': - # the step scraped is always the final integer component - comp_idx = part_container.num_int_comps() - 1 + #loop over the real attributes + if comp_name in part_container.real_comp_names: + comp_idx = part_container.real_comp_names[comp_name] for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(xp.array(soa.GetIntData(comp_idx), copy=False)) - else: - container_wrapper = ParticleContainerWrapper(species_name) - comp_idx = container_wrapper.particle_container.get_comp_index(comp_name) - for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): + data_array.append(xp.array(soa.get_real_data(comp_idx), copy=False)) + #loop over the integer attributes + elif comp_name in part_container.int_comp_names: + comp_idx = part_container.int_comp_names[comp_name] + for ii, pti in enumerate(libwarpx.libwarpx_so.BoundaryBufferParIter(part_container, level)): soa = pti.soa() - data_array.append(xp.array(soa.GetRealData(comp_idx), copy=False)) + data_array.append(xp.array(soa.get_int_data(comp_idx), copy=False)) + else: + raise RuntimeError('Name %s not found' %comp_name) return data_array diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 1b0358e2e94..959db6c701d 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -45,7 +45,7 @@ class constants: class Species(picmistandard.PICMI_Species): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -79,7 +79,7 @@ class Species(picmistandard.PICMI_Species): Whether or not to push this species warpx_do_not_gather: bool, default=False - Whether or not to gahter the fields from grids for this species + Whether or not to gather the fields from grids for this species warpx_random_theta: bool, default=True Whether or not to add random angle to the particles in theta @@ -219,10 +219,10 @@ def init(self, kw): self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) - def initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + def species_initialize_inputs(self, layout, + initialize_self_fields = False, + injection_plane_position = None, + injection_plane_normal_vector = None): self.species_number = len(pywarpx.particles.species_names) if self.name is None: @@ -271,7 +271,22 @@ def initialize_inputs(self, layout, pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: - self.initial_distribution.initialize_inputs(self.species_number, layout, self.species, self.density_scale) + distributions_is_list = np.iterable(self.initial_distribution) + layout_is_list = np.iterable(layout) + if not distributions_is_list and not layout_is_list: + self.initial_distribution.distribution_initialize_inputs(self.species_number, layout, self.species, + self.density_scale, '') + elif distributions_is_list and (layout_is_list or layout is None): + assert layout is None or (len(self.initial_distribution) == len(layout)),\ + Exception('The initial distribution and layout lists must have the same lenth') + source_names = [f'dist{i}' for i in range(len(self.initial_distribution))] + self.species.injection_sources = source_names + for i, dist in enumerate(self.initial_distribution): + layout_i = layout[i] if layout is not None else None + dist.distribution_initialize_inputs(self.species_number, layout_i, self.species, + self.density_scale, source_names[i]) + else: + raise Exception('The initial distribution and layout must both be scalars or both be lists') if injection_plane_position is not None: if injection_plane_normal_vector is not None: @@ -284,15 +299,15 @@ def initialize_inputs(self, layout, picmistandard.PICMI_MultiSpecies.Species_class = Species class MultiSpecies(picmistandard.PICMI_MultiSpecies): - def initialize_inputs(self, layout, - initialize_self_fields = False, - injection_plane_position = None, - injection_plane_normal_vector = None): + def species_initialize_inputs(self, layout, + initialize_self_fields = False, + injection_plane_position = None, + injection_plane_normal_vector = None): for species in self.species_instances_list: - species.initialize_inputs(layout, - initialize_self_fields, - injection_plane_position, - injection_plane_normal_vector) + species.species_initialize_inputs(layout, + initialize_self_fields, + injection_plane_position, + injection_plane_normal_vector) class GaussianBunchDistribution(picmistandard.PICMI_GaussianBunchDistribution): @@ -300,17 +315,17 @@ def init(self, kw): self.do_symmetrize = kw.pop('warpx_do_symmetrize', None) self.symmetrization_order = kw.pop('warpx_symmetrization_order', None) - def initialize_inputs(self, species_number, layout, species, density_scale): - species.injection_style = "gaussian_beam" - species.x_m = self.centroid_position[0] - species.y_m = self.centroid_position[1] - species.z_m = self.centroid_position[2] - species.x_rms = self.rms_bunch_size[0] - species.y_rms = self.rms_bunch_size[1] - species.z_rms = self.rms_bunch_size[2] + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): + species.add_new_group_attr(source_name, 'injection_style', "gaussian_beam") + species.add_new_group_attr(source_name, 'x_m', self.centroid_position[0]) + species.add_new_group_attr(source_name, 'y_m', self.centroid_position[1]) + species.add_new_group_attr(source_name, 'z_m', self.centroid_position[2]) + species.add_new_group_attr(source_name, 'x_rms', self.rms_bunch_size[0]) + species.add_new_group_attr(source_name, 'y_rms', self.rms_bunch_size[1]) + species.add_new_group_attr(source_name, 'z_rms', self.rms_bunch_size[2]) # --- Only PseudoRandomLayout is supported - species.npart = layout.n_macroparticles + species.add_new_group_attr(source_name, 'npart', layout.n_macroparticles) # --- Calculate the total charge. Note that charge might be a string instead of a number. charge = species.charge @@ -318,9 +333,9 @@ def initialize_inputs(self, species_number, layout, species, density_scale): charge = constants.q_e elif charge == '-q_e': charge = -constants.q_e - species.q_tot = self.n_physical_particles*charge + species.add_new_group_attr(source_name, 'q_tot', self.n_physical_particles*charge) if density_scale is not None: - species.q_tot *= density_scale + species.add_new_group_attr(source_name, 'q_tot', density_scale) # --- The PICMI standard doesn't yet have a way of specifying these values. # --- They should default to the size of the domain. They are not typically @@ -334,26 +349,26 @@ def initialize_inputs(self, species_number, layout, species, density_scale): # --- Note that WarpX takes gamma*beta as input if np.any(np.not_equal(self.velocity_divergence, 0.)): - species.momentum_distribution_type = "radial_expansion" - species.u_over_r = self.velocity_divergence[0]/constants.c - #species.u_over_y = self.velocity_divergence[1]/constants.c - #species.u_over_z = self.velocity_divergence[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "radial_expansion") + species.add_new_group_attr(source_name, 'u_over_r', self.velocity_divergence[0]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_y', self.velocity_divergence[1]/constants.c) + #species.add_new_group_attr(source_name, 'u_over_z', self.velocity_divergence[2]/constants.c) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.centroid_velocity[0]/constants.c - species.uy_m = self.centroid_velocity[1]/constants.c - species.uz_m = self.centroid_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.centroid_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.centroid_velocity[0]/constants.c - species.uy = self.centroid_velocity[1]/constants.c - species.uz = self.centroid_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.centroid_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.centroid_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.centroid_velocity[2]/constants.c) - species.do_symmetrize = self.do_symmetrize - species.symmetrization_order = self.symmetrization_order + species.add_new_group_attr(source_name, 'do_symmetrize', self.do_symmetrize) + species.add_new_group_attr(source_name, 'symmetrization_order', self.symmetrization_order) class DensityDistributionBase(object): @@ -369,98 +384,98 @@ def set_mangle_dict(self): # times self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - def set_species_attributes(self, species, layout): + def set_species_attributes(self, species, layout, source_name): if isinstance(layout, GriddedLayout): # --- Note that the grid attribute of GriddedLayout is ignored - species.injection_style = "nuniformpercell" - species.num_particles_per_cell_each_dim = layout.n_macroparticle_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nuniformpercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell_each_dim', layout.n_macroparticle_per_cell) elif isinstance(layout, PseudoRandomLayout): assert (layout.n_macroparticles_per_cell is not None), Exception('WarpX only supports n_macroparticles_per_cell for the PseudoRandomLayout with this distribution') - species.injection_style = "nrandompercell" - species.num_particles_per_cell = layout.n_macroparticles_per_cell + species.add_new_group_attr(source_name, 'injection_style', "nrandompercell") + species.add_new_group_attr(source_name, 'num_particles_per_cell', layout.n_macroparticles_per_cell) else: raise Exception('WarpX does not support the specified layout for this distribution') - species.xmin = self.lower_bound[0] - species.xmax = self.upper_bound[0] - species.ymin = self.lower_bound[1] - species.ymax = self.upper_bound[1] - species.zmin = self.lower_bound[2] - species.zmax = self.upper_bound[2] + species.add_new_group_attr(source_name, 'xmin', self.lower_bound[0]) + species.add_new_group_attr(source_name, 'xmax', self.upper_bound[0]) + species.add_new_group_attr(source_name, 'ymin', self.lower_bound[1]) + species.add_new_group_attr(source_name, 'ymax', self.upper_bound[1]) + species.add_new_group_attr(source_name, 'zmin', self.lower_bound[2]) + species.add_new_group_attr(source_name, 'zmax', self.upper_bound[2]) if self.fill_in: - species.do_continuous_injection = 1 + species.add_new_group_attr(source_name, 'do_continuous_injection', 1) # --- Note that WarpX takes gamma*beta as input if (hasattr(self, "momentum_spread_expressions") and np.any(np.not_equal(self.momentum_spread_expressions, None)) ): species.momentum_distribution_type = 'gaussian_parse_momentum_function' - self.setup_parse_momentum_functions(species, self.momentum_expressions, '_m', self.directed_velocity) - self.setup_parse_momentum_functions(species, self.momentum_spread_expressions, '_th', [0.,0.,0.]) + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '_m', self.directed_velocity) + self.setup_parse_momentum_functions(species, source_name, self.momentum_spread_expressions, '_th', [0.,0.,0.]) elif (hasattr(self, "momentum_expressions") and np.any(np.not_equal(self.momentum_expressions, None)) ): - species.momentum_distribution_type = 'parse_momentum_function' - self.setup_parse_momentum_functions(species, self.momentum_expressions, '', self.directed_velocity) + species.add_new_group_attr(source_name, 'momentum_distribution_type', 'parse_momentum_function') + self.setup_parse_momentum_functions(species, source_name, self.momentum_expressions, '', self.directed_velocity) elif np.any(np.not_equal(self.rms_velocity, 0.)): - species.momentum_distribution_type = "gaussian" - species.ux_m = self.directed_velocity[0]/constants.c - species.uy_m = self.directed_velocity[1]/constants.c - species.uz_m = self.directed_velocity[2]/constants.c - species.ux_th = self.rms_velocity[0]/constants.c - species.uy_th = self.rms_velocity[1]/constants.c - species.uz_th = self.rms_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussian") + species.add_new_group_attr(source_name, 'ux_m', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_m', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_m', self.directed_velocity[2]/constants.c) + species.add_new_group_attr(source_name, 'ux_th', self.rms_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy_th', self.rms_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz_th', self.rms_velocity[2]/constants.c) else: - species.momentum_distribution_type = "constant" - species.ux = self.directed_velocity[0]/constants.c - species.uy = self.directed_velocity[1]/constants.c - species.uz = self.directed_velocity[2]/constants.c + species.add_new_group_attr(source_name, 'momentum_distribution_type', "constant") + species.add_new_group_attr(source_name, 'ux', self.directed_velocity[0]/constants.c) + species.add_new_group_attr(source_name, 'uy', self.directed_velocity[1]/constants.c) + species.add_new_group_attr(source_name, 'uz', self.directed_velocity[2]/constants.c) - def setup_parse_momentum_functions(self, species, expressions, suffix, defaults): + def setup_parse_momentum_functions(self, species, source_name, expressions, suffix, defaults): for sdir, idir in zip(['x', 'y', 'z'], [0, 1, 2]): if expressions[idir] is not None: expression = pywarpx.my_constants.mangle_expression(expressions[idir], self.mangle_dict) else: expression = f'{defaults[idir]}' - species.__setattr__(f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') + species.add_new_group_attr(source_name, f'momentum_function_u{sdir}{suffix}(x,y,z)', f'({expression})/{constants.c}') class UniformFluxDistribution(picmistandard.PICMI_UniformFluxDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.fill_in = False self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.flux_profile = "constant" - species.flux = self.flux + species.add_new_group_attr(source_name, 'flux_profile', "constant") + species.add_new_group_attr(source_name, 'flux', self.flux) if density_scale is not None: - species.flux *= density_scale - species.flux_normal_axis = self.flux_normal_axis - species.surface_flux_pos = self.surface_flux_position - species.flux_direction = self.flux_direction - species.flux_tmin = self.flux_tmin - species.flux_tmax = self.flux_tmax + species.add_new_group_attr(source_name, 'flux', density_scale) + species.add_new_group_attr(source_name, 'flux_normal_axis', self.flux_normal_axis) + species.add_new_group_attr(source_name, 'surface_flux_pos', self.surface_flux_position) + species.add_new_group_attr(source_name, 'flux_direction', self.flux_direction) + species.add_new_group_attr(source_name, 'flux_tmin', self.flux_tmin) + species.add_new_group_attr(source_name, 'flux_tmax', self.flux_tmax) # --- Use specific attributes for flux injection - species.injection_style = "nfluxpercell" + species.add_new_group_attr(source_name, 'injection_style', "nfluxpercell") assert (isinstance(layout, PseudoRandomLayout)), Exception('UniformFluxDistribution only supports the PseudoRandomLayout in WarpX') if self.gaussian_flux_momentum_distribution: - species.momentum_distribution_type = "gaussianflux" + species.add_new_group_attr(source_name, 'momentum_distribution_type', "gaussianflux") class UniformDistribution(picmistandard.PICMI_UniformDistribution, DensityDistributionBase): - def initialize_inputs(self, species_number, layout, species, density_scale): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) # --- Only constant density is supported by this class - species.profile = "constant" - species.density = self.density + species.add_new_group_attr(source_name, 'profile', "constant") + species.add_new_group_attr(source_name, 'density', self.density) if density_scale is not None: - species.density *= density_scale + species.add_new_group_attr(source_name, 'density', density_scale) class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDistributionBase): @@ -478,35 +493,35 @@ class AnalyticDistribution(picmistandard.PICMI_AnalyticDistribution, DensityDist def init(self, kw): self.momentum_spread_expressions = kw.pop('warpx_momentum_spread_expressions', [None, None, None]) - def initialize_inputs(self, species_number, layout, species, density_scale): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): self.set_mangle_dict() - self.set_species_attributes(species, layout) + self.set_species_attributes(species, layout, source_name) - species.profile = "parse_density_function" + species.add_new_group_attr(source_name, 'profile', "parse_density_function") expression = pywarpx.my_constants.mangle_expression(self.density_expression, self.mangle_dict) if density_scale is None: - species.__setattr__('density_function(x,y,z)', expression) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', expression) else: - species.__setattr__('density_function(x,y,z)', "{}*({})".format(density_scale, expression)) + species.add_new_group_attr(source_name, 'density_function(x,y,z)', "{}*({})".format(density_scale, expression)) class ParticleListDistribution(picmistandard.PICMI_ParticleListDistribution): def init(self, kw): pass - def initialize_inputs(self, species_number, layout, species, density_scale): + def distribution_initialize_inputs(self, species_number, layout, species, density_scale, source_name): - species.injection_style = "multipleparticles" - species.multiple_particles_pos_x = self.x - species.multiple_particles_pos_y = self.y - species.multiple_particles_pos_z = self.z - species.multiple_particles_ux = np.array(self.ux)/constants.c - species.multiple_particles_uy = np.array(self.uy)/constants.c - species.multiple_particles_uz = np.array(self.uz)/constants.c - species.multiple_particles_weight = self.weight + species.add_new_group_attr(source_name, 'injection_style', "multipleparticles") + species.add_new_group_attr(source_name, 'multiple_particles_pos_x', self.x) + species.add_new_group_attr(source_name, 'multiple_particles_pos_y', self.y) + species.add_new_group_attr(source_name, 'multiple_particles_pos_z', self.z) + species.add_new_group_attr(source_name, 'multiple_particles_ux', np.array(self.ux)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uy', np.array(self.uy)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_uz', np.array(self.uz)/constants.c) + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight) if density_scale is not None: - species.multiple_particles_weight = self.weight*density_scale + species.add_new_group_attr(source_name, 'multiple_particles_weight', self.weight*density_scale) class ParticleDistributionPlanarInjector(picmistandard.PICMI_ParticleDistributionPlanarInjector): @@ -525,7 +540,7 @@ def init(self, kw): class BinomialSmoother(picmistandard.PICMI_BinomialSmoother): - def initialize_inputs(self, solver): + def smoother_initialize_inputs(self, solver): pywarpx.warpx.use_filter = 1 pywarpx.warpx.use_filter_compensation = bool(np.all(self.compensation)) if self.n_pass is None: @@ -544,10 +559,10 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): """ This assumes that WarpX was compiled with USE_RZ = TRUE - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -581,6 +596,13 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_reflect_all_velocities: bool default=False Whether the sign of all of the particle velocities are changed upon reflection on a boundary, or only the velocity normal to the surface + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -598,6 +620,9 @@ def init(self, kw): self.potential_zmax = kw.pop('warpx_potential_hi_z', None) self.reflect_all_velocities = kw.pop('warpx_reflect_all_velocities', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -605,7 +630,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -638,6 +663,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -651,10 +679,10 @@ def initialize_inputs(self): class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -672,6 +700,13 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper longitudinal boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -686,6 +721,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -693,7 +731,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -715,6 +753,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[0]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -727,10 +768,10 @@ def initialize_inputs(self): class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -760,6 +801,13 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -776,6 +824,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -783,7 +834,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -810,6 +861,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[1]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -823,10 +877,10 @@ def initialize_inputs(self): class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters - --------- + ---------- warpx_max_grid_size: integer, default=32 Maximum block size in either direction @@ -868,6 +922,13 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_potential_hi_z: float, default=0. Electrostatic potential on the upper z boundary + + warpx_start_moving_window_step: int, default=0 + The timestep at which the moving window starts + + warpx_end_moving_window_step: int, default=-1 + The timestep at which the moving window ends. If -1, the moving window + will continue until the end of the simulation. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -886,6 +947,9 @@ def init(self, kw): self.potential_zmin = kw.pop('warpx_potential_lo_z', None) self.potential_zmax = kw.pop('warpx_potential_hi_z', None) + self.start_moving_window_step = kw.pop('warpx_start_moving_window_step', None) + self.end_moving_window_step = kw.pop('warpx_end_moving_window_step', None) + # Geometry # Set these as soon as the information is available # (since these are needed to determine which shared object to load) @@ -893,7 +957,7 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound - def initialize_inputs(self): + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells # Maximum allowable size of each subdomain in the problem domain; @@ -925,6 +989,9 @@ def initialize_inputs(self): pywarpx.warpx.moving_window_dir = 'z' pywarpx.warpx.moving_window_v = self.moving_window_velocity[2]/constants.c # in units of the speed of light + pywarpx.warpx.start_moving_window_step = self.start_moving_window_step + pywarpx.warpx.end_moving_window_step = self.end_moving_window_step + if self.refined_regions: assert len(self.refined_regions) == 1, Exception('WarpX only supports one refined region.') assert self.refined_regions[0][0] == 1, Exception('The one refined region can only be level 1') @@ -938,7 +1005,7 @@ def initialize_inputs(self): class ElectromagneticSolver(picmistandard.PICMI_ElectromagneticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -995,9 +1062,9 @@ def init(self, kw): self.pml_has_particles = kw.pop('warpx_pml_has_particles', None) self.do_pml_j_damping = kw.pop('warpx_do_pml_j_damping', None) - def initialize_inputs(self): + def solver_initialize_inputs(self): - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() pywarpx.warpx.pml_ncell = self.pml_ncell @@ -1033,7 +1100,7 @@ def initialize_inputs(self): pywarpx.warpx.cfl = self.cfl if self.source_smoother is not None: - self.source_smoother.initialize_inputs(self) + self.source_smoother.smoother_initialize_inputs(self) pywarpx.warpx.do_dive_cleaning = self.divE_cleaning pywarpx.warpx.do_divb_cleaning = self.divB_cleaning @@ -1070,9 +1137,14 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): substeps: int, default=100 Number of substeps to take when updating the B-field. + + Jx/y/z_external_function: str + Function of space and time specifying external (non-plasma) currents. """ def __init__(self, grid, Te=None, n0=None, gamma=None, - n_floor=None, plasma_resistivity=None, substeps=None, **kw): + n_floor=None, plasma_resistivity=None, substeps=None, + Jx_external_function=None, Jy_external_function=None, + Jz_external_function=None, **kw): self.grid = grid self.method = "hybrid" @@ -1084,11 +1156,26 @@ def __init__(self, grid, Te=None, n0=None, gamma=None, self.substeps = substeps + self.Jx_external_function = Jx_external_function + self.Jy_external_function = Jy_external_function + self.Jz_external_function = Jz_external_function + + # Handle keyword arguments used in expressions + self.user_defined_kw = {} + for k in list(kw.keys()): + self.user_defined_kw[k] = kw[k] + del kw[k] + self.handle_init(kw) - def initialize_inputs(self): + def solver_initialize_inputs(self): + + # Add the user defined keywords to my_constants + # The keywords are mangled if there is a conflicting variable already + # defined in my_constants with the same name but different value. + self.mangle_dict = pywarpx.my_constants.add_keywords(self.user_defined_kw) - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() pywarpx.algo.maxwell_solver = self.method @@ -1097,14 +1184,27 @@ def initialize_inputs(self): pywarpx.hybridpicmodel.gamma = self.gamma pywarpx.hybridpicmodel.n_floor = self.n_floor pywarpx.hybridpicmodel.__setattr__( - 'plasma_resistivity(rho)', self.plasma_resistivity + 'plasma_resistivity(rho,J)', + pywarpx.my_constants.mangle_expression(self.plasma_resistivity, self.mangle_dict) ) pywarpx.hybridpicmodel.substeps = self.substeps + pywarpx.hybridpicmodel.__setattr__( + 'Jx_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jx_external_function, self.mangle_dict) + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jy_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jy_external_function, self.mangle_dict) + ) + pywarpx.hybridpicmodel.__setattr__( + 'Jz_external_grid_function(x,y,z,t)', + pywarpx.my_constants.mangle_expression(self.Jz_external_function, self.mangle_dict) + ) class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1112,7 +1212,7 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): Whether to use the relativistic solver or lab frame solver warpx_absolute_tolerance: float, default=0. - Absolute tolerance on the lab fram solver + Absolute tolerance on the lab frame solver warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver @@ -1123,9 +1223,9 @@ def init(self, kw): self.self_fields_verbosity = kw.pop('warpx_self_fields_verbosity', None) self.magnetostatic = kw.pop('warpx_magnetostatic', False) - def initialize_inputs(self): + def solver_initialize_inputs(self): - self.grid.initialize_inputs() + self.grid.grid_initialize_inputs() if self.relativistic: pywarpx.warpx.do_electrostatic = 'relativistic' @@ -1147,7 +1247,7 @@ def initialize_inputs(self): class GaussianLaser(picmistandard.PICMI_GaussianLaser): - def initialize_inputs(self): + def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: self.name = 'laser{}'.format(self.laser_number) @@ -1172,7 +1272,7 @@ class AnalyticLaser(picmistandard.PICMI_AnalyticLaser): def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def laser_initialize_inputs(self): self.laser_number = len(pywarpx.lasers.names) + 1 if self.name is None: self.name = 'laser{}'.format(self.laser_number) @@ -1195,7 +1295,7 @@ def initialize_inputs(self): class LaserAntenna(picmistandard.PICMI_LaserAntenna): - def initialize_inputs(self, laser): + def laser_antenna_initialize_inputs(self, laser): laser.laser.position = self.position # This point is on the laser plane if ( self.normal_vector is not None @@ -1222,7 +1322,7 @@ def initialize_inputs(self, laser): class LoadInitialField(picmistandard.PICMI_LoadGriddedField): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): pywarpx.warpx.read_fields_from_path = self.read_fields_from_path if self.load_E: pywarpx.warpx.E_ext_grid_init_style = 'read_from_file' @@ -1235,7 +1335,7 @@ def init(self, kw): self.mangle_dict = None self.maxlevel_extEMfield_init = kw.pop('warpx_maxlevel_extEMfield_init', None); - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX pywarpx.warpx.maxlevel_extEMfield_init = self.maxlevel_extEMfield_init; @@ -1262,7 +1362,7 @@ def initialize_inputs(self): class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX if (self.Ex is not None or @@ -1282,7 +1382,7 @@ class AnalyticAppliedField(picmistandard.PICMI_AnalyticAppliedField): def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def applied_field_initialize_inputs(self): # Note that lower and upper_bound are not used by WarpX if self.mangle_dict is None: @@ -1308,7 +1408,7 @@ def initialize_inputs(self): class Mirror(picmistandard.PICMI_Mirror): - def initialize_inputs(self): + def applied_field_initialize_inputs(self): try: pywarpx.warpx.num_mirrors except AttributeError: @@ -1327,7 +1427,7 @@ class FieldIonization(picmistandard.PICMI_FieldIonization): """ WarpX only has ADK ionization model implemented. """ - def initialize_inputs(self): + def interaction_initialize_inputs(self): assert self.model == 'ADK', 'WarpX only has ADK ionization model implemented' self.ionized_species.species.do_field_ionization = 1 self.ionized_species.species.physical_element = self.ionized_species.particle_type @@ -1338,7 +1438,7 @@ def initialize_inputs(self): class CoulombCollisions(picmistandard.base._ClassWithInit): """ - Custom class to handle setup of binary Coulmb collisions in WarpX. If + Custom class to handle setup of binary Coulomb collisions in WarpX. If collision initialization is added to picmistandard this can be changed to inherit that functionality. @@ -1365,7 +1465,7 @@ def __init__(self, name, species, CoulombLog=None, ndt=None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'pairwisecoulomb' collision.species = [species.name for species in self.species] @@ -1422,7 +1522,7 @@ def __init__(self, name, species, background_density, self.handle_init(kw) - def initialize_inputs(self): + def collision_initialize_inputs(self): collision = pywarpx.Collisions.newcollision(self.name) collision.type = 'background_mcc' collision.species = self.species.name @@ -1446,6 +1546,49 @@ def initialize_inputs(self): collision.add_new_attr(process+'_'+key, val) +class DSMCCollisions(picmistandard.base._ClassWithInit): + """ + Custom class to handle setup of DSMC collisions in WarpX. If collision + initialization is added to picmistandard this can be changed to inherit + that functionality. + + Parameters + ---------- + name: string + Name of instance (used in the inputs file) + + species: species instance + The species involved in the collision + + scattering_processes: dictionary + The scattering process to use and any needed information + + ndt: integer, optional + The collisions will be applied every "ndt" steps. Must be 1 or larger. + """ + + def __init__(self, name, species, scattering_processes, ndt=None, **kw): + self.name = name + self.species = species + self.scattering_processes = scattering_processes + self.ndt = ndt + + self.handle_init(kw) + + def collision_initialize_inputs(self): + collision = pywarpx.Collisions.newcollision(self.name) + collision.type = 'dsmc' + collision.species = [species.name for species in self.species] + collision.ndt = self.ndt + + collision.scattering_processes = self.scattering_processes.keys() + for process, kw in self.scattering_processes.items(): + for key, val in kw.items(): + if key == 'species': + val = val.name + collision.add_new_attr(process+'_'+key, val) + + class EmbeddedBoundary(picmistandard.base._ClassWithInit): """ Custom class to handle set up of embedded boundaries specific to WarpX. @@ -1514,7 +1657,7 @@ def __init__(self, implicit_function=None, stl_file=None, stl_scale=None, stl_ce self.handle_init(kw) - def initialize_inputs(self, solver): + def embedded_boundary_initialize_inputs(self, solver): # Add the user defined keywords to my_constants # The keywords are mangled if there is a conflicting variable already @@ -1565,13 +1708,13 @@ class PlasmaLens(picmistandard.base._ClassWithInit): The field that is applied depends on the transverse position of the particle, (x,y) - - Ex = x*stengths_E + - Ex = x*strengths_E - - Ey = y*stengths_E + - Ey = y*strengths_E - - Bx = +y*stengths_B + - Bx = +y*strengths_B - - By = -x*stengths_B + - By = -x*strengths_B """ def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, **kw): @@ -1586,7 +1729,7 @@ def __init__(self, period, starts, lengths, strengths_E=None, strengths_B=None, self.handle_init(kw) - def initialize_inputs(self): + def applied_field_initialize_inputs(self): pywarpx.particles.E_ext_particle_init_style = 'repeated_plasma_lens' pywarpx.particles.B_ext_particle_init_style = 'repeated_plasma_lens' @@ -1599,7 +1742,7 @@ def initialize_inputs(self): class Simulation(picmistandard.PICMI_Simulation): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -1744,14 +1887,19 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_sort_idx_type: list of int, optional (default: 0 0 0) This controls the type of grid used to sort the particles when sort_particles_for_deposition is true. Possible values are: - idx_type = {0, 0, 0}: Sort particles to a cell centered grid, - idx_type = {1, 1, 1}: Sort particles to a node centered grid, - idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + + * idx_type = {0, 0, 0}: Sort particles to a cell centered grid, + * idx_type = {1, 1, 1}: Sort particles to a node centered grid, + * idx_type = {2, 2, 2}: Compromise between a cell and node centered grid. + In 2D (XZ and RZ), only the first two elements are read. In 1D, only the first element is read. warpx_sort_bin_size: list of int, optional (default 1 1 1) If `sort_intervals` is activated and `sort_particles_for_deposition` is false, particles are sorted in bins of `sort_bin_size` cells. In 2D, only the first two elements are read. + + warpx_used_inputs_file: string, optional + The name of the text file that the used input parameters is written to, """ # Set the C++ WarpX interface (see _libwarpx.LibWarpX) as an extension to @@ -1794,6 +1942,7 @@ def init(self, kw): self.sort_particles_for_deposition = kw.pop('warpx_sort_particles_for_deposition', None) self.sort_idx_type = kw.pop('warpx_sort_idx_type', None) self.sort_bin_size = kw.pop('warpx_sort_bin_size', None) + self.used_inputs_file = kw.pop('warpx_used_inputs_file', None) self.collisions = kw.pop('warpx_collisions', None) self.embedded_boundary = kw.pop('warpx_embedded_boundary', None) @@ -1846,6 +1995,7 @@ def initialize_inputs(self): pywarpx.warpx.do_multi_J_n_depositions = self.do_multi_J_n_depositions pywarpx.warpx.serialize_initial_conditions = self.serialize_initial_conditions pywarpx.warpx.random_seed = self.random_seed + pywarpx.warpx.used_inputs_file = self.used_inputs_file pywarpx.warpx.do_dynamic_scheduling = self.do_dynamic_scheduling @@ -1872,7 +2022,7 @@ def initialize_inputs(self): interpolation_order = particle_shape pywarpx.algo.particle_shape = interpolation_order - self.solver.initialize_inputs() + self.solver.solver_initialize_inputs() # Initialize warpx.field_centering_no and warpx.current_centering_no # if set by the user in the input (need to access grid info from solver attribute) @@ -1890,33 +2040,33 @@ def initialize_inputs(self): pywarpx.warpx.current_centering_noz = self.current_centering_order[-1] for i in range(len(self.species)): - self.species[i].initialize_inputs(self.layouts[i], - self.initialize_self_fields[i], - self.injection_plane_positions[i], - self.injection_plane_normal_vectors[i]) + self.species[i].species_initialize_inputs(self.layouts[i], + self.initialize_self_fields[i], + self.injection_plane_positions[i], + self.injection_plane_normal_vectors[i]) for interaction in self.interactions: assert(isinstance(interaction, FieldIonization)) - interaction.initialize_inputs() + interaction.interaction_initialize_inputs() if self.collisions is not None: pywarpx.collisions.collision_names = [] for collision in self.collisions: pywarpx.collisions.collision_names.append(collision.name) - collision.initialize_inputs() + collision.collision_initialize_inputs() if self.embedded_boundary is not None: - self.embedded_boundary.initialize_inputs(self.solver) + self.embedded_boundary.embedded_boundary_initialize_inputs(self.solver) for i in range(len(self.lasers)): - self.lasers[i].initialize_inputs() - self.laser_injection_methods[i].initialize_inputs(self.lasers[i]) + self.lasers[i].laser_initialize_inputs() + self.laser_injection_methods[i].laser_antenna_initialize_inputs(self.lasers[i]) for applied_field in self.applied_fields: - applied_field.initialize_inputs() + applied_field.applied_field_initialize_inputs() for diagnostic in self.diagnostics: - diagnostic.initialize_inputs() + diagnostic.diagnostic_initialize_inputs() if self.amr_restart: pywarpx.amr.restart = self.amr_restart @@ -2026,7 +2176,7 @@ class ParticleFieldDiagnostic: class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2074,7 +2224,7 @@ def init(self, kw): self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2187,7 +2337,7 @@ class Checkpoint(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up checkpointing of the simulation, allowing for later restarts - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2212,7 +2362,7 @@ def __init__(self, period = 1, write_dir = None, name = None, **kw): self.handle_init(kw) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2226,7 +2376,7 @@ def initialize_inputs(self): class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ for more information. + See `Input Parameters `__ for more information. Parameters ---------- @@ -2272,7 +2422,7 @@ def init(self, kw): self.mangle_dict = None - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2292,8 +2442,13 @@ def initialize_inputs(self): if self.data_list is not None: for dataname in self.data_list: if dataname == 'position': - # --- The positions are alway written out anyway - pass + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') elif dataname == 'momentum': variables.add('ux') variables.add('uy') @@ -2307,8 +2462,24 @@ def initialize_inputs(self): variables.add('Bx') variables.add('By') variables.add('Bz') - elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz']: - variables.add(dataname) + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. @@ -2319,7 +2490,7 @@ def initialize_inputs(self): if self.species is None: species_names = pywarpx.particles.species_names elif np.iterable(self.species): - species_names = [specie.name for specie in self.species] + species_names = [species.name for species in self.species] else: species_names = [self.species.name] @@ -2345,7 +2516,7 @@ def initialize_inputs(self): class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2387,7 +2558,7 @@ def init(self, kw): self.lower_bound = kw.pop('warpx_lower_bound', None) self.upper_bound = kw.pop('warpx_upper_bound', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2454,7 +2625,7 @@ def initialize_inputs(self): class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, WarpXDiagnosticBase): """ - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2486,7 +2657,7 @@ def init(self, kw): self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.buffer_size = kw.pop('warpx_buffer_size', None) - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() @@ -2516,25 +2687,46 @@ def initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. variables = set() - if self.data_list is not None: - for dataname in self.data_list: - if dataname == 'position': - # --- The positions are alway written out anyway - pass - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: variables.add(dataname) # --- Convert the set to a sorted list so that the order @@ -2546,9 +2738,9 @@ def initialize_inputs(self): if self.species is None: species_names = pywarpx.particles.species_names elif np.iterable(self.species): - species_names = [specie.name for specie in self.species] + species_names = [species.name for species in self.species] else: - species_names = [species.name] + species_names = [self.species.name] for name in species_names: diag = pywarpx.Bucket.Bucket(self.name + '.' + name, @@ -2560,7 +2752,7 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): """ Sets up a reduced diagnostic in the simulation. - See `Input Parameters `_ + See `Input Parameters `__ for more information. Parameters @@ -2616,7 +2808,7 @@ class ReducedDiagnostic(picmistandard.base._ClassWithInit, WarpXDiagnosticBase): reduction_type: {'Maximum', 'Minimum', or 'Integral'} For diagnostic type 'FieldReduction', the type of reduction - probe_geometry: {'Point', 'Line', 'Plane'}, defaut='Point' + probe_geometry: {'Point', 'Line', 'Plane'}, default='Point' For diagnostic type 'FieldProbe', the geometry of the probe integrate: bool, default=false @@ -2778,7 +2970,7 @@ def _handle_charge_on_eb(self, **kw): return kw - def initialize_inputs(self): + def diagnostic_initialize_inputs(self): self.add_diagnostic() diff --git a/Python/setup.py b/Python/setup.py index 62294fcaae2..3c6c0f605c0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '23.11', + version = '24.02', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 3110120b70a..01e4c687292 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552174367742e-08, - "Bt": 2380179.567792259, - "Bz": 2.4079075035536954e-08, + "Br": 1.8705552264208163e-08, + "Bt": 2380179.5677922587, + "Bz": 2.4079077116116535e-08, "Er": 497571119514841.25, - "Et": 7.048225282653208, - "Ez": 137058870936728.17, + "Et": 7.048225464720808, + "Ez": 137058870936728.28, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json new file mode 100644 index 00000000000..693faf43882 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -0,0 +1,96 @@ +{ + "lev=0": { + "Bx": 970573570349.2142, + "By": 970493693614.8823, + "Bz": 18743533.08373785, + "Ex": 2.9100804648817213e+20, + "Ey": 2.910188719414779e+20, + "Ez": 1.309938057448976e+17, + "rho_beam1": 7.969171294602906e+16, + "rho_beam2": 7.969079911431987e+16, + "rho_ele1": 234837475722889.97, + "rho_ele2": 277285508564124.12, + "rho_pos1": 229824415763789.62, + "rho_pos2": 286513388076434.4 + }, + "beam1": { + "particle_opticalDepthQSR": 104947.58821047074, + "particle_position_x": 0.0015001846430437182, + "particle_position_y": 0.0015002200838688795, + "particle_position_z": 0.004965556992831605, + "particle_momentum_x": 6.207861637949496e-15, + "particle_momentum_y": 6.16591876835476e-15, + "particle_momentum_z": 6.806755306915448e-12, + "particle_weight": 635859587.7915188 + }, + "beam2": { + "particle_opticalDepthQSR": 104180.42997257302, + "particle_position_x": 0.0015001132145418016, + "particle_position_y": 0.001500162338878405, + "particle_position_z": 0.004965528365042041, + "particle_momentum_x": 6.201791193788119e-15, + "particle_momentum_y": 6.188946480983165e-15, + "particle_momentum_z": 6.796967564632488e-12, + "particle_weight": 635853166.1373858 + }, + "ele1": { + "particle_opticalDepthQSR": 385.3959944370225, + "particle_position_x": 5.063867198487091e-06, + "particle_position_y": 5.323267522706671e-06, + "particle_position_z": 1.81974337459613e-05, + "particle_momentum_x": 5.220808698440275e-18, + "particle_momentum_y": 5.2833645967924376e-18, + "particle_momentum_z": 2.4960852396311436e-15, + "particle_weight": 1883777.3071500752 + }, + "ele2": { + "particle_opticalDepthQSR": 391.0032239817431, + "particle_position_x": 5.442400321293304e-06, + "particle_position_y": 5.3621200149906786e-06, + "particle_position_z": 1.716649458876156e-05, + "particle_momentum_x": 4.7056270741663116e-18, + "particle_momentum_y": 4.414438514376292e-18, + "particle_momentum_z": 2.3816287305827113e-15, + "particle_weight": 2255238.1204111334 + }, + "pho1": { + "particle_opticalDepthBW": 9912.332305353064, + "particle_position_x": 0.0001424857999095925, + "particle_position_y": 0.00014366766674682975, + "particle_position_z": 0.0004764478273708734, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 61671336.44071695 + }, + "pho2": { + "particle_opticalDepthBW": 10361.142481614075, + "particle_position_x": 0.00014718841339786984, + "particle_position_y": 0.00014538267635727008, + "particle_position_z": 0.00048768591004702896, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 61889405.34553001 + }, + "pos1": { + "particle_opticalDepthQSR": 343.0370605565725, + "particle_position_x": 5.41949101836001e-06, + "particle_position_y": 5.5292865311090835e-06, + "particle_position_z": 1.7063207973991194e-05, + "particle_momentum_x": 4.76704175252458e-18, + "particle_momentum_y": 5.049007427826035e-18, + "particle_momentum_z": 2.533578387785534e-15, + "particle_weight": 1821809.4309160584 + }, + "pos2": { + "particle_opticalDepthQSR": 390.0211924195479, + "particle_position_x": 5.053169658257247e-06, + "particle_position_y": 5.039302796539821e-06, + "particle_position_z": 1.8526669235892217e-05, + "particle_momentum_x": 5.755216173987019e-18, + "particle_momentum_y": 5.056618594918483e-18, + "particle_momentum_z": 2.52324147594341e-15, + "particle_weight": 2328558.5523011778 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 382c6010c1c..23231eab789 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -1,23 +1,8 @@ { - "deuterium_1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356807324363973e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240779905e+29 + "lev=0": { + "rho": 0.0 }, - "helium4_1": { + "neutron_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -26,19 +11,25 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "helium4_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.7635828326228758e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503, + "particle_weight": 1.0240001137713503e+30 }, - "neutron_1": { + "helium4_1": { "particle_momentum_x": 1.7519716491839538e-15, "particle_momentum_y": 1.7523289312260283e-15, "particle_momentum_z": 1.7480231586369996e-15, @@ -47,24 +38,33 @@ "particle_position_z": 325970.4138010667, "particle_weight": 4.421535775967805e-28 }, - "neutron_2": { - "particle_momentum_x": 1.5330942227771018e-15, - "particle_momentum_y": 1.5328473121602395e-15, - "particle_momentum_z": 1.549297051563983e-15, - "particle_position_x": 137011.89739173267, - "particle_position_y": 136605.24328988983, - "particle_position_z": 290143.4673994485, - "particle_weight": 5.756530048087129e+18 - }, - "tritium_1": { + "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, + "neutron_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.5535605933245691e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, + "helium4_2": { + "particle_momentum_x": 1.538345593941914e-15, + "particle_momentum_y": 1.536969107959402e-15, + "particle_momentum_z": 1.769309377628039e-15, + "particle_position_x": 137088.2335716813, + "particle_position_y": 136370.55273167047, + "particle_position_z": 290148.26756873244, + "particle_weight": 6.427760042646557e+18 + }, "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -72,6 +72,6 @@ "particle_position_x": 409798.0158217681, "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999424347e+29 + "particle_weight": 1.0239999999357226e+29 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json index 2d26136cc5e..4e9bde52a78 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504757806138167, - "Ey": 6.504757806138166, - "Ez": 6.501489620648247, - "rho": 2.60925680083338e-10 + "Ex": 6.504803536004636, + "Ey": 6.504803536004637, + "Ez": 6.504803536004637, + "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 1.0083594348819436e-23, - "particle_momentum_y": 1.0083594348819436e-23, - "particle_momentum_z": 1.0159789930728084e-23, - "particle_position_x": 518.4112403470117, - "particle_position_y": 518.4112403470117, - "particle_position_z": 520.2110302538686, + "particle_momentum_x": 1.0084159184928337e-23, + "particle_momentum_y": 1.008415918492834e-23, + "particle_momentum_z": 1.0084159184928338e-23, + "particle_position_x": 518.418176976446, + "particle_position_y": 518.4181769764461, + "particle_position_z": 518.418176976446, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json index d46a8cfcda4..648e805c938 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504586436288052, - "Ey": 6.504586436288051, - "Ez": 6.501896077968277, - "rho": 2.6092568008333797e-10 + "Ex": 6.504631859055906, + "Ey": 6.504631859055904, + "Ez": 6.504631859055904, + "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 9.881704085619681e-24, - "particle_momentum_y": 9.881704085619682e-24, - "particle_momentum_z": 9.969862126470096e-24, - "particle_position_x": 513.5147715983729, - "particle_position_y": 513.5147715983729, - "particle_position_z": 515.4187595183639, + "particle_momentum_x": 9.882423407883042e-24, + "particle_momentum_y": 9.882423407883041e-24, + "particle_momentum_z": 9.882423407883041e-24, + "particle_position_x": 513.522905933945, + "particle_position_y": 513.522905933945, + "particle_position_z": 513.522905933945, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json index 038ea5a6449..a94e8a4cd0b 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRZ.json @@ -1,17 +1,17 @@ { "lev=0": { - "Er": 0.11588932103656224, + "Er": 0.11588944816311227, "Et": 0.0, - "Ez": 0.11483033274834441, - "rho": 9.716216490680392e-12 + "Ez": 0.11492556391155323, + "rho": 9.71614990653583e-12 }, "electron": { - "particle_momentum_x": 4.54997005723341e-25, - "particle_momentum_y": 4.434893182689167e-25, - "particle_momentum_z": 6.989989085028144e-25, - "particle_position_x": 35.73642348640023, - "particle_position_y": 35.69851886414351, + "particle_momentum_x": 4.550172089304043e-25, + "particle_momentum_y": 4.435063230214811e-25, + "particle_momentum_z": 6.953391709067729e-25, + "particle_position_x": 35.73677703655427, + "particle_position_y": 35.61000307679542, "particle_theta": 823.0413054971611, "particle_weight": 6406.017620347038 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json b/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json new file mode 100644 index 00000000000..3451ac42a9f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/EnergyConservingThermalPlasma.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 14828575131693.746, + "Ey": 0.0, + "Ez": 15137612614715.773, + "jx": 5.572707185530917e+18, + "jy": 7.091118076558307e+18, + "jz": 6.668532755465409e+18 + }, + "protons": { + "particle_momentum_x": 3.187482462935171e-20, + "particle_momentum_y": 3.397549094666928e-20, + "particle_momentum_z": 3.300591612645518e-20, + "particle_position_x": 6.8052980975094785e-06, + "particle_position_y": 6.8011432612301085e-06, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 7.610483420803999e-22, + "particle_momentum_y": 8.024587481438078e-22, + "particle_momentum_z": 8.138108658606551e-22, + "particle_position_x": 6.8822971884226895e-06, + "particle_position_y": 6.8608357509110685e-06, + "particle_weight": 2823958719279159.5 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json new file mode 100644 index 00000000000..cd4120002be --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3730.0029376363264, + "By": 1593.5906541698305, + "Bz": 0.0, + "Ex": 797541065253.7858, + "Ey": 981292393404.0359, + "Ez": 3528134993266.091, + "divE": 2.0829069134855788e+21, + "jx": 7.639291641209293e+17, + "jy": 1.4113587963237038e+18, + "jz": 1.3506587033985085e+18, + "rho": 18442449008.581665 + }, + "protons": { + "particle_momentum_x": 5.231105747759245e-19, + "particle_momentum_y": 5.367982834807453e-19, + "particle_momentum_z": 5.253213507906386e-19, + "particle_position_x": 0.00010628272743703996, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196379551301037e-20, + "particle_momentum_y": 1.2271443795645239e-20, + "particle_momentum_z": 1.2277752539495415e-20, + "particle_position_x": 0.00010649569055433632, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json b/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json new file mode 100644 index 00000000000..d97eb04883f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 72730.70321925254, + "By": 89276.6097395453, + "Bz": 66911.00019634314, + "Ex": 92036838733000.64, + "Ey": 15583500940725.84, + "Ez": 89163420502164.97, + "divE": 8.998871921763322e+22, + "jx": 2.7748639888523993e+19, + "jy": 2.9501400595579277e+19, + "jz": 2.6976140199337787e+19, + "rho": 796777020986.2787 + }, + "protons": { + "particle_momentum_x": 2.0873315539608036e-17, + "particle_momentum_y": 2.0858882907322405e-17, + "particle_momentum_z": 2.0877345477243595e-17, + "particle_position_x": 0.004251275869323399, + "particle_position_y": 0.0042512738905209615, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 4.882673707817137e-19, + "particle_momentum_y": 4.879672470952739e-19, + "particle_momentum_z": 4.872329687213274e-19, + "particle_position_x": 0.004251641684258687, + "particle_position_y": 0.004251751978637919, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json new file mode 100644 index 00000000000..10f16df934f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4.json @@ -0,0 +1,29 @@ +{ + "electrons": { + "particle_momentum_x": 5.696397887862144e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.696397887862156e-20, + "particle_position_x": 0.6553600000001614, + "particle_position_y": 0.6553600000001614, + "particle_weight": 3200000000000000.5 + }, + "lev=0": { + "Ex": 3616987165668.129, + "Ey": 0.0, + "Ez": 3616987165667.9756, + "divE": 2.269072850514105e+18, + "jx": 1.0121499183289864e+16, + "jy": 0.0, + "jz": 1.0121499183289892e+16, + "part_per_cell": 131072.0, + "rho": 20090797.21028113 + }, + "positrons": { + "particle_momentum_x": 5.696397887862144e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.696397887862156e-20, + "particle_position_x": 0.6553600000001614, + "particle_position_y": 0.6553600000001614, + "particle_weight": 3200000000000000.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json index 4f8c01586cd..12f1f37dfd7 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_rz_psatd_current_correction.json @@ -1,30 +1,29 @@ { - "electrons": { - "particle_momentum_x": 3.059978628132309e-36, - "particle_momentum_y": 2.003371642665396e-20, - "particle_momentum_z": 2.785324114261674e-20, - "particle_position_x": 0.5290000114309512, - "particle_position_y": 0.5888000690527484, - "particle_theta": 92488.48772168353, - "particle_weight": 81147583679.15044 + "lev=0": { + "Bt": 6.633814882712189, + "Er": 478973170490.2528, + "Ez": 662610386678.0768, + "divE": 4.354742261016806e+17, + "jr": 893973344247781.6, + "jz": 1251505325555575.2, + "rho": 3855770.585538005 }, "ions": { - "particle_momentum_x": 1.4925140714935799e-36, - "particle_momentum_y": 1.4765381278752215e-21, - "particle_momentum_z": 1.9505526940463635e-21, - "particle_position_x": 0.5290000097195765, - "particle_position_y": 0.5887999999975628, + "particle_momentum_x": 9.303284812749536e-37, + "particle_momentum_y": 1.475435436402189e-21, + "particle_momentum_z": 1.9507023826482256e-21, + "particle_position_x": 0.5290000097194892, + "particle_position_y": 0.5887999999995832, "particle_theta": 92488.48772168353, "particle_weight": 81147583679.15044 }, - "lev=0": { - "Bt": 6.632261581203373, - "Er": 478937338832.87415, - "Ez": 662607529111.8489, - "divE": 4.355808435163123e+17, - "jr": 893968704248372.6, - "jz": 1251505252231323.8, - "rho": 3856714.5961512737 + "electrons": { + "particle_momentum_x": 2.5301631422412464e-36, + "particle_momentum_y": 2.0033141618079855e-20, + "particle_momentum_z": 2.785324108949854e-20, + "particle_position_x": 0.5290000111160095, + "particle_position_y": 0.5888000013680398, + "particle_theta": 92488.48772168353, + "particle_weight": 81147583679.15044 } } - diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index ee19b84c7af..0efcdaeca3c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 3.535284585563231e-19, - "particle_momentum_y": 4.403094613346061e-19, - "particle_momentum_z": 5.658013779496569e-17, - "particle_position_x": 0.008317876520240174, - "particle_position_y": 1.1704335094514386, - "particle_weight": 62415090744.60765 + "lev=0": { + "Bx": 4818955.480792835, + "By": 1752.8025402207227, + "Bz": 14516.21278267981, + "Ex": 2366115496505.249, + "Ey": 1446112025634143.0, + "Ez": 21864189507353.19, + "jx": 1996366349839211.5, + "jy": 5.312583827165288e+16, + "jz": 2.049135262445976e+16, + "rho": 68443961.71835628 }, "electrons": { - "particle_momentum_x": 2.2135959939611405e-23, - "particle_momentum_y": 2.822519730011994e-22, - "particle_momentum_z": 5.260625039372931e-22, - "particle_position_x": 0.010800577787577741, - "particle_position_y": 0.21115060628258137, + "particle_momentum_x": 2.2135945391227107e-23, + "particle_momentum_y": 2.8224559499572622e-22, + "particle_momentum_z": 5.260626010211241e-22, + "particle_position_x": 0.010800577787628053, + "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472008953412e-23, - "particle_momentum_y": 4.449200926395666e-22, - "particle_momentum_z": 5.768167708374496e-22, - "particle_position_x": 0.010800001678510793, - "particle_position_y": 0.21114947608115497, + "particle_momentum_x": 6.248472277235318e-23, + "particle_momentum_y": 4.449097689427615e-22, + "particle_momentum_z": 5.768168724780326e-22, + "particle_position_x": 0.010800001678510512, + "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, - "lev=0": { - "Bx": 4818965.813977506, - "By": 1752.781451346485, - "Bz": 14516.29947347909, - "Ex": 2366115950699.717, - "Ey": 1446115205306590.0, - "Ez": 21864183756771.12, - "jx": 1996366451911408.0, - "jy": 5.312642483130767e+16, - "jz": 2.049134468340688e+16, - "rho": 68443935.06252655 + "beam": { + "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_y": 4.363391839372122e-19, + "particle_momentum_z": 5.658606416951657e-17, + "particle_position_x": 0.008314723025211447, + "particle_position_y": 1.1704335743854242, + "particle_weight": 62415090744.60765 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json index 976a3bdee69..1602058dbdc 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationRZ.json @@ -1,64 +1,64 @@ { - "beam": { - "particle_momentum_x": 3.880115499392648e-20, - "particle_momentum_y": 5.07820548292446e-20, - "particle_momentum_z": 1.3503614921828295e-17, - "particle_position_x": 6.242131246165356e-05, - "particle_position_y": 0.0026764363296957524, - "particle_theta": 151.40798444325364, - "particle_weight": 6241509.074460764 + "lev=0": { + "Br": 104965.61239547684, + "Br_0_real": 0.2747311066289638, + "Br_1_imag": 104.10451752777048, + "Br_1_real": 104965.62478916143, + "Bt": 1296.2723277779253, + "Btheta_0_real": 1297.3296772773874, + "Btheta_1_imag": 105725.25398122355, + "Btheta_1_real": 141.2554810196911, + "Bz": 5076.74476238461, + "Bz_0_real": 0.4603819957306249, + "Bz_1_imag": 1.0073608129672305, + "Bz_1_real": 5076.632351216658, + "Er": 273570493775.55588, + "Er_0_real": 271974091013.7082, + "Er_1_imag": 39530787242966.72, + "Er_1_real": 42616886403.53069, + "Et": 39016542462571.68, + "Etheta_0_real": 112249482.93975669, + "Etheta_1_imag": 33602799006.874825, + "Etheta_1_real": 39016517454881.91, + "Ez": 511653064863.048, + "Ez_0_real": 496845125310.57947, + "Ez_1_imag": 1245709520854.488, + "Ez_1_real": 24849976994.356266, + "Jr_0_real": 1264417193503.146, + "Jr_1_imag": 2.3356633334993878e+17, + "Jr_1_real": 2272922066062.586, + "Jtheta_0_real": 475304056513.7001, + "Jtheta_1_imag": 1028929701334.8286, + "Jtheta_1_real": 2.1766379427377802e+17, + "Jz_0_real": 1832468408628522.8, + "Jz_1_imag": 556484600934089.9, + "Jz_1_real": 602703622358902.4, + "jr": 1749985661974.403, + "jt": 2.1766379408857264e+17, + "jz": 1954078684852864.2, + "rho": 39314210.931097575, + "rho_0_real": 38889615.839967206, + "rho_1_imag": 21546499.619336277, + "rho_1_real": 2012888.5658883736 }, "electrons": { - "particle_momentum_x": 1.886388623495423e-24, - "particle_momentum_y": 3.989021920324841e-22, - "particle_momentum_z": 1.2499427438675522e-23, + "particle_momentum_x": 1.3203417227476271e-24, + "particle_momentum_y": 4.0070625287284664e-22, + "particle_momentum_z": 1.2493391415020162e-23, "particle_orig_x": 0.026508328457558912, "particle_orig_z": 0.04789125000000001, - "particle_position_x": 0.041602500066712574, - "particle_position_y": 0.047891250456211995, - "particle_theta": 7325.121562757762, + "particle_position_x": 0.04160250006680718, + "particle_position_y": 0.04789125046517409, + "particle_theta": 7325.160426984388, "particle_weight": 813672305.532158 }, - "lev=0": { - "Br_0_real": 0.27473108145012964, - "Br_1_imag": 104.10424416504374, - "Br_1_real": 104965.62622212195, - "Btheta_0_real": 1297.3299824026299, - "Btheta_1_imag": 105725.25637121125, - "Btheta_1_real": 141.25524413452112, - "Br": 104965.6138283532, - "Bt": 1296.2727613183374, - "Bz": 5076.743764997268, - "Bz_0_real": 0.46038147543152824, - "Bz_1_imag": 1.007452397747621, - "Bz_1_real": 5076.631353934757, - "Er_0_real": 271974182110.8858, - "Er_1_imag": 39530787290253.98, - "Er_1_real": 42616765306.284, - "Etheta_0_real": 112249661.08828562, - "Etheta_1_imag": 33602739100.133934, - "Etheta_1_real": 39016517445019.95, - "Er": 273570655229.63535, - "Et": 39016542452492.57, - "Ez": 511653044133.8529, - "Ez_0_real": 496845145101.94775, - "Ez_1_imag": 1245709559726.1033, - "Ez_1_real": 24849961919.57713, - "Jr_0_real": 1264766566288.467, - "Jr_1_imag": 2.335663089152921e+17, - "Jr_1_real": 2273346021690.177, - "Jtheta_0_real": 475301266327.4687, - "Jtheta_1_imag": 1028946515774.2778, - "Jtheta_1_real": 2.176637922654668e+17, - "Jz_0_real": 1832469154489130.8, - "Jz_1_imag": 556484676782123.3, - "Jz_1_real": 602703174081284.9, - "jr": 1750423139263.8418, - "jt": 2.176637920803457e+17, - "jz": 1954078914516329.2, - "rho": 39314213.383921936, - "rho_0_real": 38889619.16177814, - "rho_1_imag": 21546507.958223887, - "rho_1_real": 2012888.0198535046 + "beam": { + "particle_momentum_x": 3.8798910618915136e-20, + "particle_momentum_y": 5.078287631744985e-20, + "particle_momentum_z": 1.3503610556163801e-17, + "particle_position_x": 6.242134237822313e-05, + "particle_position_y": 0.0026764363296840257, + "particle_theta": 151.40797316586125, + "particle_weight": 6241509.074460764 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index 6b30064365c..dd224516c5c 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497087553.4743837, - "By": 252364344.72213668, - "Bz": 16986109.62608196, - "Ex": 7.304419665394173e+16, - "Ey": 1.4583459400150573e+17, - "Ez": 2.261309965998215e+16, - "jx": 8.273582439836952e+17, - "jy": 2.1044181577417728e+18, - "jz": 2.84208498868846e+19, - "rho": 91637594416.50476 + "Bx": 497756265.44724107, + "By": 252197580.117894, + "Bz": 16988475.5444833, + "Ex": 7.299911104263803e+16, + "Ey": 1.460116488178734e+17, + "Ez": 2.26033100286114e+16, + "jx": 8.27644503757345e+17, + "jy": 2.1062078409959675e+18, + "jz": 2.8491727096438305e+19, + "rho": 91863764144.41415 }, "electrons": { - "particle_momentum_x": 5.922677819206403e-20, - "particle_momentum_y": 2.7778096010261295e-19, - "particle_momentum_z": 4.183846407463124e-19, - "particle_position_x": 0.0025865980636989934, - "particle_position_y": 0.002652673793955211, - "particle_position_z": 0.18411922027879662, - "particle_weight": 1731649095408.5505 + "particle_momentum_x": 5.76165226700654e-20, + "particle_momentum_y": 2.7567389504898156e-19, + "particle_momentum_z": 4.134562048099117e-19, + "particle_position_x": 0.0025269863605945427, + "particle_position_y": 0.0024538321295153346, + "particle_position_z": 0.17818421763751244, + "particle_weight": 1675789447169.5652 }, "beam": { - "particle_momentum_x": 1.1925686969448298e-17, - "particle_momentum_y": 2.570810132551278e-17, - "particle_momentum_z": 5.156190278524037e-14, - "particle_position_x": 0.0005608477913384702, - "particle_position_y": 0.0008431191912437461, - "particle_position_z": 2.9800048756186395, + "particle_momentum_x": 1.1926313055043134e-17, + "particle_momentum_y": 2.7056205218547404e-17, + "particle_momentum_z": 5.1562131494813424e-14, + "particle_position_x": 0.000560821518739447, + "particle_position_y": 0.000800729816549036, + "particle_position_z": 2.9800048650570097, "particle_weight": 62415.090744607616 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json index 793d8c11cb3..30eef0cedb7 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_single_precision_comms.json @@ -1,25 +1,25 @@ { + "lev=0": { + "Bx": 5863879.051452597, + "By": 2411.5124004699082, + "Bz": 116025.40178726945, + "Ex": 6267725953308.365, + "Ey": 1670763222566621.8, + "Ez": 104345977762699.77, + "jx": 555688154318027.4, + "jy": 1595897074125252.8, + "jz": 1045267363178542.9, + "rho": 2205684400.3678846 + }, "electrons": { "particle_initialenergy": 0.0, - "particle_momentum_x": 1.7921229236407606e-20, - "particle_momentum_y": 7.225825184653885e-20, - "particle_momentum_z": 4.231731488281653e-20, - "particle_position_x": 0.7139122621535502, - "particle_position_y": 0.7150340889556129, - "particle_position_z": 1.3175770602802896, "particle_regionofinterest": 1936.0, + "particle_position_x": 0.7139122620952513, + "particle_position_y": 0.7150340902640349, + "particle_position_z": 1.317577060220835, + "particle_momentum_x": 1.7921232385614662e-20, + "particle_momentum_y": 7.22582607277354e-20, + "particle_momentum_z": 4.231730764749506e-20, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5863879.02030842, - "By": 2411.501904737579, - "Bz": 116025.42462998998, - "Ex": 6267728094111.663, - "Ey": 1670763233105822.0, - "Ez": 104345989831222.4, - "jx": 555687912108453.8, - "jy": 1595896363359136.0, - "jz": 1045267552192496.5, - "rho": 2205684400.307701 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index d98aabd84e9..242cf00f0c3 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { "Br": 100278645.72225758, - "Bz": 2509008.1146699777, - "Er": 3.258880159474263, + "Bz": 2509008.1146699786, + "Er": 3.263343388037509, "Et": 3.0045297157982412e+16, - "Ez": 0.24010979707502123, - "jr": 50.13668665903749, - "jt": 4.305139762894391e+17, + "Ez": 0.2422526411295923, + "jr": 49.67802097760541, + "jt": 4.3051397628943917e+17, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json index 36b2bc66aed..81850192121 100644 --- a/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/LaserIonAcc2d.json @@ -1,34 +1,36 @@ { "electrons": { - "particle_momentum_x": 3.856058278845215e-19, + "particle_momentum_x": 3.8463518636930147e-19, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.633111056849423e-18, - "particle_position_x": 0.008155341094488198, - "particle_position_y": 0.03087362289643631, - "particle_weight": 2.6462205036771795e+17 + "particle_momentum_z": 1.6287398567676136e-18, + "particle_position_x": 0.008139313907538123, + "particle_position_y": 0.0, + "particle_position_z": 0.0308605164193468, + "particle_weight": 2.647983436428149e+17 }, "hydrogen": { - "particle_momentum_x": 2.237944099185357e-18, - "particle_momentum_z": 1.0744000122946915e-18, - "particle_orig_x": 0.007768349609375002, - "particle_orig_z": 0.035127407226562504, - "particle_position_x": 0.007767975241766832, - "particle_position_y": 0.03512562609400349, - "particle_weight": 2.68584931046923e+17 + "particle_momentum_x": 2.2575255298569516e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.0773391029868316e-18, + "particle_origX": 0.007763427734375002, + "particle_origZ": 0.0351975439453125, + "particle_position_x": 0.007763034484095496, + "particle_position_y": 0.0, + "particle_position_z": 0.035195761508116416, + "particle_weight": 2.6792730619156992e+17 }, "lev=0": { "Bx": 0.0, - "By": 11399442.711372176, + "By": 11411047.898351604, "Bz": 0.0, - "Ex": 2033057399920464.5, + "Ex": 2031641076084948.8, "Ey": 0.0, - "Ez": 339244862686685.9, - "jx": 1.6477362358927364e+19, + "Ez": 334777106874158.75, + "jx": 1.660205025572598e+19, "jy": 0.0, - "jz": 9.633815033523364e+18, - "rho": 68121082972.17873, - "rho_electrons": 17448735668294.348, - "rho_hydrogen": 17441797017887.941 + "jz": 9.545451466990307e+18, + "rho": 67237998613.860535, + "rho_electrons": 17449705826002.387, + "rho_hydrogen": 17441792654743.145 } -} - +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PlasmaMirror.json b/Regression/Checksum/benchmarks_json/PlasmaMirror.json index 9a686e78d56..861f42f2d74 100644 --- a/Regression/Checksum/benchmarks_json/PlasmaMirror.json +++ b/Regression/Checksum/benchmarks_json/PlasmaMirror.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 2.000457748960032e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.446670640741138e-17, - "particle_position_x": 0.37688793289418476, - "particle_position_y": 0.1714047288635171, - "particle_weight": 2.904173860599889e+18 - }, - "ions": { - "particle_momentum_x": 2.805690782833416e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 9.605283484426823e-17, - "particle_position_x": 0.3828171295926821, - "particle_position_y": 0.17237242725257168, - "particle_weight": 2.9212132379167017e+18 - }, "lev=0": { "Bx": 0.0, - "By": 75294856.83091721, + "By": 75293687.71417463, "Bz": 0.0, - "Ex": 2.081697252453305e+16, + "Ex": 2.0817302986029584e+16, "Ey": 0.0, - "Ez": 2.350511387513211e+16, - "jx": 5.496779405360711e+19, + "Ez": 2.350518662171605e+16, + "jx": 5.496864993586538e+19, "jy": 0.0, - "jz": 7.159348923050661e+19 + "jz": 7.159302467888838e+19 + }, + "ions": { + "particle_momentum_x": 2.8056907319288714e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 9.605283524169195e-17, + "particle_position_x": 0.382817129592504, + "particle_position_y": 0.1723724272525192, + "particle_weight": 2.9212132379167017e+18 + }, + "electrons": { + "particle_momentum_x": 2.000448978774197e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.446668081246458e-17, + "particle_position_x": 0.376887964713013, + "particle_position_y": 0.17140472970668535, + "particle_weight": 2.9041738605998894e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json new file mode 100644 index 00000000000..357e0823e39 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_3d.json @@ -0,0 +1,14 @@ +{ + "electron": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_weight": 0.0 + }, + "lev=0": { + "Ex": 0.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json new file mode 100644 index 00000000000..422f31548d2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Point_of_contact_EB_rz.json @@ -0,0 +1,15 @@ +{ + "lev=0": { + "Er": 0.0 + }, + "electron": { + "particle_position_x": 0.0, + "particle_position_y": 0.0, + "particle_position_z": 0.0, + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_weight": 0.0 + } +} + diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 979cffc0aa7..7f359f76e94 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -1,117 +1,117 @@ { - "alpha1": { - "particle_momentum_x": 4.714227948839551e-15, - "particle_momentum_y": 4.676078701959904e-15, - "particle_momentum_z": 4.676142266462326e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.940000093807323e-28 + "lev=0": { + "rho": 0.0 }, "alpha2": { - "particle_momentum_x": 4.081538674143691e-15, - "particle_momentum_y": 4.122349693618236e-15, - "particle_momentum_z": 4.201699189448955e-15, + "particle_momentum_x": 4.0815251136381854e-15, + "particle_momentum_y": 4.122335955939925e-15, + "particle_momentum_z": 4.201726051973563e-15, "particle_position_x": 410664.0477768091, "particle_position_y": 868193.1483606595, - "particle_weight": 2.959852107088932e+18 + "particle_weight": 2.947401325355292e+18 }, "alpha3": { - "particle_momentum_x": 4.934249084589244e-16, - "particle_momentum_y": 4.791039564460439e-16, - "particle_momentum_z": 4.698462130524534e-16, - "particle_position_x": 52309.63968324501, - "particle_position_y": 104322.0547863817, - "particle_weight": 1.491736282762796e+27 - }, - "alpha4": { - "particle_momentum_x": 2.342240913445794e-14, - "particle_momentum_y": 2.34177318709557e-14, - "particle_momentum_z": 2.353174771675334e-14, - "particle_position_x": 2457367.458278153, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.330088308142745e-14, - "particle_momentum_y": 2.344976927616691e-14, - "particle_momentum_z": 2.361827647381584e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 4914659.635379327, - "particle_weight": 3.839999999999998e-19 + "particle_momentum_x": 4.925995964082186e-16, + "particle_momentum_y": 4.780894181553476e-16, + "particle_momentum_z": 4.688680302272935e-16, + "particle_position_x": 52240.870920562535, + "particle_position_y": 103873.30862578771, + "particle_weight": 1.4784234459335885e+27 }, - "boron1": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, "particle_weight": 128.00000000000261 }, - "boron2": { + "alpha4": { + "particle_momentum_x": 2.3422553081132076e-14, + "particle_momentum_y": 2.3416860254545166e-14, + "particle_momentum_z": 2.3532380279126124e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373058, + "particle_weight": 384.0000000000002 + }, + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.015821768, - "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999016446e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 }, "boron3": { "particle_momentum_x": 9.277692671587846e-15, "particle_momentum_y": 9.268409636965691e-15, "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.166422465, - "particle_position_y": 8192499.706038672, - "particle_weight": 6.399502754572407e+30 + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399507192184686e+30 }, - "boron5": { + "boron2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 - }, - "lev=0": { - "rho": 0.0 + "particle_position_x": 409798.015821768, + "particle_position_y": 819270.9858143466, + "particle_weight": 1.2799999999017534e+28 }, - "proton1": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, "particle_weight": 128.00000000000261 }, - "proton2": { + "proton3": { + "particle_momentum_x": 1.684673285246867e-15, + "particle_momentum_y": 1.6827557106531144e-15, + "particle_momentum_z": 1.6802642612723895e-15, + "particle_position_x": 2457259.537951346, + "particle_position_y": 4914248.771185745, + "particle_weight": 1.2795071921846893e+30 + }, + "proton5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.371679089706304e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789516445e+28 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 }, - "proton3": { - "particle_momentum_x": 1.684214433067055e-15, - "particle_momentum_y": 1.682290595962152e-15, - "particle_momentum_z": 1.679806121802876e-15, - "particle_position_x": 2457258.407562205, - "particle_position_y": 4914234.891662369, - "particle_weight": 1.279502754572413e+30 + "alpha1": { + "particle_momentum_x": 4.714228774936194e-15, + "particle_momentum_y": 4.676079523757908e-15, + "particle_momentum_z": 4.676143086393951e-15, + "particle_position_x": 463834.9497057527, + "particle_position_y": 977592.6452214213, + "particle_weight": 2.9280275175431764e-28 }, - "proton4": { + "alpha5": { + "particle_momentum_x": 2.3300446729308862e-14, + "particle_momentum_y": 2.3449941485701153e-14, + "particle_momentum_z": 2.3618372938672865e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 4914659.635379325, + "particle_weight": 3.839999999999998e-19 + }, + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512789517532e+28 }, - "proton5": { + "boron5": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index b510fb10ff9..c3517c32b5b 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { - "alpha1": { - "particle_momentum_x": 4.659631949650719e-15, - "particle_momentum_y": 4.617308377231923e-15, - "particle_momentum_z": 4.617389330519975e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.9043802455895325e-27 - }, - "alpha2": { - "particle_momentum_x": 4.071678402460901e-15, - "particle_momentum_y": 4.1196410276880726e-15, - "particle_momentum_z": 4.181784710058622e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9261737121655108e+19 - }, - "alpha3": { - "particle_momentum_x": 4.853582081563116e-16, - "particle_momentum_y": 4.797297548379446e-16, - "particle_momentum_z": 4.759501504671248e-16, - "particle_position_x": 51342.57223394147, - "particle_position_y": 50913.6612890173, - "particle_position_z": 101306.34059008301, - "particle_weight": 1.0360147595748265e+28 - }, - "alpha4": { - "particle_momentum_x": 2.3373908106523292e-14, - "particle_momentum_y": 2.3439419725501755e-14, - "particle_momentum_z": 2.3559411845892612e-14, - "particle_position_x": 2457367.4582781526, - "particle_position_y": 2457512.044373058, - "particle_position_z": 4914475.776513073, - "particle_weight": 3072.000000000002 - }, - "alpha5": { - "particle_momentum_x": 2.333624251961068e-14, - "particle_momentum_y": 2.3455939695798916e-14, - "particle_momentum_z": 2.3574526216382372e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793237, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357942e+29 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119654661746806e+31 - }, - "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "lev=0": { - "rho": 0.0 - }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524242836246531e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 - }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3738870507189214e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240713489e+29 - }, - "proton3": { - "particle_momentum_x": 1.6843510515198723e-15, - "particle_momentum_y": 1.6823818968683368e-15, - "particle_momentum_z": 1.6799961456539653e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236546617468092e+31 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 - }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7581275870306353e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 - } + "lev=0": { + "rho": 0.0 + }, + "proton3": { + "particle_momentum_x": 1.684809640261926e-15, + "particle_momentum_y": 1.6828399494797829e-15, + "particle_momentum_z": 1.6804535487104095e-15, + "particle_position_x": 2457338.899376694, + "particle_position_y": 2457069.647393952, + "particle_position_z": 4914642.288898885, + "particle_weight": 1.0236580897818594e+31 + }, + "alpha5": { + "particle_momentum_x": 2.3336421176154494e-14, + "particle_momentum_y": 2.345583787205298e-14, + "particle_momentum_z": 2.3574859187827772e-14, + "particle_position_x": 2457556.857163842, + "particle_position_y": 2457059.6353793233, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119658089781855e+31 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "alpha4": { + "particle_momentum_x": 2.3374367002622734e-14, + "particle_momentum_y": 2.3439200330576348e-14, + "particle_momentum_z": 2.3559200621107925e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 2457512.044373057, + "particle_position_z": 4914475.776513074, + "particle_weight": 3072.000000000002 + }, + "alpha3": { + "particle_momentum_x": 4.857656981301579e-16, + "particle_momentum_y": 4.811177053359803e-16, + "particle_momentum_z": 4.766749863500389e-16, + "particle_position_x": 51282.9136300544, + "particle_position_y": 51045.39533309579, + "particle_position_z": 101578.82008857065, + "particle_weight": 1.0257306544231767e+28 + }, + "alpha2": { + "particle_momentum_x": 4.071664828713061e-15, + "particle_momentum_y": 4.119627260272026e-15, + "particle_momentum_z": 4.181812341443065e-15, + "particle_position_x": 405848.1200755021, + "particle_position_y": 408011.53191288817, + "particle_position_z": 866330.5923068221, + "particle_weight": 1.9180757241165136e+19 + }, + "proton2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.0227810240716198e+29 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 + }, + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 + }, + "alpha1": { + "particle_momentum_x": 4.659632773478301e-15, + "particle_momentum_y": 4.617309192789389e-15, + "particle_momentum_z": 4.6173901462667306e-15, + "particle_position_x": 453219.3436004627, + "particle_position_y": 457393.42114332493, + "particle_position_z": 970481.0580153911, + "particle_weight": 1.8966442168729786e-27 + }, + "boron2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.023999999936064e+29 + } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 0b0355b7cb6..08969db023e 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 5866866.518334419, + "By": 11175.108378013305, + "Bz": 116026.79919117832, + "Ex": 8178060925462.576, + "Ey": 1671615340558021.2, + "Ez": 106548534464553.72, + "jx": 555903248104584.56, + "jy": 1595980426561245.2, + "jz": 1366292265497407.5, + "rho": 2206755250.226452 + }, "beam": { - "particle_momentum_x": 4.707586336874016e-20, - "particle_momentum_y": 4.4850722108112576e-20, - "particle_momentum_z": 1.36054441043288e-17, - "particle_position_x": 4.058764306495361e-05, - "particle_position_y": 3.7888695722549883e-05, - "particle_position_z": 0.00019656701118308398, + "particle_momentum_x": 4.7075864374777216e-20, + "particle_momentum_y": 4.4875025179072e-20, + "particle_momentum_z": 1.3605444179112827e-17, + "particle_position_x": 4.058764328440382e-05, + "particle_position_y": 3.788866110052658e-05, + "particle_position_z": 0.00019656700944015084, "particle_weight": 6241509.074460764 }, "electrons": { - "particle_momentum_x": 1.7921232203945004e-20, - "particle_momentum_y": 7.225819894813053e-20, - "particle_momentum_z": 4.231725460173154e-20, - "particle_position_x": 0.7139122621161993, - "particle_position_y": 0.7150340887578637, - "particle_position_z": 1.3175770600690966, + "particle_momentum_x": 1.7921232210868553e-20, + "particle_momentum_y": 7.225819896136567e-20, + "particle_momentum_z": 4.2317254599358777e-20, + "particle_position_x": 0.713912262116188, + "particle_position_y": 0.7150340887578024, + "particle_position_z": 1.31757706006908, "particle_weight": 12926557617.187498 - }, - "lev=0": { - "Bx": 5866866.85492377, - "By": 11177.920546471447, - "Bz": 116026.93444649166, - "Ex": 8178548880638.266, - "Ey": 1671614207789070.8, - "Ez": 106548168484665.61, - "jx": 555903247963958.4, - "jy": 1595974150308405.2, - "jz": 1366292284444382.5, - "rho": 2206755250.226321 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json index 1c75b69dc9b..720148888e2 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAccelerationRZ.json @@ -1,62 +1,62 @@ { - "beam": { - "particle_momentum_x": 3.880109055649298e-20, - "particle_momentum_y": 5.0781930103830196e-20, - "particle_momentum_z": 1.3503608494680855e-17, - "particle_position_x": 6.242131236443886e-05, - "particle_position_y": 0.0026764363296979446, - "particle_theta": 151.4079870868123, - "particle_weight": 6241509.074460764 + "lev=0": { + "Br": 142258.01161290894, + "Br_0_real": 0.36356680791855334, + "Br_1_imag": 115.41915541351706, + "Br_1_real": 142258.0180774376, + "Bt": 1301.5691003159159, + "Btheta_0_real": 1299.8813681794945, + "Btheta_1_imag": 143318.04184449295, + "Btheta_1_real": 155.37799428725944, + "Bz": 5993.641173367065, + "Bz_0_real": 0.4737418612272665, + "Bz_1_imag": 1.1408917538312353, + "Bz_1_real": 5993.529100973486, + "Er": 278531478379.8872, + "Er_0_real": 276179480674.09875, + "Er_1_imag": 47911368149173.86, + "Er_1_real": 46900731766.219345, + "Et": 47328876349791.85, + "Etheta_0_real": 135868018.2930996, + "Etheta_1_imag": 36802947213.299324, + "Etheta_1_real": 47328835574237.43, + "Ez": 514006688162.3537, + "Ez_0_real": 499008057521.7594, + "Ez_1_imag": 1565161964565.9724, + "Ez_1_real": 28898941539.846146, + "Jr_0_real": 1458783813352.7048, + "Jr_1_imag": 2.335663323286376e+17, + "Jr_1_real": 2725720908168.6816, + "Jtheta_0_real": 499386783042.5499, + "Jtheta_1_imag": 1179203035377.2214, + "Jtheta_1_real": 2.1766371791828432e+17, + "Jz_0_real": 1832469778455967.8, + "Jz_1_imag": 621923937700173.0, + "Jz_1_real": 660910016274555.4, + "jr": 2108641839684.7007, + "jt": 2.1766371088502803e+17, + "jz": 1954236547059862.2, + "rho": 39480728.08547403, + "rho_0_real": 39055923.162689775, + "rho_1_imag": 21660762.696674515, + "rho_1_real": 2131498.5888879234 }, "electrons": { - "particle_momentum_x": 1.787201778017949e-24, - "particle_momentum_y": 3.9234822345143987e-22, - "particle_momentum_z": 1.0100062552925791e-23, - "particle_position_x": 0.041602500069929174, - "particle_position_y": 0.047891250477036906, - "particle_theta": 7325.1193688944695, + "particle_momentum_x": 1.237777638089911e-24, + "particle_momentum_y": 3.9465600403988994e-22, + "particle_momentum_z": 1.0097949593654349e-23, + "particle_position_x": 0.04160250006989259, + "particle_position_y": 0.047891250488567585, + "particle_theta": 7325.162679147719, "particle_weight": 813672305.532158 }, - "lev=0": { - "Br_0_real": 0.36356649135193925, - "Br_1_imag": 115.41886748920795, - "Br_1_real": 142258.01965536995, - "Btheta_0_real": 1299.8816721733124, - "Btheta_1_imag": 143318.04456658955, - "Btheta_1_real": 155.37774833024366, - "Br": 142258.01319076555, - "Bt": 1301.5695263567557, - "Bz": 5993.640969075834, - "Bz_0_real": 0.4737412745527051, - "Bz_1_imag": 1.1409956493384725, - "Bz_1_real": 5993.528898267216, - "Er_0_real": 276179575540.0639, - "Er_1_imag": 47911367858371.875, - "Er_1_real": 46900598536.03668, - "Etheta_0_real": 135868121.7945822, - "Etheta_1_imag": 36802874200.20133, - "Etheta_1_real": 47328835452079.97, - "Er": 278531658643.55005, - "Et": 47328876227481.125, - "Ez": 514006664374.9789, - "Ez_0_real": 499008075334.7451, - "Ez_1_imag": 1565161989236.6174, - "Ez_1_real": 28898922272.75169, - "Jr_0_real": 1459118139844.9216, - "Jr_1_imag": 2.3356630589200717e+17, - "Jr_1_real": 2726204346551.3022, - "Jtheta_0_real": 499384029970.2145, - "Jtheta_1_imag": 1179215927404.282, - "Jtheta_1_real": 2.17663715880068e+17, - "Jz_0_real": 1832470462501306.8, - "Jz_1_imag": 621924149855721.0, - "Jz_1_real": 660909646259030.1, - "jr": 2109207014985.481, - "jt": 2.1766370884715638e+17, - "jz": 1954236712029783.0, - "rho": 39480730.556067616, - "rho_0_real": 39055926.50167212, - "rho_1_imag": 21660770.34248945, - "rho_1_real": 2131498.060778751 + "beam": { + "particle_momentum_x": 3.8798818966213747e-20, + "particle_momentum_y": 5.078274625117952e-20, + "particle_momentum_z": 1.3503604116563051e-17, + "particle_position_x": 6.242134249270767e-05, + "particle_position_y": 0.0026764363296859625, + "particle_theta": 151.40797595010355, + "particle_weight": 6241509.074460764 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json new file mode 100644 index 00000000000..baaf29bec59 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json @@ -0,0 +1,34 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 11406516.737324182, + "Bz": 0.0, + "Ex": 2032785026975781.0, + "Ey": 0.0, + "Ez": 315269171271122.75, + "jx": 1.6224680921022521e+19, + "jy": 0.0, + "jz": 8.875043590548824e+18, + "rho": 61329928310.45101, + "rho_electrons": 17450329551552.684, + "rho_hydrogen": 17441799909767.852 + }, + "electrons": { + "particle_position_x": 0.008152777855374778, + "particle_position_y": 0.0, + "particle_position_z": 0.030714485915215906, + "particle_momentum_x": 3.594599848069649e-19, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.6339243089017708e-18, + "particle_weight": 2.6507336926909222e+17 + }, + "hydrogen": { + "particle_position_x": 0.008197892199782453, + "particle_position_y": 0.0, + "particle_position_z": 0.0365646600930625, + "particle_momentum_x": 2.2464037026384752e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.0873094324185116e-18, + "particle_weight": 2.703612070965676e+17 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json new file mode 100644 index 00000000000..0b38f78e6f9 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -0,0 +1,27 @@ +{ + "lev=0": { + "rho_electrons": 0.00443743609125863, + "rho_he_ions": 0.005198801518328451 + }, + "neutrals": { + "particle_momentum_x": 1.404700281648976e-19, + "particle_momentum_y": 1.4028127127618884e-19, + "particle_momentum_z": 1.4090901433394346e-19, + "particle_position_x": 1120.7727446759352, + "particle_weight": 6.4588e+19 + }, + "he_ions": { + "particle_momentum_x": 2.770386771117138e-19, + "particle_momentum_y": 2.7568040242914223e-19, + "particle_momentum_z": 3.619756966185903e-19, + "particle_position_x": 2200.683185473434, + "particle_weight": 17185500000000.002 + }, + "electrons": { + "particle_momentum_x": 3.5129762363657864e-20, + "particle_momentum_y": 3.5431134517510143e-20, + "particle_momentum_z": 1.2592093336142964e-19, + "particle_position_x": 2142.0662480700303, + "particle_weight": 14593699218750.002 + } +} diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 35aa2c07a60..31f426aa362 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554466, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3483401471475687e-07 + }, "electrons": { - "particle_momentum_x": 4.410992240916769e-18, + "particle_momentum_x": 4.4206237143449475e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.637306716013194e-18, - "particle_position_x": 0.1095060155214851, - "particle_position_y": 0.6411875726789248, - "particle_weight": 3.44453125e-10 + "particle_momentum_z": 2.6361297302081026e-18, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324019342538e-18, + "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887006212893e-23, - "particle_position_x": 0.03199998810579664, - "particle_position_y": 0.12800000462171646, + "particle_momentum_z": 3.644887053263054e-23, + "particle_position_x": 0.03200001189420337, + "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.355517269126987e-07 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json index 5e5369ed9c2..f5606cfee2f 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json @@ -1,17 +1,17 @@ { "lev=0": { - "Bx": 0.0846232979356012, - "By": 0.0748927888009307, + "Bx": 0.08460920626188952, + "By": 0.07485331147996604, "Bz": 256.0, - "Ex": 1960.2420822749025, - "Ey": 2368.794327568534, - "Ez": 4873.496044339846 + "Ex": 1954.3267519602405, + "Ey": 2363.4281756166347, + "Ez": 4873.508158589938 }, "ions": { - "particle_momentum_x": 1.615113301362171e-19, - "particle_momentum_y": 1.6152332353795267e-19, - "particle_momentum_z": 1.6134581585733078e-19, - "particle_position_x": 3678.484650863704, + "particle_momentum_x": 1.6151135948675135e-19, + "particle_momentum_y": 1.6152336151551518e-19, + "particle_momentum_z": 1.6134581268392543e-19, + "particle_position_x": 3678.484650899751, "particle_weight": 4.220251350277737e+21 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json index 870e4a82180..ca50554c394 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -1,12 +1,12 @@ { "lev=0": {}, "ions": { - "particle_momentum_x": 5.043873837579827e-17, - "particle_momentum_y": 5.04445113937769e-17, - "particle_momentum_z": 5.051930416996439e-17, - "particle_position_x": 143164.4268324243, - "particle_position_y": 143166.5184643828, - "particle_theta": 2573261.7450408577, + "particle_momentum_x": 5.043858996017125e-17, + "particle_momentum_y": 5.0444743275098145e-17, + "particle_momentum_z": 5.05192925609973e-17, + "particle_position_x": 143164.42694236583, + "particle_position_y": 143166.51848290052, + "particle_theta": 2573261.754119082, "particle_weight": 8.12868064536689e+18 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json index b0897c7372f..a602eaa68ee 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_ion_beam_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "Bx": 5.334191428317849, - "By": 5.304367780440314, + "Bx": 5.334151697582419, + "By": 5.304303632258862, "Bz": 256.0, - "Ex": 4.023743699806469e+05, - "Ey": 3.989324532762268e+05, - "Ez": 2.281344128344855e+04, - "jx": 1.525180463636894e+11, - "jy": 1.541352360760910e+11, - "jz": 3.182915979017075e+10 + "Ex": 402369.6682213072, + "Ey": 398930.6880468441, + "Ez": 22813.908607048776, + "jx": 152516944704.4641, + "jy": 154134670950.55667, + "jz": 31828054802.990257 }, "beam_ions": { - "particle_momentum_x": 1.267740609380299e-18, - "particle_momentum_y": 1.274270632846882e-18, - "particle_momentum_z": 1.391082122553233e-17, - "particle_position_x": 4.598149111137194e+03, + "particle_momentum_x": 1.2677390613057533e-18, + "particle_momentum_y": 1.2742687194492975e-18, + "particle_momentum_z": 1.3910821615724517e-17, + "particle_position_x": 4598.14909517292, "particle_weight": 2.1101256751388695e+20 }, "ions": { - "particle_momentum_x": 1.619717848597040e-18, - "particle_momentum_y": 1.619530182887040e-18, - "particle_momentum_z": 1.673190364768869e-18, - "particle_position_x": 9.193260427295945e+03, + "particle_momentum_x": 1.6197180437411583e-18, + "particle_momentum_y": 1.6195304119412495e-18, + "particle_momentum_z": 1.6731897345410398e-18, + "particle_position_x": 9193.260490317181, "particle_weight": 1.0550628375694317e+22 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json index f16c1a7a96a..e61206d19e3 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_landau_damping_2d.json @@ -1,19 +1,19 @@ { "lev=0": { "Bx": 0.0, - "By": 7.079680412579356e-06, + "By": 7.079679609748623e-06, "Bz": 0.0, - "Ex": 2726044.053666623, + "Ex": 2726044.0536666242, "Ey": 0.0, - "Ez": 4060168.641409589, - "jx": 177543428.8941277, + "Ez": 4060168.6414095857, + "jx": 177543428.8941278, "jy": 187432087.03814715, - "jz": 594259755.4658134 + "jz": 594259755.4658135 }, "ions": { "particle_momentum_x": 9.141594694084731e-17, "particle_momentum_y": 9.135546407258978e-17, - "particle_momentum_z": 9.137866220861256e-17, + "particle_momentum_z": 9.137866220861254e-17, "particle_position_x": 1197.3344862524336, "particle_position_y": 153269.17690371818, "particle_weight": 8.032598963696067e+16 diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json index c1788127e04..c7b467ad5f4 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json @@ -1,18 +1,18 @@ { "lev=0": { - "Bx": 1524.8789585917777, + "Bx": 1524.8789585554075, "By": 639.8314135126764, - "Bz": 7.47620755452188, - "Ex": 56501626.46458994, - "Ey": 75077196.84777552, - "Ez": 309549625.87520117 + "Bz": 7.476000395458839, + "Ex": 56499935.20497879, + "Ey": 75068107.08471295, + "Ez": 309535670.18599623 }, "ions": { - "particle_momentum_x": 7.14295522521196e-15, - "particle_momentum_y": 7.138078058060595e-15, - "particle_momentum_z": 7.14113444229561e-15, - "particle_position_x": 11170689.203149928, - "particle_position_y": 5585328.083195208, + "particle_momentum_x": 7.14295522755638e-15, + "particle_momentum_y": 7.1380780610425e-15, + "particle_momentum_z": 7.141134469227045e-15, + "particle_position_x": 11170689.203149632, + "particle_position_y": 5585328.083196239, "particle_weight": 9.036667901693183e+18 } } diff --git a/Regression/Checksum/benchmarks_json/RefinedInjection.json b/Regression/Checksum/benchmarks_json/RefinedInjection.json index 8be545f3887..43cc8e3072c 100644 --- a/Regression/Checksum/benchmarks_json/RefinedInjection.json +++ b/Regression/Checksum/benchmarks_json/RefinedInjection.json @@ -1,11 +1,27 @@ { - "beam": { - "particle_momentum_x": 4.5603842543853726e-20, - "particle_momentum_y": 4.191542263916011e-20, - "particle_momentum_z": 1.363947302075425e-17, - "particle_position_x": 4.672031050423284e-05, - "particle_position_y": 0.00024387147640533396, - "particle_weight": 12483018148921.525 + "lev=0": { + "Bx": 26338425.87401078, + "By": 160257.72293376247, + "Bz": 694883.1248007242, + "Ex": 56513906294889.77, + "Ey": 7806060318610865.0, + "Ez": 65816740142650.89, + "jx": 6873283794395257.0, + "jy": 4286847636945161.5, + "jz": 7506503405703726.0, + "rho": 278060758.9118884 + }, + "lev=1": { + "Bx": 41971351.93872842, + "By": 347834.73215718823, + "Bz": 1093501.9056767717, + "Ex": 136247073074384.92, + "Ey": 1.2435868091440666e+16, + "Ez": 92449081296414.34, + "jx": 1.6499002361434856e+16, + "jy": 1568934856430368.5, + "jz": 2.2190671920159212e+16, + "rho": 298322648.1883917 }, "electrons": { "particle_momentum_x": 4.599605609558194e-19, @@ -15,28 +31,12 @@ "particle_position_y": 0.3695766840020302, "particle_weight": 216500976562500.75 }, - "lev=0": { - "Bx": 26338374.3731115, - "By": 160257.72168459874, - "Bz": 694868.2285976049, - "Ex": 56513906233209.63, - "Ey": 7806074074131210.0, - "Ez": 65816739650512.98, - "jx": 6873283793354275.0, - "jy": 4287548168426209.0, - "jz": 7506499988368183.0, - "rho": 278060758.9118884 - }, - "lev=1": { - "Bx": 41971136.54330983, - "By": 347834.7278931723, - "Bz": 1093489.2683699469, - "Ex": 136247072743280.97, - "Ey": 1.2435931966910504e+16, - "Ez": 92449079327862.72, - "jx": 1.6499002357270928e+16, - "jy": 1571736982354559.0, - "jz": 2.219065825081704e+16, - "rho": 298322648.1883917 + "beam": { + "particle_momentum_x": 4.560382242970367e-20, + "particle_momentum_y": 4.2263074230751857e-20, + "particle_momentum_z": 1.3639472455787962e-17, + "particle_position_x": 4.672031008809431e-05, + "particle_position_y": 0.0002438714470005561, + "particle_weight": 12483018148921.525 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json new file mode 100644 index 00000000000..2c9859b037d --- /dev/null +++ b/Regression/Checksum/benchmarks_json/SemiImplicitPicard_1d.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 3559.0541122456157, + "By": 1685.942868827529, + "Bz": 0.0, + "Ex": 796541204346.5195, + "Ey": 961740397927.6577, + "Ez": 3528140764527.8877, + "divE": 2.0829159085076083e+21, + "jx": 7.683674095607745e+17, + "jy": 1.4132459141738875e+18, + "jz": 1.350650739310074e+18, + "rho": 18442528652.19583 + }, + "protons": { + "particle_momentum_x": 5.231109104020749e-19, + "particle_momentum_y": 5.367985047933474e-19, + "particle_momentum_z": 5.253213505843665e-19, + "particle_position_x": 0.00010628272743703473, + "particle_weight": 5.314093261582036e+22 + }, + "electrons": { + "particle_momentum_x": 1.196357181461066e-20, + "particle_momentum_y": 1.2271903040162696e-20, + "particle_momentum_z": 1.2277743615209627e-20, + "particle_position_x": 0.0001064956905491333, + "particle_weight": 5.314093261582036e+22 + } +} diff --git a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json index d62cd008670..8b03899369b 100644 --- a/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/comoving_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 6.875024052604206e-19, - "particle_momentum_y": 4.375444487966637e-19, - "particle_momentum_z": 6.432593176909762e-18, - "particle_position_x": 0.0012933598223665212, - "particle_position_y": 0.35872101659459354, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 7.058217306102507e-19, - "particle_momentum_y": 2.2042314720139012e-18, - "particle_momentum_z": 2.5305214299582585e-16, - "particle_position_x": 1.5006580327952221, - "particle_position_y": 16.454388306646475, - "particle_weight": 1.234867020725368e+18 + "lev=0": { + "Bx": 1118808.3686978193, + "By": 3248970.5506422943, + "Bz": 280612.7921641442, + "Ex": 975536649649286.1, + "Ey": 402861835403418.1, + "Ez": 159049265640492.28, + "jx": 2.9996888133195436e+16, + "jy": 8.866654944519546e+16, + "jz": 3.164008885453435e+17, + "rho": 1059988299.6088305 }, "ions": { - "particle_momentum_x": 1.6150569264030943e-18, - "particle_momentum_y": 2.2334239793537862e-18, - "particle_momentum_z": 4.279249530683602e-13, - "particle_position_x": 1.4883816864868449, - "particle_position_y": 16.452386504130835, + "particle_momentum_x": 1.6150513873065298e-18, + "particle_momentum_y": 2.233426695677123e-18, + "particle_momentum_z": 4.279249529993671e-13, + "particle_position_x": 1.4883816864183497, + "particle_position_y": 16.452386504127254, "particle_weight": 1.234867369440658e+18 }, - "lev=0": { - "Bx": 1118823.5061574595, - "By": 3248957.084212374, - "Bz": 280624.78102759377, - "Ex": 975532719061463.4, - "Ey": 402851207946673.1, - "Ez": 159049601047523.06, - "jx": 2.9997052529118484e+16, - "jy": 8.866616130905646e+16, - "jz": 3.163974567840701e+17, - "rho": 1059977703.1166165 + "electrons": { + "particle_momentum_x": 7.058167362825288e-19, + "particle_momentum_y": 2.204239326446281e-18, + "particle_momentum_z": 2.530521998715408e-16, + "particle_position_x": 1.5006581263609764, + "particle_position_y": 16.454388313398017, + "particle_weight": 1.234867020725368e+18 + }, + "beam": { + "particle_momentum_x": 6.869222298759882e-19, + "particle_momentum_y": 4.374719809060106e-19, + "particle_momentum_z": 6.4523206583503136e-18, + "particle_position_x": 0.001290816359726098, + "particle_position_y": 0.3586691102823157, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json b/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json new file mode 100644 index 00000000000..e7c4ec424dc --- /dev/null +++ b/Regression/Checksum/benchmarks_json/focusing_gaussian_beam.json @@ -0,0 +1,14 @@ +{ + "lev=0": { + "rho_beam1": 5637659324885.731 + }, + "beam1": { + "particle_momentum_x": 2.113335069047891e-14, + "particle_momentum_y": 5.653782702472731e-16, + "particle_momentum_z": 6.68023056405556e-11, + "particle_position_x": 0.5640698094900541, + "particle_position_y": 0.011947117763454793, + "particle_position_z": 239.4325333696459, + "particle_weight": 19999620000.0 + } +} diff --git a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json index cfd4bc83a85..dd56f8170a9 100644 --- a/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json +++ b/Regression/Checksum/benchmarks_json/galilean_2d_psatd_hybrid.json @@ -1,38 +1,38 @@ { - "beam": { - "particle_momentum_x": 7.005046981277183e-19, - "particle_momentum_y": 4.375209352729912e-19, - "particle_momentum_z": 6.175459223479959e-18, - "particle_position_x": 0.001602571749982411, - "particle_position_y": 0.35897924403061005, - "particle_weight": 3120754537230.3823 - }, - "electrons": { - "particle_momentum_x": 6.240989852249656e-19, - "particle_momentum_y": 1.5790301798509742e-18, - "particle_momentum_z": 2.5064352637575283e-16, - "particle_position_x": 1.501413662801212, - "particle_position_y": 16.523781706919216, - "particle_weight": 1.2372401466086835e+18 + "lev=0": { + "Bx": 1086729.9718613266, + "By": 2886554.482275311, + "Bz": 264259.55093734514, + "Ex": 867387781289915.2, + "Ey": 392666724461952.5, + "Ez": 146897592531660.03, + "jx": 2.702866174672266e+16, + "jy": 8.615938361747776e+16, + "jz": 2.7329155817806224e+17, + "rho": 915945723.7934376 }, "ions": { - "particle_momentum_x": 1.4394968631660078e-18, - "particle_momentum_y": 1.5967458174525868e-18, - "particle_momentum_z": 4.2873406586841774e-13, - "particle_position_x": 1.4911814217840753, - "particle_position_y": 16.52196497877435, + "particle_momentum_x": 1.4394902513923003e-18, + "particle_momentum_y": 1.5967629157922875e-18, + "particle_momentum_z": 4.287340658051679e-13, + "particle_position_x": 1.4911814217142487, + "particle_position_y": 16.521964978771, "particle_weight": 1.2372405194129536e+18 }, - "lev=0": { - "Bx": 1086731.2285090145, - "By": 2886537.258822082, - "Bz": 264280.80501631316, - "Ex": 867382744066315.0, - "Ey": 392669698675093.25, - "Ez": 146897949570681.72, - "jx": 2.7028852724044516e+16, - "jy": 8.615686606708202e+16, - "jz": 2.7328720244417827e+17, - "rho": 915931431.8889401 + "electrons": { + "particle_momentum_x": 6.240933687389075e-19, + "particle_momentum_y": 1.5790611427694247e-18, + "particle_momentum_z": 2.5064357834741096e-16, + "particle_position_x": 1.501413766926399, + "particle_position_y": 16.523781713952324, + "particle_weight": 1.2372401466086835e+18 + }, + "beam": { + "particle_momentum_x": 7.000932845220306e-19, + "particle_momentum_y": 4.374936866729326e-19, + "particle_momentum_z": 6.194468548032543e-18, + "particle_position_x": 0.0016030835496557787, + "particle_position_y": 0.3589262705964349, + "particle_weight": 3120754537230.3823 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json b/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json index 0fb51eb303a..79af45a3f44 100644 --- a/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json +++ b/Regression/Checksum/benchmarks_json/galilean_rz_psatd.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bt": 0.0001427751611710155, + "Er": 42705.92836687861, + "Et": 198517.8121599192, + "Ez": 4124.588455632591, + "divE": 50681.523267918594, + "jr": 7.07688551916584, + "jz": 469.93281767722846, + "rho": 4.488636594605648e-07 + }, "electrons": { - "particle_momentum_x": 7.188262495831869e-22, - "particle_momentum_y": 9.366670272998972e-24, - "particle_momentum_z": 8.903838605112867e-17, + "particle_momentum_x": 7.188262495831913e-22, + "particle_momentum_y": 9.366670273418532e-24, + "particle_momentum_z": 8.903838605112889e-17, "particle_position_x": 633733.1377069331, "particle_position_y": 7862152.008302979, "particle_theta": 51471.82060096203, "particle_weight": 1.0261080645329302e+20 }, "ions": { - "particle_momentum_x": 1.3139399113084718e-18, - "particle_momentum_y": 9.36530169181897e-24, + "particle_momentum_x": 1.313939911308472e-18, + "particle_momentum_y": 9.365301692233452e-24, "particle_momentum_z": 1.6348806629857118e-13, "particle_position_x": 633733.1365353089, "particle_position_y": 7862152.007285856, "particle_theta": 51471.6099169301, "particle_weight": 1.0261080645329302e+20 - }, - "lev=0": { - "Bt": 0.00014277516113528766, - "Er": 42705.928351478186, - "Et": 198517.8121599189, - "Ez": 4124.588451444761, - "divE": 50681.52326965637, - "jr": 7.0768855187909985, - "jz": 469.9328176825038, - "rho": 4.488636594800136e-07 - } -} + } + } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json index eddc22348f0..c412bebafb0 100644 --- a/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json +++ b/Regression/Checksum/benchmarks_json/galilean_rz_psatd_current_correction_psb.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bt": 0.0001546688348134161, + "Er": 46301.81902926975, + "Et": 198512.5412869523, + "Ez": 4269.5944463710985, + "divE": 56474.76102581718, + "jr": 5.550914333033371, + "jz": 149.7445144478365, + "rho": 5.000381408055163e-07 + }, "electrons": { - "particle_momentum_x": 7.188314912300234e-22, - "particle_momentum_y": 1.0394436737288987e-23, - "particle_momentum_z": 8.903837852634527e-17, - "particle_position_x": 633733.136187464, - "particle_position_y": 7862152.0078902915, - "particle_theta": 51471.82060104632, + "particle_momentum_x": 7.188316348440827e-22, + "particle_momentum_y": 1.0394552237458006e-23, + "particle_momentum_z": 8.903837851755449e-17, + "particle_position_x": 633733.136190813, + "particle_position_y": 7862152.007883845, + "particle_theta": 51471.82060123148, "particle_weight": 1.0261080645329302e+20 }, "ions": { - "particle_momentum_x": 1.3139399186198985e-18, - "particle_momentum_y": 1.0393368162404926e-23, - "particle_momentum_z": 1.6348806630609593e-13, - "particle_position_x": 633733.1365361367, - "particle_position_y": 7862152.007286081, - "particle_theta": 51471.609916929985, + "particle_momentum_x": 1.3139399189530324e-18, + "particle_momentum_y": 1.0393490021212813e-23, + "particle_momentum_z": 1.6348806630610471e-13, + "particle_position_x": 633733.1365361346, + "particle_position_y": 7862152.007286085, + "particle_theta": 51471.609916929934, "particle_weight": 1.0261080645329302e+20 - }, - "lev=0": { - "Bt": 0.00015467138145729178, - "Er": 46302.55104771345, - "Et": 198512.61435989104, - "Ez": 4269.500411629442, - "divE": 56473.85900924703, - "jr": 5.550691097640847, - "jz": 149.74203244538995, - "rho": 5.000301541815611e-07 } } diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index cb02eeba5b0..35b2db84a1a 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { + "lev=0": { + "Bx": 0.0, + "By": 18263123.342891, + "Bz": 0.0, + "Ex": 5472992180428804.0, + "Ey": 0.0, + "Ez": 922.5589707939612, + "jx": 12440856508004.96, + "jy": 0.0, + "jz": 78616.0000011086 + }, "electrons": { - "particle_momentum_x": 2.137334032699983e-17, + "particle_momentum_x": 2.1386770170623736e-17, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.729179869336611e-17, - "particle_position_x": 0.11036860836114047, - "particle_position_y": 1.7121959960266107, - "particle_weight": 3.075501431906104e-09 + "particle_momentum_z": 1.7287241262743654e-17, + "particle_position_x": 0.11064981928849067, + "particle_position_y": 1.7121826057017473, + "particle_weight": 3.0755014319061045e-09 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.63061972838759e-18, + "particle_momentum_x": 3.630619728387593e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, - "particle_position_x": 0.021440031727258925, + "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 - }, - "lev=0": { - "Bx": 0.0, - "By": 18263123.342890996, - "Bz": 0.0, - "Ex": 5472992180428804.0, - "Ey": 0.0, - "Ez": 922.6036749671132, - "jx": 12440856508004.96, - "jy": 0.0, - "jz": 78616.00000110848 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 3fafb968f29..82984141b59 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -1,30 +1,31 @@ { - "electrons": { - "particle_momentum_x": 4.407898469197755e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.642991174223682e-18, - "particle_position_x": 0.1095015206652257, - "particle_position_y": 0.6413864600981052, - "particle_weight": 3.443203125e-10 + "lev=0": { + "Bx": 0.0, + "By": 26296568.434868, + "Bz": 0.0, + "Ex": 7878103122971888.0, + "Ey": 0.0, + "Ez": 3027.995293554467, + "jx": 1.2111358330750162e+16, + "jy": 0.0, + "jz": 1.3575651149270143e-07 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.761324005206226e-18, + "particle_momentum_x": 1.7613240052056494e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.618696690270335e-23, - "particle_position_x": 0.03199998819289686, + "particle_momentum_z": 3.6186967375178554e-23, + "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 }, - "lev=0": { - "Bx": 0.0, - "By": 26296568.43487075, - "Bz": 0.0, - "Ex": 7878103122973058.0, - "Ey": 0.0, - "Ez": 3027.995293554496, - "jx": 1.2111358333819302e+16, - "jy": 0.0, - "jz": 1.346772722412443e-07 + "electrons": { + "particle_momentum_x": 4.428135211584547e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6627518442038486e-18, + "particle_orig_z": 0.43043207622230534, + "particle_position_x": 0.11023346490802505, + "particle_position_y": 0.6417814148540429, + "particle_weight": 3.44453125e-10 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json b/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json index 5ea6881b5ca..f30e773e7e0 100644 --- a/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json +++ b/Regression/Checksum/benchmarks_json/multi_J_rz_psatd.json @@ -1,13 +1,13 @@ { "lev=0": { - "Bt": 24080.41671546337, - "Er": 4536303784778.673, - "Ez": 4298815071343.0728, + "Bt": 24080.416715463354, + "Er": 4536303784778.672, + "Ez": 4298815071343.07, "jr": 361182004529074.8, - "jz": 1802215706551553.8, - "rho": 4884623.9573680265, + "jz": 1802215706551553.5, + "rho": 4884623.957368025, "rho_driver": 6288266.101815153, - "rho_plasma_e": 49568366.405371524, + "rho_plasma_e": 49568366.40537152, "rho_plasma_p": 50769182.21072973 }, "driver": { @@ -31,10 +31,10 @@ "plasma_e": { "particle_momentum_x": 6.655027717314839e-20, "particle_momentum_y": 6.730480164464723e-20, - "particle_momentum_z": 2.807381166958142e-20, + "particle_momentum_z": 2.8073811669581434e-20, "particle_position_x": 1.1423427658689635, "particle_position_y": 0.6140113094028803, "particle_theta": 20188.939948727297, "particle_weight": 1002457942911.3788 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json b/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json index 11c0a2a6108..dfe63ddc1c2 100644 --- a/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json +++ b/Regression/Checksum/benchmarks_json/particles_in_pml_2d_MR.json @@ -1,22 +1,22 @@ { "lev=0": { "Bx": 0.0, - "By": 3.578051588629885e-09, + "By": 1.959394057951299e-09, "Bz": 0.0, - "Ex": 1.9699822913484977, + "Ex": 1.2177330062543195, "Ey": 0.0, - "Ez": 0.5356004212513483, + "Ez": 0.28770171464461436, "jx": 0.0, "jy": 0.0, "jz": 0.0 }, "lev=1": { "Bx": 0.0, - "By": 2.7629151306453947e-09, + "By": 1.4999363537195907e-09, "Bz": 0.0, - "Ex": 2.4597363065789337, + "Ex": 1.5591272748392493, "Ey": 0.0, - "Ez": 0.45901250290130735, + "Ez": 0.29412053281248907, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/Checksum/benchmarks_json/restart.json b/Regression/Checksum/benchmarks_json/restart.json index 9ed02d0faad..d8aa0788997 100644 --- a/Regression/Checksum/benchmarks_json/restart.json +++ b/Regression/Checksum/benchmarks_json/restart.json @@ -1,12 +1,15 @@ { - "beam": { - "particle_momentum_x": 4.178482505909375e-19, - "particle_momentum_y": 4.56492260137707e-19, - "particle_momentum_z": 2.733972888170628e-17, - "particle_position_x": 0.0003995213395426269, - "particle_position_y": 0.0004148795632360405, - "particle_position_z": 1.9019426942919677, - "particle_weight": 3120754537.230381 + "lev=0": { + "Bx": 115361.74185283793, + "By": 242516.32397638055, + "Bz": 62078.1602361236, + "Ex": 119701543932824.69, + "Ey": 20420953176049.12, + "Ez": 53561498853753.336, + "jx": 2.1550943713704252e+16, + "jy": 328452566832977.2, + "jz": 4525238578330174.0, + "rho": 22112877.392750103 }, "driver": { "particle_momentum_x": 4.700436405078562e+21, @@ -26,34 +29,31 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 115361.85363382133, - "By": 242516.41036288123, - "Bz": 62078.12299476949, - "Ex": 119701563770527.33, - "Ey": 20420746160110.2, - "Ez": 53561518084714.88, - "jx": 2.155094272131813e+16, - "jy": 328438528912277.4, - "jz": 4525204093100645.0, - "rho": 22113102.212642767 + "beam": { + "particle_momentum_x": 4.178482505909375e-19, + "particle_momentum_y": 4.56492260137707e-19, + "particle_momentum_z": 2.733972888170628e-17, + "particle_position_x": 0.0003995213395426269, + "particle_position_y": 0.0004148795632360405, + "particle_position_z": 1.9019426942919677, + "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 2.6421618201433585e-19, - "particle_momentum_y": 1.3141433018232061e-20, - "particle_momentum_z": 2.6326692402728787e-17, - "particle_position_x": 0.49916042906025937, - "particle_position_y": 0.4991834641855549, - "particle_position_z": 0.45626372602154797, + "plasma_p": { + "particle_momentum_x": 2.6392309174575904e-19, + "particle_momentum_y": 5.301543151656233e-21, + "particle_momentum_z": 4.829600619848831e-14, + "particle_position_x": 0.4991250033821341, + "particle_position_y": 0.49912499879083855, + "particle_position_z": 0.4563845472726038, "particle_weight": 33067341227104.594 }, - "plasma_p": { - "particle_momentum_x": 2.639231047445633e-19, - "particle_momentum_y": 5.302522423284582e-21, - "particle_momentum_z": 4.829600619851657e-14, - "particle_position_x": 0.4991250033821394, - "particle_position_y": 0.49912499879083844, - "particle_position_z": 0.45638454727260425, + "plasma_e": { + "particle_momentum_x": 2.6421607111722265e-19, + "particle_momentum_y": 1.3141424232991868e-20, + "particle_momentum_z": 2.6326692443085376e-17, + "particle_position_x": 0.49916042919135184, + "particle_position_y": 0.49918346422905135, + "particle_position_z": 0.4562637258155577, "particle_weight": 33067341227104.594 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index 81df3c64c4c..d22fb5b57e7 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,4 +1,16 @@ { + "lev=0": { + "Bx": 116102.13010406512, + "By": 245467.5412026496, + "Bz": 65305.35287114741, + "Ex": 118565481099326.73, + "Ey": 18640100573642.96, + "Ez": 51777969821120.38, + "jx": 2.1177246490642464e+16, + "jy": 351544815339900.3, + "jz": 4573466562652713.0, + "rho": 22294533.587530266 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -26,34 +38,22 @@ "particle_position_z": 0.4899808854005956, "particle_weight": 6241509074.460762 }, - "lev=0": { - "Bx": 116102.28415732287, - "By": 245468.57438100342, - "Bz": 65305.38781838063, - "Ex": 118565257880521.4, - "Ey": 18640027127418.004, - "Ez": 51777964618631.914, - "jx": 2.1177240176169996e+16, - "jy": 351529204246424.9, - "jz": 4573419547907262.0, - "rho": 22294773.912022192 - }, "plasma_e": { - "particle_momentum_x": 2.0912370354448323e-19, - "particle_momentum_y": 1.3510736736168866e-20, - "particle_momentum_z": 2.633607024821822e-17, - "particle_position_x": 0.4991733347875822, - "particle_position_y": 0.49919828067421096, - "particle_position_z": 0.4562416755755165, + "particle_momentum_x": 2.0912364079892767e-19, + "particle_momentum_y": 1.3510804924903876e-20, + "particle_momentum_z": 2.6336070297356917e-17, + "particle_position_x": 0.49917333523960167, + "particle_position_y": 0.49919828074154127, + "particle_position_z": 0.45624167532314724, "particle_weight": 33067341227104.625 }, "plasma_p": { - "particle_momentum_x": 2.105303160755537e-19, - "particle_momentum_y": 3.3281768098792752e-21, - "particle_momentum_z": 4.829599953650214e-14, - "particle_position_x": 0.4991250029759827, - "particle_position_y": 0.49912499825258744, - "particle_position_z": 0.4563845470979585, + "particle_momentum_x": 2.1053025446547977e-19, + "particle_momentum_y": 3.3272838305044046e-21, + "particle_momentum_z": 4.829599953646669e-14, + "particle_position_x": 0.49912500297599893, + "particle_position_y": 0.49912499825258866, + "particle_position_z": 0.45638454709795806, "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 344495f8144..50971f0d45e 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,4 +1,25 @@ { + "lev=0": { + "Bx": 115622.1878941125, + "By": 243403.66715242947, + "Bz": 63602.66529020351, + "Ex": 117118934931289.56, + "Ey": 18448377440588.27, + "Ez": 50821967818817.24, + "jx": 2.1044522674691452e+16, + "jy": 329314843111847.94, + "jz": 4524787623275627.0, + "rho": 22128883.53068064 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "beam": { "particle_momentum_x": 4.178482505909375e-19, "particle_momentum_y": 4.56492260137707e-19, @@ -8,6 +29,24 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_e": { + "particle_momentum_x": 1.9905288525315664e-19, + "particle_momentum_y": 1.2685401564810352e-20, + "particle_momentum_z": 2.6334746597885943e-17, + "particle_position_x": 0.4991702885212871, + "particle_position_y": 0.4991949978513417, + "particle_position_z": 0.4562491611716817, + "particle_weight": 33067341227104.625 + }, + "plasma_p": { + "particle_momentum_x": 2.002897995377179e-19, + "particle_momentum_y": 2.87423099731012e-21, + "particle_momentum_z": 4.8296000849128737e-14, + "particle_position_x": 0.49912500259846493, + "particle_position_y": 0.4991249979476612, + "particle_position_z": 0.45638454712861487, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -16,44 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "lev=0": { - "Bx": 115622.38124168929, - "By": 243405.25199746038, - "Bz": 63602.713209078014, - "Ex": 117118723119070.72, - "Ey": 18448325716024.688, - "Ez": 50821980097172.29, - "jx": 2.104451599585796e+16, - "jy": 329301545706958.4, - "jz": 4524740919108259.0, - "rho": 22129124.901803844 - }, - "plasma_e": { - "particle_momentum_x": 1.9905296213155052e-19, - "particle_momentum_y": 1.2685326003720766e-20, - "particle_momentum_z": 2.6334746549552728e-17, - "particle_position_x": 0.4991702881158205, - "particle_position_y": 0.4991949977965001, - "particle_position_z": 0.45624916139143146, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.0028987523417718e-19, - "particle_momentum_y": 2.875037979316222e-21, - "particle_momentum_z": 4.8296000849162443e-14, - "particle_position_x": 0.49912500259844994, - "particle_position_y": 0.4991249979476608, - "particle_position_z": 0.4563845471286154, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json new file mode 100644 index 00000000000..d9f753e1df2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -0,0 +1,30 @@ +{ + "lev=0": { + "Er": 75713.05000099652, + "Ez": 75260.78239853957, + "phi": 55650.30604185804, + "rho": 1.4793075271598396e-06, + "rho_electrons": 6.506538129003745e-06, + "rho_protons": 6.98347902659172e-06 + }, + "electrons": { + "particle_position_x": 38158.7364935527, + "particle_position_y": 37779.25499255196, + "particle_position_z": 45010.371467374425, + "particle_momentum_x": 8.27307207197173e-20, + "particle_momentum_y": 8.264475255806164e-20, + "particle_momentum_z": 8.271327169054914e-20, + "particle_weight": 1140673608016.2212 + }, + "protons": { + "particle_position_x": 751407.372588289, + "particle_position_y": 751687.788498272, + "particle_position_z": 644420.0485148785, + "particle_momentum_x": 1.468116154656724e-17, + "particle_momentum_y": 1.4650318746367807e-17, + "particle_momentum_z": 1.1638654342620123e-17, + "particle_weight": 1175692137613.312 + } +} + + diff --git a/Regression/Checksum/checksum.py b/Regression/Checksum/checksum.py index cc3c53a24c2..454edfc2606 100644 --- a/Regression/Checksum/checksum.py +++ b/Regression/Checksum/checksum.py @@ -11,6 +11,8 @@ from benchmark import Benchmark import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c import yt yt.funcs.mylog.setLevel(50) @@ -20,19 +22,22 @@ class Checksum: """Class for checksum comparison of one test. """ - def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): + def __init__(self, test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Checksum constructor. - Store test_name and plotfile name, and compute checksum - from plotfile and store it in self.data. + Store test_name, output file name and format, compute checksum + from output file and store it in self.data. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to compare fields in the checksum. @@ -42,83 +47,142 @@ def __init__(self, test_name, plotfile, do_fields=True, do_particles=True): """ self.test_name = test_name - self.plotfile = plotfile - self.data = self.read_plotfile(do_fields=do_fields, - do_particles=do_particles) + self.output_file = output_file + self.output_format = output_format + self.data = self.read_output_file(do_fields=do_fields, + do_particles=do_particles) - def read_plotfile(self, do_fields=True, do_particles=True): + def read_output_file(self, do_fields=True, do_particles=True): """ - Get checksum from plotfile. - Read an AMReX plotfile with yt, compute 1 checksum per field and return - all checksums in a dictionary. + Get checksum from output file. + Read an AMReX plotfile with yt or an openPMD file with openPMD viewer, + compute 1 checksum per field and return all checksums in a dictionary. The checksum of quantity Q is max(abs(Q)). Parameters ---------- do_fields: bool, default=True - Whether to read fields from the plotfile. + Whether to read fields from the output file. do_particles: bool, default=True - Whether to read particles from the plotfile. + Whether to read particles from the output file. """ - ds = yt.load(self.plotfile) - # yt 4.0+ has rounding issues with our domain data: - # RuntimeError: yt attempted to read outside the boundaries - # of a non-periodic domain along dimension 0. - if 'force_periodicity' in dir(ds): ds.force_periodicity() - grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] - - # "fields"/"species" we remove: - # - nbody: added by yt by default, unused by us - species_list = set([item[0] for item in ds.field_list if - item[1][:9] == 'particle_' and - item[0] != 'all' and - item[0] != 'nbody']) - - data = {} - - # Compute checksum for field quantities - if do_fields: - for lev in range(ds.max_level+1): - data_lev = {} - lev_grids = [grid for grid in ds.index.grids - if grid.Level == lev] - # Warning: For now, we assume all levels are rectangular - LeftEdge = np.min( - np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) - all_data_level = ds.covering_grid( - level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) - for field in grid_fields: - Q = all_data_level[field].v.squeeze() - data_lev[field[1]] = np.sum(np.abs(Q)) - data['lev=' + str(lev)] = data_lev - - # Compute checksum for particle quantities - if do_particles: - ad = ds.all_data() - for species in species_list: - # properties we remove: - # - particle_cpu/id: they depend on the parallelism: MPI-ranks and - # on-node acceleration scheme, thus not portable - # and irrelevant for physics benchmarking - part_fields = [item[1] for item in ds.field_list - if item[0] == species and - item[1] != 'particle_cpu' and - item[1] != 'particle_id' - ] - data_species = {} - for field in part_fields: - Q = ad[(species, field)].v - data_species[field] = np.sum(np.abs(Q)) - data[species] = data_species + if self.output_format == 'plotfile': + ds = yt.load(self.output_file) + # yt 4.0+ has rounding issues with our domain data: + # RuntimeError: yt attempted to read outside the boundaries + # of a non-periodic domain along dimension 0. + if 'force_periodicity' in dir(ds): ds.force_periodicity() + grid_fields = [item for item in ds.field_list if item[0] == 'boxlib'] + + # "fields"/"species" we remove: + # - nbody: added by yt by default, unused by us + species_list = set([item[0] for item in ds.field_list if + item[1][:9] == 'particle_' and + item[0] != 'all' and + item[0] != 'nbody']) + + data = {} + + # Compute checksum for field quantities + if do_fields: + for lev in range(ds.max_level+1): + data_lev = {} + lev_grids = [grid for grid in ds.index.grids + if grid.Level == lev] + # Warning: For now, we assume all levels are rectangular + LeftEdge = np.min( + np.array([grid.LeftEdge.v for grid in lev_grids]), axis=0) + all_data_level = ds.covering_grid( + level=lev, left_edge=LeftEdge, dims=ds.domain_dimensions) + for field in grid_fields: + Q = all_data_level[field].v.squeeze() + data_lev[field[1]] = np.sum(np.abs(Q)) + data['lev=' + str(lev)] = data_lev + + # Compute checksum for particle quantities + if do_particles: + ad = ds.all_data() + for species in species_list: + # properties we remove: + # - particle_cpu/id: they depend on the parallelism: MPI-ranks and + # on-node acceleration scheme, thus not portable + # and irrelevant for physics benchmarking + part_fields = [item[1] for item in ds.field_list + if item[0] == species and + item[1] != 'particle_cpu' and + item[1] != 'particle_id' + ] + data_species = {} + for field in part_fields: + Q = ad[(species, field)].v + data_species[field] = np.sum(np.abs(Q)) + data[species] = data_species + + elif self.output_format == 'openpmd': + # Load time series + ts = OpenPMDTimeSeries(self.output_file) + data = {} + # Compute number of MR levels + # TODO This calculation of nlevels assumes that the last element + # of level_fields is by default on the highest MR level. + level_fields = [field for field in ts.avail_fields if 'lvl' in field] + nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) + # Compute checksum for field quantities + if do_fields: + for lev in range(nlevels+1): + # Create list of fields specific to level lev + grid_fields = [] + if lev == 0: + grid_fields = [field for field in ts.avail_fields if 'lvl' not in field] + else: + grid_fields = [field for field in ts.avail_fields if f'lvl{lev}' in field] + data_lev = {} + for field in grid_fields: + vector_components = ts.fields_metadata[field]['avail_components'] + if vector_components != []: + for coord in vector_components: + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1], coord=coord) + # key stores strings composed of field name and vector components + # (e.g., field='B' or field='B_lvl1' + coord='y' results in key='By') + key = field.replace(f'_lvl{lev}', '') + coord + data_lev[key] = np.sum(np.abs(Q)) + else: # scalar field + Q, info = ts.get_field(field=field, iteration=ts.iterations[-1]) + data_lev[field] = np.sum(np.abs(Q)) + data[f'lev={lev}'] = data_lev + # Compute checksum for particle quantities + if do_particles: + species_list = [] + if ts.avail_record_components is not None: + species_list = ts.avail_record_components.keys() + for species in species_list: + data_species = {} + part_fields = [item for item in ts.avail_record_components[species] + if item != 'id' and item != 'charge' and item != 'mass'] + # Convert the field name to the name used in plotfiles + for field in part_fields: + Q = ts.get_particle(var_list=[field], species=species, iteration=ts.iterations[-1]) + if field in ['x', 'y', 'z']: + field_name = 'particle_position_' + field + elif field in ['ux', 'uy', 'uz']: + field_name = 'particle_momentum_' + field[-1] + m, = ts.get_particle(['mass'], species=species, iteration=ts.iterations[-1]) + Q *= m*c + elif field in ['w']: + field_name = 'particle_weight' + else: + field_name = 'particle_' + field + data_species[field_name] = np.sum(np.abs(Q)) + data[species] = data_species return data def evaluate(self, rtol=1.e-9, atol=1.e-40): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert that they are equal. Almost all the body of this functions is for user-readable print statements. @@ -136,10 +200,10 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same outer keys (levels, species)? if (self.data.keys() != ref_benchmark.data.keys()): - print("ERROR: Benchmark and plotfile checksum " + print("ERROR: Benchmark and output file checksum " "have different outer keys:") print("Benchmark: %s" % ref_benchmark.data.keys()) - print("Plotfile : %s" % self.data.keys()) + print("Test file: %s" % self.data.keys()) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) print("----------------") @@ -148,12 +212,12 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): # Dictionaries have same inner keys (field and particle quantities)? for key1 in ref_benchmark.data.keys(): if (self.data[key1].keys() != ref_benchmark.data[key1].keys()): - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different inner keys:") print("Common outer keys: %s" % ref_benchmark.data.keys()) print("Benchmark inner keys in %s: %s" % (key1, ref_benchmark.data[key1].keys())) - print("Plotfile inner keys in %s: %s" + print("Test file inner keys in %s: %s" % (key1, self.data[key1].keys())) print("\n----------------\nNew file for " + self.test_name + ":") print(json.dumps(self.data, indent=2)) @@ -168,11 +232,11 @@ def evaluate(self, rtol=1.e-9, atol=1.e-40): ref_benchmark.data[key1][key2], rtol=rtol, atol=atol) if not passed: - print("ERROR: Benchmark and plotfile checksum have " + print("ERROR: Benchmark and output file checksum have " "different value for key [%s,%s]" % (key1, key2)) print("Benchmark: [%s,%s] %.15e" % (key1, key2, ref_benchmark.data[key1][key2])) - print("Plotfile : [%s,%s] %.15e" + print("Test file: [%s,%s] %.15e" % (key1, key2, self.data[key1][key2])) checksums_differ = True # Print absolute and relative error for each failing key diff --git a/Regression/Checksum/checksumAPI.py b/Regression/Checksum/checksumAPI.py index c45b6e233f4..cc13ceefa28 100755 --- a/Regression/Checksum/checksumAPI.py +++ b/Regression/Checksum/checksumAPI.py @@ -23,23 +23,23 @@ Example: add these lines to a WarpX CI Python analysis script to run the checksum test. > import checksumAPI - > checksumAPI.evaluate_checksum(test_name, plotfile) + > checksumAPI.evaluate_checksum(test_name, output_file, output_format) - As a script, to evaluate or to reset a benchmark: * Evaluate a benchmark. From a bash terminal: - $ ./checksumAPI.py --evaluate --plotfile \ - --test-name + $ ./checksumAPI.py --evaluate --output-file \ + --output-format 'plotfile' --test-name * Reset a benchmark. From a bash terminal: - $ ./checksumAPI.py --reset-benchmark --plotfile \ - --test-name + $ ./checksumAPI.py --reset-benchmark --output-file \ + --output-format 'openpmd' --test-name """ -def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, +def evaluate_checksum(test_name, output_file, output_format='plotfile', rtol=1.e-9, atol=1.e-40, do_fields=True, do_particles=True): """ - Compare plotfile checksum with benchmark. - Read checksum from input plotfile, read benchmark + Compare output file checksum with benchmark. + Read checksum from output file, read benchmark corresponding to test_name, and assert their equality. Parameters @@ -47,8 +47,11 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, test_name: string Name of test, as found between [] in .ini file. - plotfile : string - Plotfile from which the checksum is computed. + output_file : string + Output file from which the checksum is computed. + + output_format : string + Format of the output file (plotfile, openpmd). rtol: float, default=1.e-9 Relative tolerance for the comparison. @@ -62,24 +65,27 @@ def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40, do_particles: bool, default=True Whether to compare particles in the checksum. """ - test_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + test_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) test_checksum.evaluate(rtol=rtol, atol=atol) -def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): +def reset_benchmark(test_name, output_file, output_format='plotfile', do_fields=True, do_particles=True): """ Update the benchmark (overwrites reference json file). Overwrite value of benchmark corresponding to - test_name with checksum read from input plotfile. + test_name with checksum read from output file. Parameters ---------- test_name: string Name of test, as found between [] in .ini file. - plotfile: string - Plotfile from which the checksum is computed. + output_file: string + Output file from which the checksum is computed. + + output_format: string + Format of the output file (plotfile, openpmd). do_fields: bool, default=True Whether to write field checksums in the benchmark. @@ -87,34 +93,37 @@ def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True): do_particles: bool, default=True Whether to write particles checksums in the benchmark. """ - ref_checksum = Checksum(test_name, plotfile, do_fields=do_fields, - do_particles=do_particles) + ref_checksum = Checksum(test_name, output_file, output_format, + do_fields=do_fields, do_particles=do_particles) ref_benchmark = Benchmark(test_name, ref_checksum.data) ref_benchmark.reset() -def reset_all_benchmarks(path_to_all_plotfiles): +def reset_all_benchmarks(path_to_all_output_files, output_format='plotfile'): """ Update all benchmarks (overwrites reference json files) - found in path_to_all_plotfiles + found in path_to_all_output_files Parameters ---------- - path_to_all_plotfiles: string - Path to all plotfiles for which the benchmarks - are to be reset. The plotfiles should be named _plt, which is + path_to_all_output_files: string + Path to all output files for which the benchmarks + are to be reset. The output files should be named _plt, which is what regression_testing.regtests.py does, provided we're careful enough. + + output_format: string + Format of the output files (plotfile, openpmd). """ - # Get list of plotfiles in path_to_all_plotfiles - plotfile_list = glob.glob(path_to_all_plotfiles + '*_plt*[0-9]', - recursive=True) - plotfile_list.sort() + # Get list of output files in path_to_all_output_files + output_file_list = glob.glob(path_to_all_output_files + '*_plt*[0-9]', + recursive=True) + output_file_list.sort() - # Loop over plotfiles and reset the corresponding benchmark - for plotfile in plotfile_list: - test_name = os.path.split(plotfile)[1][:-9] - reset_benchmark(test_name, plotfile) + # Loop over output files and reset the corresponding benchmark + for output_file in output_file_list: + test_name = os.path.split(output_file)[1][:-9] + reset_benchmark(test_name, output_file, output_format) if __name__ == '__main__': @@ -131,11 +140,14 @@ def reset_all_benchmarks(path_to_all_plotfiles): required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, help='Name of the test (as in WarpX-tests.ini)') - parser.add_argument('--plotfile', dest='plotfile', type=str, default='', + parser.add_argument('--output-file', dest='output_file', type=str, default='', required='--evaluate' in sys.argv or '--reset-benchmark' in sys.argv, - help='Name of WarpX plotfile') - + help='Name of WarpX output file') + parser.add_argument('--output-format', dest='output_format', type=str, default='plotfile', + required='--evaluate' in sys.argv or + '--reset-benchmark' in sys.argv, + help='Format of the output file (plotfile, openpmd)') parser.add_argument('--skip-fields', dest='do_fields', default=True, action='store_false', help='If used, do not read/write field checksums') @@ -143,7 +155,7 @@ def reset_all_benchmarks(path_to_all_plotfiles): default=True, action='store_false', help='If used, do not read/write particle checksums') - # Fields and/or particles are read from plotfile/written to benchmark? + # Fields and/or particles are read from output file/written to benchmark? parser.add_argument('--rtol', dest='rtol', type=float, default=1.e-9, help='relative tolerance for comparison') @@ -155,26 +167,27 @@ def reset_all_benchmarks(path_to_all_plotfiles): parser.add_argument('--reset-all-benchmarks', dest='reset_all_benchmarks', action='store_true', default=False, help='Reset all benchmarks.') - parser.add_argument('--path-to-all-plotfiles', - dest='path_to_all_plotfiles', type=str, default='', + parser.add_argument('--path-to-all-output-files', + dest='path_to_all_output_files', type=str, default='', required='--reset-all-benchmarks' in sys.argv, - help='Directory containing all benchmark plotfiles, \ + help='Directory containing all benchmark output files, \ typically WarpX-benchmarks generated by \ regression_testing/regtest.py') args = parser.parse_args() if args.reset_benchmark: - reset_benchmark(args.test_name, args.plotfile, + reset_benchmark(args.test_name, args.output_file, args.output_format, do_fields=args.do_fields, do_particles=args.do_particles) if args.evaluate: - evaluate_checksum(args.test_name, args.plotfile, rtol=args.rtol, - atol=args.atol, do_fields=args.do_fields, - do_particles=args.do_particles) + evaluate_checksum(args.test_name, args.output_file, args.output_format, + rtol=args.rtol, atol=args.atol, + do_fields=args.do_fields, do_particles=args.do_particles) if args.reset_all_benchmarks: - # WARNING: this mode does not support skip-fields/particles - # and tolerances - reset_all_benchmarks(args.path_to_all_plotfiles) + if args.output_format == 'openpmd': + sys.exit('Option --reset-all-benchmarks does not work with openPMD format') + # WARNING: this mode does not support skip-fields/particles and tolerances + reset_all_benchmarks(args.path_to_all_output_files, args.output_format) diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index e11976205d1..ef25ee0dfbe 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 9e35dc19489dc5d312e92781cb0471d282cf8370 +branch = 2ecafcff40132f56eb2b494e1a374684ff97117a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index c8e9a7e1a90..b25d41db5d2 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 9e35dc19489dc5d312e92781cb0471d282cf8370 +branch = 2ecafcff40132f56eb2b494e1a374684ff97117a [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -1434,6 +1434,25 @@ particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png +[Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] +buildDir = . +inputFile = Examples/Tests/langmuir/inputs_2d +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 +dim = 2 +addToCompileString = USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons positrons +analysisRoutine = Examples/Tests/langmuir/analysis_2d.py +analysisOutputImage = langmuir_multi_2d_analysis.png + [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d @@ -1812,7 +1831,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserAcceleration_1d] buildDir = . @@ -1973,7 +1993,8 @@ compileTest = 0 doVis = 0 compareParticles = 1 particleTypes = electrons -analysisRoutine = Examples/analysis_default_regression.py +outputFile = LaserAcceleration_single_precision_comms_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserInjection] buildDir = . @@ -2162,8 +2183,9 @@ doVis = 0 [LaserIonAcc2d] buildDir = . -inputFile = Examples/Physics_applications/laser_ion/inputs -runtime_params = amr.n_cell=384 512 max_step=100 +inputFile = Examples/Physics_applications/laser_ion/inputs_2d +outputFile = LaserIonAcc2d_plt +runtime_params = dim = 2 addToCompileString = USE_OPENPMD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON @@ -2174,7 +2196,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -analysisRoutine = Examples/analysis_default_regression.py +analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserOnFine] buildDir = . @@ -2212,11 +2234,12 @@ analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py [LoadExternalFieldRZ] buildDir = . inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON -restartTest = 0 +restartTest = 1 +restartFileNum = 150 useMPI = 1 numprocs = 1 useOMP = 1 @@ -2502,7 +2525,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 0 -analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py +analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml_2dmr.py [particles_in_pml_3d_MR] buildDir = . @@ -3076,6 +3099,25 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py +[Python_dsmc_1d] +buildDir = . +inputFile = Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_1d.py --test --dsmc +dim = 1 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE +cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON -DWarpX_QED=OFF +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py + [Python_ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/PICMI_inputs_3d.py @@ -3352,6 +3394,25 @@ compareParticles = 1 particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py +[Python_LaserIonAcc2d] +buildDir = . +inputFile = Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +outputFile= diags/Python_LaserIonAcc2d_plt +runtime_params = +customRunCmd = python3 PICMI_inputs_2d.py +dim = 2 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/analysis_default_openpmd_regression.py + [Python_LoadExternalField3D] buildDir = . inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d.py @@ -4219,6 +4280,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py +[scraping_filter] +buildDir = . +inputFile = Examples/Tests/scraping/inputs_rz_filter +runtime_params = warpx.abort_on_warning_threshold = medium +dim = 2 +addToCompileString = USE_EB=TRUE USE_RZ=TRUE USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py + [silver_mueller_1d] buildDir = . inputFile = Examples/Tests/silver_mueller/inputs_1d @@ -4441,3 +4519,165 @@ compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py + +[BeamBeamCollision] +buildDir = . +inputFile = Examples/Physics_applications/beam-beam_collision/inputs +runtime_params = warpx.abort_on_warning_threshold=high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +outputFile = BeamBeamCollision_plt +analysisRoutine = Examples/analysis_default_openpmd_regression.py + +[spacecraft_charging] +buildDir = . +inputFile = Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +runtime_params = +customRunCmd = python3 PICMI_inputs_rz.py +dim = 2 +addToCompileString = USE_PYTHON_MAIN=TRUE USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS="RZ" -DWarpX_EB=ON -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = electrons protons +outputFile = spacecraft_charging_plt +analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py +analysisOutputImage = min_phi_analysis.png + +[Point_of_contact_EB_3d] +buildDir = . +inputFile = Examples/Tests/point_of_contact_EB/inputs_3d +runtime_params = +dim = 3 +addToCompileString = USE_EB=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_EB=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +particleTypes = electrons +outputFile = Point_of_contact_EB_3d_plt +analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py + +[Point_of_contact_EB_rz] +buildDir = . +inputFile = Examples/Tests/point_of_contact_EB/inputs_rz +runtime_params = +dim = 2 +addToCompileString = USE_RZ=TRUE USE_EB=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +particleTypes = electrons +outputFile = Point_of_contact_EB_rz_plt +analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py + +[ImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[ImplicitPicard_VandB_2d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_vandb_2d +runtime_params = warpx.abort_on_warning_threshold=high +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_2d.py + +[SemiImplicitPicard_1d] +buildDir = . +inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit +runtime_params = warpx.abort_on_warning_threshold=high +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_1d.py + +[EnergyConservingThermalPlasma] +buildDir = . +inputFile = Examples/Tests/energy_conserving_thermal_plasma/inputs_2d_electrostatic +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py + +[focusing_gaussian_beam] +buildDir = . +inputFile = Examples/Tests/gaussian_beam/inputs_focusing_beam +runtime_params = +dim = 3 +addToCompileString = USE_OPENPMD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +doComparison = 0 +analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index 1fa6b4c9a0a..d52d05b1139 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -7,7 +7,7 @@ import os # This script modifies `WarpX-test.ini` (which is used for nightly builds) -# and creates the file `ci-test.ini` (which is used for continous +# and creates the file `ci-test.ini` (which is used for continuous # integration) # The subtests that are selected are controlled by WARPX_TEST_DIM # The architecture (CPU/GPU) is selected by WARPX_TEST_ARCH diff --git a/Source/AcceleratorLattice/AcceleratorLattice.cpp b/Source/AcceleratorLattice/AcceleratorLattice.cpp index e05167c261d..8dc0983d622 100644 --- a/Source/AcceleratorLattice/AcceleratorLattice.cpp +++ b/Source/AcceleratorLattice/AcceleratorLattice.cpp @@ -88,8 +88,9 @@ AcceleratorLattice::InitElementFinder (int const lev, amrex::BoxArray const & ba } void -AcceleratorLattice::UpdateElementFinder (int const lev) -{ +AcceleratorLattice::UpdateElementFinder (int const lev) // NOLINT(readability-make-member-function-const) +{ // Techniquely clang-tidy is correct because + // m_element_finder is unique_ptr, not const*. if (m_lattice_defined) { for (amrex::MFIter mfi(*m_element_finder); mfi.isValid(); ++mfi) { diff --git a/Source/AcceleratorLattice/LatticeElementFinder.H b/Source/AcceleratorLattice/LatticeElementFinder.H index 24cdb691bb0..6773ed56a65 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.H +++ b/Source/AcceleratorLattice/LatticeElementFinder.H @@ -72,8 +72,8 @@ struct LatticeElementFinder * @param[in] a_offset particle index offset needed to access particle info * @param[in] accelerator_lattice a reference to the accelerator lattice at the refinement level */ - LatticeElementFinderDevice GetFinderDeviceInstance (WarpXParIter const& a_pti, int a_offset, - AcceleratorLattice const& accelerator_lattice); + [[nodiscard]] LatticeElementFinderDevice GetFinderDeviceInstance ( + WarpXParIter const& a_pti, int a_offset, AcceleratorLattice const& accelerator_lattice) const; /* The index lookup tables for each lattice element type */ amrex::Gpu::DeviceVector d_quad_indices; @@ -89,52 +89,7 @@ struct LatticeElementFinder */ void setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, amrex::Gpu::DeviceVector const & ze, - amrex::Gpu::DeviceVector & indices) - { - - using namespace amrex::literals; - - const auto nelements = static_cast(zs.size()); - amrex::ParticleReal const * zs_arr = zs.data(); - amrex::ParticleReal const * ze_arr = ze.data(); - int * indices_arr = indices.data(); - - amrex::Real const zmin = m_zmin; - amrex::Real const dz = m_dz; - - amrex::ParticleReal const gamma_boost = m_gamma_boost; - amrex::ParticleReal const uz_boost = m_uz_boost; - amrex::Real const time = m_time; - - amrex::ParallelFor( m_nz, - [=] AMREX_GPU_DEVICE (int iz) { - - // Get the location of the grid node - amrex::Real z_node = zmin + iz*dz; - - if (gamma_boost > 1._prt) { - // Transform to lab frame - z_node = gamma_boost*z_node + uz_boost*time; - } - - // Find the index to the element that is closest to the grid cell. - // For now, this assumes that there is no overlap among elements of the same type. - for (int ie = 0 ; ie < nelements ; ie++) { - // Find the mid points between element ie and the ones before and after it. - // The first and last element need special handling. - const amrex::ParticleReal zcenter_left = (ie == 0)? - (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); - const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? - (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); - if (zcenter_left <= z_node && z_node < zcenter_right) { - indices_arr[iz] = ie; - } - - } - } - ); - } - + amrex::Gpu::DeviceVector & indices) const; }; /** diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index 62ebdcc1f4f..ba2e51715fb 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -78,14 +78,13 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, LatticeElementFinderDevice LatticeElementFinder::GetFinderDeviceInstance (WarpXParIter const& a_pti, int const a_offset, - AcceleratorLattice const& accelerator_lattice) + AcceleratorLattice const& accelerator_lattice) const { LatticeElementFinderDevice result; result.InitLatticeElementFinderDevice(a_pti, a_offset, accelerator_lattice, *this); return result; } - void LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& a_pti, int const a_offset, AcceleratorLattice const& accelerator_lattice, @@ -97,7 +96,7 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& int const lev = a_pti.GetLevel(); m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; @@ -121,3 +120,50 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const& } } + +void +LatticeElementFinder::setup_lattice_indices (amrex::Gpu::DeviceVector const & zs, + amrex::Gpu::DeviceVector const & ze, + amrex::Gpu::DeviceVector & indices) const +{ + + using namespace amrex::literals; + + const auto nelements = static_cast(zs.size()); + amrex::ParticleReal const * zs_arr = zs.data(); + amrex::ParticleReal const * ze_arr = ze.data(); + int * indices_arr = indices.data(); + + amrex::Real const zmin = m_zmin; + amrex::Real const dz = m_dz; + + amrex::ParticleReal const gamma_boost = m_gamma_boost; + amrex::ParticleReal const uz_boost = m_uz_boost; + amrex::Real const time = m_time; + + amrex::ParallelFor( m_nz, + [=] AMREX_GPU_DEVICE (int iz) { + + // Get the location of the grid node + amrex::Real z_node = zmin + iz*dz; + + if (gamma_boost > 1._prt) { + // Transform to lab frame + z_node = gamma_boost*z_node + uz_boost*time; + } + + // Find the index to the element that is closest to the grid cell. + // For now, this assumes that there is no overlap among elements of the same type. + for (int ie = 0 ; ie < nelements ; ie++) { + // Find the mid points between element ie and the ones before and after it. + // The first and last element need special handling. + const amrex::ParticleReal zcenter_left = (ie == 0)? + (std::numeric_limits::lowest()) : (0.5_prt*(ze_arr[ie-1] + zs_arr[ie])); + const amrex::ParticleReal zcenter_right = (ie < nelements - 1)? + (0.5_prt*(ze_arr[ie] + zs_arr[ie+1])) : (std::numeric_limits::max()); + if (zcenter_left <= z_node && z_node < zcenter_right) { + indices_arr[iz] = ie; + } + } + }); +} diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp index 9c8726a9c4b..21b1f164777 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedPlasmaLens.cpp @@ -58,7 +58,7 @@ HardEdgedPlasmaLensDevice::InitHardEdgedPlasmaLensDevice (HardEdgedPlasmaLens co nelements = h_plasmalens.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_plasmalens.d_zs.data(); d_ze_arr = h_plasmalens.d_ze.data(); diff --git a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp index 51bcf7a2497..bcf50c55666 100644 --- a/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp +++ b/Source/AcceleratorLattice/LatticeElements/HardEdgedQuadrupole.cpp @@ -58,7 +58,7 @@ HardEdgedQuadrupoleDevice::InitHardEdgedQuadrupoleDevice (HardEdgedQuadrupole co nelements = h_quad.nelements; - if (nelements == 0) return; + if (nelements == 0) { return; } d_zs_arr = h_quad.d_zs.data(); d_ze_arr = h_quad.d_ze.data(); diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index b2016e65d2a..805b4fec181 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -513,7 +513,7 @@ MultiSigmaBox::MultiSigmaBox (const BoxArray& ba, const DistributionMapping& dm, void MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) { - if (dt == dt_B) return; + if (dt == dt_B) { return; } dt_B = dt; @@ -529,7 +529,7 @@ MultiSigmaBox::ComputePMLFactorsB (const Real* dx, Real dt) void MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) { - if (dt == dt_E) return; + if (dt == dt_E) { return; } dt_E = dt; @@ -560,44 +560,62 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri m_geom(geom), m_cgeom(cgeom) { - // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain - // (instead of extending `ncell` outside of the physical domain) - // In order to implement this, a reduced domain is created here (decreased by ncells in all direction) - // and passed to `MakeBoxArray`, which surrounds it by PML boxes - // (thus creating the PML boxes at the right position, where they overlap with the original domain) - // minimalBox provides the bounding box around grid_ba for level, lev. - // Note that this is okay to build pml inside domain for a single patch, or joint patches - // with same [min,max]. But it does not support multiple disjoint refinement patches. - Box domain0 = grid_ba.minimalBox(); + // When `do_pml_in_domain` is true, the PML overlap with the last `ncell` of the physical domain or fine patch(es) + // (instead of extending `ncell` outside of the physical domain or fine patch(es)) + // In order to implement this, we define a new reduced Box Array ensuring that it does not + // include ncells from the edges of the physical domain or fine patch. + // (thus creating the PML boxes at the right position, where they overlap with the original domain or fine patch(es)) + + BoxArray grid_ba_reduced = grid_ba; if (do_pml_in_domain) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (do_pml_Lo[idim]){ - domain0.growLo(idim, -ncell); - } - if (do_pml_Hi[idim]){ - domain0.growHi(idim, -ncell); + BoxList bl = grid_ba.boxList(); + // Here we loop over all the boxes in the original grid_ba BoxArray + // For each box, we find if its in the edge (or boundary), and the size of those boxes are decreased by ncell + for (auto& b : bl) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (do_pml_Lo[idim]) { + // Get neighboring box on lower side in direction idim and check if it intersects with any of the boxes + // in grid_ba. If no intersection, then the box, b, in the boxlist, is in the edge and we decrase + // the size by ncells using growLo(idim,-ncell) + Box const& bb = amrex::adjCellLo(b, idim); + if ( ! grid_ba.intersects(bb) ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(b.length(idim) > ncell, " box length must be greater that pml size"); + b.growLo(idim, -ncell); + } + } + if (do_pml_Hi[idim]) { + // Get neighboring box on higher side in direction idim and check if it intersects with any of the boxes + // in grid_ba. If no intersection, then the box, b, in the boxlist, is in the edge and we decrase + // the size by ncells using growHi(idim,-ncell) + Box const& bb = amrex::adjCellHi(b, idim); + if ( ! grid_ba.intersects(bb) ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(b.length(idim) > ncell, " box length must be greater that pml size"); + b.growHi(idim, -ncell); + } + } } } + grid_ba_reduced = BoxArray(std::move(bl)); } - const BoxArray grid_ba_reduced = (do_pml_in_domain) ? - BoxArray(grid_ba.boxList().intersect(domain0)) : grid_ba; - + Box const domain0 = grid_ba_reduced.minimalBox(); const bool is_single_box_domain = domain0.numPts() == grid_ba_reduced.numPts(); const BoxArray& ba = MakeBoxArray(is_single_box_domain, domain0, *geom, grid_ba_reduced, IntVect(ncell), do_pml_in_domain, do_pml_Lo, do_pml_Hi); + if (ba.empty()) { m_ok = false; return; } else { m_ok = true; } - - // Define the number of guard cells in each direction, for E, B, and F + // Define the number of guard cells in each di;rection, for E, B, and F auto nge = IntVect(AMREX_D_DECL(2, 2, 2)); auto ngb = IntVect(AMREX_D_DECL(2, 2, 2)); int ngf_int = 0; - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ngf_int = std::max( ngf_int, 1 ); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { + ngf_int = std::max( ngf_int, 1 ); + } auto ngf = IntVect(AMREX_D_DECL(ngf_int, ngf_int, ngf_int)); if (do_moving_window) { @@ -758,24 +776,28 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri BoxArray grid_cba = grid_ba; grid_cba.coarsen(ref_ratio); - // assuming that the bounding box around grid_cba is a single patch, and not disjoint patches, similar to fine patch. - amrex::Box cdomain = grid_cba.minimalBox(); + BoxArray grid_cba_reduced = grid_cba; if (do_pml_in_domain) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (do_pml_Lo[idim]){ - // ncell is divided by refinement ratio to ensure that the - // physical width of the PML region is equal in fine and coarse patch - cdomain.growLo(idim, -ncell/ref_ratio[idim]); - } - if (do_pml_Hi[idim]){ - // ncell is divided by refinement ratio to ensure that the - // physical width of the PML region is equal in fine and coarse patch - cdomain.growHi(idim, -ncell/ref_ratio[idim]); + BoxList bl = grid_cba.boxList(); + for (auto& b : bl) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (do_pml_Lo[idim]) { + Box const& bb = amrex::adjCellLo(b, idim); + if ( ! grid_cba.intersects(bb) ) { + b.growLo(idim, -ncell/ref_ratio[idim]); + } + } + if (do_pml_Hi[idim]) { + Box const& bb = amrex::adjCellHi(b, idim); + if ( ! grid_cba.intersects(bb) ) { + b.growHi(idim, -ncell/ref_ratio[idim]); + } + } } } + grid_cba_reduced = BoxArray(std::move(bl)); } - const BoxArray grid_cba_reduced = (do_pml_in_domain) ? - BoxArray(grid_cba.boxList().intersect(cdomain)) : grid_cba; + Box const cdomain = grid_cba_reduced.minimalBox(); const IntVect cncells = IntVect(ncell)/ref_ratio; const IntVect cdelta = IntVect(delta)/ref_ratio; @@ -790,7 +812,6 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } else { cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,0).ixType().toIntVect()); const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,1).ixType().toIntVect()); const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,2).ixType().toIntVect()); @@ -1074,9 +1095,9 @@ void PML::Exchange (const std::array& mf_pml, const int do_pml_in_domain) { const amrex::Geometry& geom = (patch_type == PatchType::fine) ? *m_geom : *m_cgeom; - if (mf_pml[0] && mf[0]) Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); - if (mf_pml[1] && mf[1]) Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); - if (mf_pml[2] && mf[2]) Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); + if (mf_pml[0] && mf[0]) { Exchange(*mf_pml[0], *mf[0], geom, do_pml_in_domain); } + if (mf_pml[1] && mf[1]) { Exchange(*mf_pml[1], *mf[1], geom, do_pml_in_domain); } + if (mf_pml[2] && mf[2]) { Exchange(*mf_pml[2], *mf[2], geom, do_pml_in_domain); } } void diff --git a/Source/BoundaryConditions/PMLComponent.H b/Source/BoundaryConditions/PMLComponent.H index 6fa4f8af784..83dca1e8b9a 100644 --- a/Source/BoundaryConditions/PMLComponent.H +++ b/Source/BoundaryConditions/PMLComponent.H @@ -9,7 +9,7 @@ /* In WarpX, the split fields of the PML (e.g. Eyx, Eyz) are stored as * components of a MultiFab (e.g. component 0 and 1 of the MultiFab for Ey) - * The correspondance between the component index (0,1) and its meaning + * The correspondence between the component index (0,1) and its meaning * (yx, yz, etc.) is defined in the present file */ struct PMLComp { diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index 8f3c5a1b49b..af721d70b6d 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -51,13 +51,13 @@ void WarpX::DampPML (const int lev) { DampPML(lev, PatchType::fine); - if (lev > 0) DampPML(lev, PatchType::coarse); + if (lev > 0) { DampPML(lev, PatchType::coarse); } } void WarpX::DampPML (const int lev, PatchType patch_type) { - if (!do_pml) return; + if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -228,15 +228,15 @@ void WarpX::DampJPML (int lev) { DampJPML(lev, PatchType::fine); - if (lev > 0) DampJPML(lev, PatchType::coarse); + if (lev > 0) { DampJPML(lev, PatchType::coarse); } } void WarpX::DampJPML (int lev, PatchType patch_type) { - if (!do_pml) return; - if (!do_pml_j_damping) return; - if (!pml[lev]) return; + if (!do_pml) { return; } + if (!do_pml_j_damping) { return; } + if (!pml[lev]) { return; } WARPX_PROFILE("WarpX::DampJPML()"); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index fc4b563915d..af7a385b729 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -41,6 +41,18 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) } } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), + get_pointer_Efield_fp(lev, 1), + get_pointer_Efield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), + get_pointer_Efield_cp(lev, 1), + get_pointer_Efield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) @@ -69,27 +81,123 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d applySilverMueller = true; } } - if(applySilverMueller) m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( Efield_fp[lev], Bfield_fp[lev], Geom(lev).Domain(), dt[lev], WarpX::field_boundary_lo, WarpX::field_boundary_hi); + } } } + +#ifdef WARPX_DIM_RZ + if (patch_type == PatchType::fine) { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), + get_pointer_Bfield_fp(lev, 1), + get_pointer_Bfield_fp(lev, 2), lev); + } else { + ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), + get_pointer_Bfield_cp(lev, 1), + get_pointer_Bfield_cp(lev, 2), lev); + } +#endif } void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoRhofield(rho, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoRhofield(rho, lev, patch_type); } } void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); + if (PEC::isAnyBoundaryPEC()) { PEC::ApplyPECtoJfield(Jx, Jy, Jz, lev, patch_type); } +} + +#ifdef WARPX_DIM_RZ +// Applies the boundary conditions that are specific to the axis when in RZ. +void +WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const +{ + const amrex::IntVect ngE = get_ng_fieldgather(); + + constexpr int NODE = amrex::IndexType::NODE; + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Er, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Box const & tilebox = mfi.tilebox(); + + // Lower corner of tile box physical domain + // Note that this is done before the tilebox.grow so that + // these do not include the guard cells. + const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin[0]; + + // Skip blocks that don't touch the axis + if (rmin > 0._rt) { continue; } + + amrex::Array4 const& Er_arr = Er->array(mfi); + amrex::Array4 const& Et_arr = Et->array(mfi); + amrex::Array4 const& Ez_arr = Ez->array(mfi); + + amrex::Box tbr = amrex::convert( tilebox, Er->ixType().toIntVect() ); + amrex::Box tbt = amrex::convert( tilebox, Et->ixType().toIntVect() ); + amrex::Box tbz = amrex::convert( tilebox, Ez->ixType().toIntVect() ); + + // For ishift, 1 means cell centered, 0 means node centered + int const ishift_r = (tbr.type(0) != NODE); + int const ishift_t = (tbt.type(0) != NODE); + int const ishift_z = (tbz.type(0) != NODE); + + // Set tileboxes to only include the axis guard cells + // (including the corners in z). + tbr.setRange(0, -ngE[0], ngE[0]); + tbt.setRange(0, -ngE[0], ngE[0]); + tbz.setRange(0, -ngE[0], ngE[0]); + tbr.grow(1, ngE[1]); + tbt.grow(1, ngE[1]); + tbz.grow(1, ngE[1]); + + const int nmodes = n_rz_azimuthal_modes; + + amrex::ParallelFor(tbr, tbt, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Er_arr(i,j,0,0) = -Er_arr(-i-ishift_r,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Er_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode-1); + Er_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Er_arr(-i-ishift_r,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Et_arr(i,j,0,0) = -Et_arr(-i-ishift_t,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Et_arr(i,j,0,2*imode-1) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode-1); + Et_arr(i,j,0,2*imode) = std::pow(-1._rt, imode+1._rt)*Et_arr(-i-ishift_t,j,0,2*imode); + } + }, + [=] AMREX_GPU_DEVICE (int i, int j, int /*k*/) + { + Ez_arr(i,j,0,0) = Ez_arr(-i-ishift_z,j,0,0); + + for (int imode=1 ; imode < nmodes ; imode++) { + Ez_arr(i,j,0,2*imode-1) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode-1); + Ez_arr(i,j,0,2*imode) = -std::pow(-1._rt, imode+1._rt)*Ez_arr(-i-ishift_z,j,0,2*imode); + } + + }); + } } +#endif void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index 3d3e4729d45..62b92ba94b5 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -147,7 +147,7 @@ using namespace amrex; amrex::GpuArray const& fbndry_lo, amrex::GpuArray const& fbndry_hi ) { - // Tangential Efield componentes in guard cells set equal and opposite to cells + // Tangential Efield components in guard cells set equal and opposite to cells // in the mirror locations across the PEC boundary, whereas normal E-field // components are set equal to values in the mirror locations across the PEC // boundary. Here we just initialize it. @@ -191,7 +191,17 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) sign *= -1._rt; + if (is_tangent_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // is PEC boundary } // loop over iside @@ -319,7 +329,15 @@ using namespace amrex; : (dom_hi[idim] + 1 - ig)); GuardCell = true; // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) sign *= -1._rt; + if (is_normal_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif } } // if PEC Boundary } // loop over sides @@ -373,17 +391,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) field(ijk_vec, n) = 0._rt; + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; // otherwise update the internal cell if the mirror guard cell exists - else if (fabbox.contains(iv_mirror)) - { + } else if (fabbox.contains(iv_mirror)) { field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); } } @@ -394,16 +412,17 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } amrex::IntVect iv_mirror = ijk_vec; iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) { - if (tangent_to_bndy[idim]) + if (tangent_to_bndy[idim]) { field(iv_mirror, n) = -field(ijk_vec, n); - else + } else { field(iv_mirror, n) = field(ijk_vec, n); + } } } } @@ -436,7 +455,7 @@ using namespace amrex; { for (int iside = 0; iside < 2; ++iside) { - if (!is_pec[idim][iside]) continue; + if (!is_pec[idim][iside]) { continue; } // Get the mirror guard cell index amrex::IntVect iv_mirror = ijk_vec; @@ -446,7 +465,7 @@ using namespace amrex; // first value in the domain (nodal fields) if (ijk_vec == iv_mirror) { iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) field(ijk_vec, n) = field(iv_mirror, n); + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } } // otherwise set the mirror guard cell equal to the internal cell value else if (fabbox.contains(iv_mirror)) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0b8d1c0fcc8..4692d8e980f 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -18,8 +18,8 @@ using namespace amrex::literals; bool PEC::isAnyBoundaryPEC() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } } return false; } @@ -250,8 +250,8 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } // rho values inside guard cells are updated the same as tangential // components of the current density @@ -276,7 +276,7 @@ PEC::ApplyPECtoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_ty // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& rho_array = rho->array(mfi); @@ -342,8 +342,8 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } for (int icomp=0; icomp < 3; ++icomp) { // Set the psign value correctly for each current component for each @@ -398,7 +398,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jx_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jx_array = Jx->array(mfi); @@ -433,7 +433,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jy_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jy_array = Jy->array(mfi); @@ -468,7 +468,7 @@ PEC::ApplyPECtoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box grown_domain_box.convert(Jz_nodal); - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Jz_array = Jz->array(mfi); @@ -520,8 +520,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; - if (!is_pec[idim][0]) grown_domain_box.growLo(idim, ng_fieldgather[idim]); - if (!is_pec[idim][1]) grown_domain_box.growHi(idim, ng_fieldgather[idim]); + if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } + if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - Pe_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - Pe_nodal[idim]); @@ -538,7 +538,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - if (grown_domain_box.contains(fabbox)) continue; + if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data auto const& Pe_array = Pefield->array(mfi); diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 209113dedeb..0a22774bb8d 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -144,9 +144,9 @@ private: /** Number of Fabs in the plotfile. */ int m_numFabs; /** Lower corner physical coordinates of each fab in the plotfile. */ - amrex::Vector > m_glo {{AMREX_D_DECL(0.,0.,0.)}}; + amrex::Vector > m_glo; /** Upper corner physical coordinates of each fab in the plotfile. */ - amrex::Vector > m_ghi {{AMREX_D_DECL(1.,1.,1.)}}; + amrex::Vector > m_ghi; std::string m_CellPath; }; @@ -171,11 +171,11 @@ class BTDMultiFabHeaderImpl void WriteMultiFabHeader (); /** Returns size, m_ba_size, of the Box Array, m_ba.*/ - int ba_size () {return m_ba_size;} + [[nodiscard]] int ba_size () const {return m_ba_size;} /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Returns prefix of the ith-fab on disk, i.e., ith fab of the MultiFab data. * \param[in] ifab index of the ith fab in the MultiFab data. */ @@ -316,9 +316,9 @@ public: /** Reads the particle header file at m_Header_path and stores its data*/ void ReadHeader (); /** Writes the meta-data of particle box array in header file, with path, m_Header_path*/ - void WriteHeader (); + void WriteHeader () const; /** Returns the size of the box array, m_ba_size */ - int ba_size () {return m_ba_size; } + [[nodiscard]] int ba_size () const {return m_ba_size; } /** Increases Box array size, m_ba_size, by add_size * \param[in] add_size */ @@ -326,7 +326,7 @@ public: /** Returns box corresponding to the ith box in the BoxArray, m_ba. * \param[in] ibox index of the box in the BoxArray. */ - amrex::Box ba_box (int ibox) {return m_ba[ibox]; } + [[nodiscard]] amrex::Box ba_box (int ibox) const {return m_ba[ibox]; } /** Resize boxArray, m_ba, to size, m_ba_size. */ void ResizeBoxArray () { m_ba.resize(m_ba_size); } /** Set Box indices of the ith-box in Box Array, m_ba, to the new Box, ba_box. diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index 9f4976c20cb..cd3eb962405 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -22,7 +22,9 @@ using namespace amrex::literals; BTDPlotfileHeaderImpl::BTDPlotfileHeaderImpl (std::string const & Headerfile_path) - : m_Header_path{Headerfile_path} + : m_Header_path{Headerfile_path}, + m_glo{{AMREX_D_DECL(0., 0., 0.)}}, + m_ghi{{AMREX_D_DECL(1., 1., 1.)}} { } @@ -118,7 +120,7 @@ BTDPlotfileHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -241,7 +243,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_minval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_minval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } ablastr::utils::text::goto_next_line(is); @@ -251,7 +253,7 @@ BTDMultiFabHeaderImpl::ReadMultiFabHeader () m_maxval[ifab].resize(m_ncomp); for (int icomp = 0; icomp < m_ncomp; ++icomp) { is >> m_maxval[ifab][icomp] >> ch; - if( ch != ',' ) amrex::Error("Expected a ',' got something else"); + if( ch != ',' ) { amrex::Error("Expected a ',' got something else"); } } } @@ -266,9 +268,9 @@ BTDMultiFabHeaderImpl::WriteMultiFabHeader () } std::ofstream FabHeaderFile; FabHeaderFile.open(m_Header_path.c_str(), std::ofstream::out | - std::ofstream::trunc | - std::ofstream::binary); - if ( !FabHeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + std::ofstream::trunc | + std::ofstream::binary); + if ( !FabHeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } FabHeaderFile.precision(17); @@ -421,7 +423,7 @@ BTDSpeciesHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); @@ -516,7 +518,7 @@ BTDParticleDataHeaderImpl::ReadHeader () } void -BTDParticleDataHeaderImpl::WriteHeader () +BTDParticleDataHeaderImpl::WriteHeader () const { if (amrex::FileExists(m_Header_path)) { amrex::FileSystem::Remove(m_Header_path); @@ -525,7 +527,7 @@ BTDParticleDataHeaderImpl::WriteHeader () HeaderFile.open(m_Header_path.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if ( !HeaderFile.good()) amrex::FileOpenFailed(m_Header_path); + if ( !HeaderFile.good()) { amrex::FileOpenFailed(m_Header_path); } HeaderFile.precision(17); m_ba.writeOn(HeaderFile); diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 4198846fa5b..f6c44c777ea 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -24,8 +24,7 @@ #include #include -class -BTDiagnostics final : public Diagnostics +class BTDiagnostics final : public Diagnostics { public: @@ -276,7 +275,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZBoostCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_boost = (t_lab / m_gamma_boost - t_boost) * PhysConst::c / m_beta_boost; return current_z_boost; @@ -285,7 +284,7 @@ private: * \param[in] t_lab lab-frame time of the snapshot * \param[in] t_boost boosted-frame time at level, lev */ - amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) + [[nodiscard]] amrex::Real UpdateCurrentZLabCoordinate(amrex::Real t_lab, amrex::Real t_boost) const { const amrex::Real current_z_lab = (t_lab - t_boost / m_gamma_boost ) * PhysConst::c / m_beta_boost; return current_z_lab; @@ -295,13 +294,13 @@ private: * \param[in] ref_ratio refinement ratio in the z-direction at level, lev-1. * The ref-ratio in the z-direction for single-level diagnostics is 1. */ - amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio); + [[nodiscard]] amrex::Real dz_lab (amrex::Real dt, amrex::Real ref_ratio) const; /** Compute k-index corresponding to current lab-frame z co-ordinate (m_current_z_lab) * for the ith buffer i_buffer, and at level, lev. * \param[in] i_buffer snapshot index * \param[in] lev mesh-refinement level at which the lab-frame z-index is computed */ - int k_index_zlab (int i_buffer, int lev); + [[nodiscard]] int k_index_zlab (int i_buffer, int lev) const; /** whether field buffer is full * \param[in] i_buffer buffer id for which the buffer size is checked. * returns bool = true is buffer is full, that is, @@ -381,8 +380,8 @@ private: /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader( std::string Buffer_FabHeaderFilename, - std::string snapshot_FabHeaderFilename, + void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, std::string newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data @@ -400,8 +399,6 @@ private: lab-frame data. */ void InitializeParticleFunctors () override; - /** Update total number of particles flushed for all species for ith snapshot */ - void UpdateTotalParticlesFlushed(int i_buffer); /** Reset total number of particles in the particle buffer to 0 for ith snapshot */ void ResetTotalParticlesInBuffer(int i_buffer); /** Clear particle data stored in the particle buffer */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 52d3656a425..41c710805e9 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -112,8 +112,9 @@ void BTDiagnostics::DerivedInitData () const amrex::ParmParse pp_diag_name(m_diag_name); int write_species = 1; pp_diag_name.query("write_species", write_species); - if ((m_output_species_names.empty()) && (write_species == 1)) + if ((m_output_species_names.empty()) && (write_species == 1)) { m_output_species_names = mpc.GetSpeciesNames(); + } m_do_back_transformed_particles = ((!m_output_species_names.empty()) && (write_species == 1)); @@ -128,7 +129,6 @@ void BTDiagnostics::DerivedInitData () } } m_particles_buffer.resize(m_num_buffers); - m_totalParticles_flushed_already.resize(m_num_buffers); m_totalParticles_in_buffer.resize(m_num_buffers); // check that simulation can fill all BTD snapshots @@ -233,7 +233,7 @@ BTDiagnostics::ReadParameters () pp_diag_name.query("do_back_transformed_fields", m_do_back_transformed_fields); pp_diag_name.query("do_back_transformed_particles", m_do_back_transformed_particles); AMREX_ALWAYS_ASSERT(m_do_back_transformed_fields or m_do_back_transformed_particles); - if (!m_do_back_transformed_fields) m_varnames.clear(); + if (!m_do_back_transformed_fields) { m_varnames.clear(); } std::vector intervals_string_vec = {"0"}; @@ -348,7 +348,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) amrex::RealBox diag_dom; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim ) { // Setting lo-coordinate for the diag domain by taking the max of user-defined - // lo-cordinate and lo-coordinat of the simulation domain at level, lev + // lo-cordinate and lo-coordinate of the simulation domain at level, lev diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); // Setting hi-coordinate for the diag domain by taking the max of user-defined // hi-cordinate and hi-coordinate of the simulation domain at level, lev @@ -424,11 +424,11 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) // computed using the coarsened cell-size in the lab-frame obtained using // the ref_ratio at level, lev-1. auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } // Number of lab-frame cells in z-direction at level, lev const int num_zcells_lab = static_cast( std::floor ( ( zmax_buffer_lab - zmin_buffer_lab) - / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]) ) ); + / dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]))); // Take the max of 0 and num_zcells_lab const int Nz_lab = std::max( 0, num_zcells_lab ); #if (AMREX_SPACEDIM >= 2) @@ -502,7 +502,7 @@ BTDiagnostics::InitializeBufferData ( int i_buffer , int lev, bool restart) void BTDiagnostics::DefineCellCenteredMultiFab(int lev) { - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } // Creating MultiFab to store cell-centered data in boosted-frame for the entire-domain // This MultiFab will store all the user-requested fields in the boosted-frame auto & warpx = WarpX::GetInstance(); @@ -524,7 +524,7 @@ void BTDiagnostics::InitializeFieldFunctors (int lev) { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } #ifdef WARPX_DIM_RZ // For RZ, initialize field functors RZ for openpmd @@ -611,16 +611,16 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () const auto m_varnames_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(m_cellcenter_varnames_fields.size()); for (int comp=0; comp(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ - m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } } @@ -770,10 +770,11 @@ BTDiagnostics::UpdateBufferData () for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer ) { const bool ZSliceInDomain = GetZSliceInDomainFlag (i_buffer, lev); - if (ZSliceInDomain) ++m_buffer_counter[i_buffer]; + if (ZSliceInDomain) { ++m_buffer_counter[i_buffer]; } // when the z-index is equal to the smallEnd of the snapshot box, then set lastValidZSlice to 1 - if (k_index_zlab(i_buffer, lev) == m_snapshot_box[i_buffer].smallEnd(m_moving_window_dir)) + if (k_index_zlab(i_buffer, lev) == m_snapshot_box[i_buffer].smallEnd(m_moving_window_dir)) { m_lastValidZSlice[i_buffer] = 1; + } } } } @@ -783,11 +784,11 @@ void BTDiagnostics::PrepareFieldDataForOutput () { // Initialize fields functors only if do_back_transformed_fields is selected - if (!m_do_back_transformed_fields) return; + if (!m_do_back_transformed_fields) { return; } auto & warpx = WarpX::GetInstance(); // In this function, we will get cell-centered data for every level, lev, - // using the cell-center functors and their respective opeators() + // using the cell-center functors and their respective operators() // Call m_cell_center_functors->operator for (int lev = 0; lev < nmax_lev; ++lev) { int icomp_dst = 0; @@ -861,19 +862,20 @@ BTDiagnostics::PrepareFieldDataForOutput () amrex::Real -BTDiagnostics::dz_lab (amrex::Real dt, amrex::Real ref_ratio){ +BTDiagnostics::dz_lab (amrex::Real dt, amrex::Real ref_ratio) const +{ return PhysConst::c * dt * 1._rt/m_beta_boost * 1._rt/m_gamma_boost * 1._rt/ref_ratio; } int -BTDiagnostics::k_index_zlab (int i_buffer, int lev) +BTDiagnostics::k_index_zlab (int i_buffer, int lev) const { auto & warpx = WarpX::GetInstance(); const amrex::Real prob_domain_zmin_lab = m_snapshot_domain_lab[i_buffer].lo( m_moving_window_dir ); auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); - const int k_lab = static_cast(floor ( + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } + const int k_lab = static_cast(std::floor ( ( m_current_z_lab[i_buffer] - (prob_domain_zmin_lab ) ) / dz_lab( warpx.getdt(lev), ref_ratio[m_moving_window_dir] ) @@ -884,17 +886,16 @@ BTDiagnostics::k_index_zlab (int i_buffer, int lev) void BTDiagnostics::SetSnapshotFullStatus (const int i_buffer) { - if (m_snapshot_full[i_buffer] == 1) return; + if (m_snapshot_full[i_buffer] == 1) { return; } // if the last valid z-index of the snapshot, which is 0, is filled, then // set the snapshot full integer to 1 - if (m_lastValidZSlice[i_buffer] == 1) m_snapshot_full[i_buffer] = 1; - + if (m_lastValidZSlice[i_buffer] == 1) { m_snapshot_full[i_buffer] = 1; } } void BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) { - if (m_field_buffer_multifab_defined[i_buffer] == 1) return; + if (m_field_buffer_multifab_defined[i_buffer] == 1) { return; } auto & warpx = WarpX::GetInstance(); const int hi_k_lab = m_buffer_k_index_hi[i_buffer]; @@ -911,7 +912,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) m_mf_output[i_buffer][lev].setVal(0.); auto ref_ratio = amrex::IntVect(1); - if (lev > 0 ) ref_ratio = WarpX::RefRatio(lev-1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const amrex::Real cellsize = (idim < WARPX_ZINDEX)? warpx.Geom(lev).CellSize(idim): @@ -954,7 +955,7 @@ BTDiagnostics::DefineFieldBufferMultiFab (const int i_buffer, const int lev) void BTDiagnostics::DefineSnapshotGeometry (const int i_buffer, const int lev) { - if (m_snapshot_geometry_defined[i_buffer] == 1) return; + if (m_snapshot_geometry_defined[i_buffer] == 1) { return; } if (lev == 0) { // Default non-periodic geometry for diags @@ -1063,12 +1064,12 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) } } m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + labtime, + m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, - use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter[i_buffer], - m_max_buffer_multifabs[i_buffer], m_geom_snapshot[i_buffer][0], isLastBTDFlush, - m_totalParticles_flushed_already[i_buffer]); + use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), + m_max_buffer_multifabs.at(i_buffer), m_geom_snapshot.at(i_buffer).at(0), isLastBTDFlush); // Rescaling the box for plotfile after WriteToFile. This is because, for plotfiles, when writing particles, amrex checks if the particles are within the bounds defined by the box. However, in BTD, particles can be (at max) 1 cell outside the bounds of the geometry. So we keep a one-cell bigger box for plotfile when writing out the particle data and rescale after. if (m_format == "plotfile") { @@ -1102,7 +1103,6 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) NullifyFirstFlush(i_buffer); // if particles are selected for output then update and reset counters if (!m_output_species_names.empty()) { - UpdateTotalParticlesFlushed(i_buffer); ResetTotalParticlesInBuffer(i_buffer); ClearParticleBuffer(i_buffer); } @@ -1151,17 +1151,20 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) // Create directory only when the first buffer is flushed out. if (m_buffer_flush_counter[i_snapshot] == 0 || m_first_flush_after_restart[i_snapshot] == 1) { // Create Level_0 directory to store all Cell_D and Cell_H files - if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) + if (!amrex::UtilCreateDirectory(snapshot_Level0_path, permission_flag_rwxrxrx) ) { amrex::CreateDirectoryFailed(snapshot_Level0_path); + } // Create directory for each species selected for diagnostic for (int i = 0; i < m_particles_buffer[i_snapshot].size(); ++i) { const std::string snapshot_species_path = snapshot_path + "/" + m_output_species_names[i]; - if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(snapshot_species_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(snapshot_species_path); + } // Create Level_0 directory for particles to store Particle_H and DATA files const std::string species_Level0_path = snapshot_species_path + "/Level_0"; - if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) + if ( !amrex::UtilCreateDirectory(species_Level0_path, permission_flag_rwxrxrx)) { amrex::CreateDirectoryFailed(species_Level0_path); + } } const std::string buffer_WarpXHeader_path = recent_Buffer_filepath + "/WarpXHeader"; const std::string snapshot_WarpXHeader_path = snapshot_path + "/WarpXHeader"; @@ -1254,7 +1257,7 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_species_Header.c_str(), snapshot_species_Header.c_str()) == 0, std::string("Renaming ").append(recent_species_Header).append(" to ").append(snapshot_species_Header).append(" has failed")); - if (BufferSpeciesHeader.m_total_particles == 0) continue; + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } // if finite number of particles in the output, copy ParticleHdr and Data file WARPX_ALWAYS_ASSERT_WITH_MESSAGE( std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, @@ -1265,11 +1268,11 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } else { InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); - if (BufferSpeciesHeader.m_total_particles == 0) continue; - if (m_totalParticles_flushed_already[i_snapshot][i]==0) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, - std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); + if (BufferSpeciesHeader.m_total_particles == 0) { continue; } + if (!amrex::FileExists(snapshot_ParticleHdrFilename)) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, + std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); } else { InterleaveParticleDataHeader(recent_ParticleHdrFilename, snapshot_ParticleHdrFilename); @@ -1296,7 +1299,7 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat BTDPlotfileHeaderImpl buffer_HeaderImpl(buffer_Header_path); buffer_HeaderImpl.ReadHeaderData(); - // Update timestamp of snapshot with recently flushed buffer + // Update step_scraped of snapshot with recently flushed buffer snapshot_HeaderImpl.set_time( buffer_HeaderImpl.time() ); snapshot_HeaderImpl.set_timestep( buffer_HeaderImpl.timestep() ); @@ -1332,9 +1335,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader(std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, + std::string snapshot_FabHeader_path, + std::string newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); @@ -1430,10 +1433,8 @@ BTDiagnostics::InitializeParticleBuffer () const MultiParticleContainer& mpc = warpx.GetPartContainer(); for (int i = 0; i < m_num_buffers; ++i) { m_particles_buffer[i].resize(m_output_species_names.size()); - m_totalParticles_flushed_already[i].resize(m_output_species_names.size()); m_totalParticles_in_buffer[i].resize(m_output_species_names.size()); for (int isp = 0; isp < m_particles_buffer[i].size(); ++isp) { - m_totalParticles_flushed_already[i][isp] = 0; m_totalParticles_in_buffer[i][isp] = 0; m_particles_buffer[i][isp] = std::make_unique(WarpX::GetInstance().GetParGDB()); const int idx = mpc.getSpeciesID(m_output_species_names[isp]); @@ -1484,15 +1485,6 @@ BTDiagnostics::PrepareParticleDataForOutput() } } -void -BTDiagnostics::UpdateTotalParticlesFlushed(int i_buffer) -{ - for (int isp = 0; isp < m_totalParticles_flushed_already[i_buffer].size(); ++isp) { - m_totalParticles_flushed_already[i_buffer][isp] += static_cast( - m_particles_buffer[i_buffer][isp]->TotalNumberOfParticles()); - } -} - void BTDiagnostics::ResetTotalParticlesInBuffer(int i_buffer) { diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 097835297dd..60d184c30e2 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -14,8 +14,7 @@ /** collect the particles that are absorbed at the embedded boundary, throughout the simulation */ -class -BoundaryScrapingDiagnostics final : public Diagnostics +class BoundaryScrapingDiagnostics final : public Diagnostics { public: diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 9797b77ac1d..11ffce02f09 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -102,15 +102,6 @@ BoundaryScrapingDiagnostics::InitializeParticleBuffer () m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species_name, pc, bnd_buffer)); } } - // Initialize total number of particles flushed - m_totalParticles_flushed_already.resize(m_num_buffers); - for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = static_cast(m_output_species_names.size()); - m_totalParticles_flushed_already[i_buffer].resize(n_species); - for (int i_species=0; i_speciesWriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), + nlev_output, file_prefix, m_file_min_digits, false, false, use_pinned_pc, isBTD, warpx.getistep(0), bufferID, numBTDBuffers, geom, - isLastBTD, m_totalParticles_flushed_already[i_buffer]); + isLastBTD); // Now that the data has been written out, clear out the buffer particle_buffer.clearParticles(i_buffer); diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index ac361db98fa..bef40ae1ce0 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -29,8 +29,7 @@ * lab-frame field data is then stored in mf_dst. */ -class -BackTransformFunctor final : public ComputeDiagFunctor +class BackTransformFunctor final : public ComputeDiagFunctor { public: /** Constructor description @@ -59,7 +58,7 @@ public: * The data is then lorentz-transformed in-place using LorenzTransformZ (). * The user-requested fields are then copied to mf_dst. * - * \param[out] mf_dst output MuliFab where the back-transformed data is written + * \param[out] mf_dst output MultiFab where the back-transformed data is written * \param[in] dcomp first component of mf_dst in which the back-transformed * lab-frame data for the user-request fields is written. * \param[in] i_buffer buffer index for which the data is transformed. diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index f6f2bcd1205..22754f51a53 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -162,7 +162,7 @@ BackTransformFunctor::PrepareFunctorData (int i_buffer, m_current_z_boost[i_buffer] = current_z_boost; m_k_index_zlab[i_buffer] = k_index_zlab; m_perform_backtransform[i_buffer] = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform[i_buffer] = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform[i_buffer] = 1; } } void diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H index afc3e6e8511..fa00f6288f9 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H @@ -143,19 +143,19 @@ struct LorentzTransformParticles const amrex::ParticleReal uzp = uz_old_p * weight_old + uz_new_p * weight_new; #if defined (WARPX_DIM_3D) - dst.m_aos[i_dst].pos(0) = xp; - dst.m_aos[i_dst].pos(1) = yp; - dst.m_aos[i_dst].pos(2) = zp; + dst.m_rdata[PIdx::x][i_dst] = xp; + dst.m_rdata[PIdx::y][i_dst] = yp; + dst.m_rdata[PIdx::z][i_dst] = zp; #elif defined (WARPX_DIM_RZ) - dst.m_aos[i_dst].pos(0) = std::sqrt(xp*xp + yp*yp); - dst.m_aos[i_dst].pos(1) = zp; + dst.m_rdata[PIdx::x][i_dst] = std::sqrt(xp*xp + yp*yp); + dst.m_rdata[PIdx::z][i_dst] = zp; dst.m_rdata[PIdx::theta][i_dst] = std::atan2(yp, xp); #elif defined (WARPX_DIM_XZ) - dst.m_aos[i_dst].pos(0) = xp; - dst.m_aos[i_dst].pos(1) = zp; + dst.m_rdata[PIdx::x][i_dst] = xp; + dst.m_rdata[PIdx::z][i_dst] = zp; amrex::ignore_unused(yp); -#elif defined (WARPX_DIM_1D) - dst.m_aos[i_dst].pos(0) = zp; +#elif defined (WARPX_DIM_1D_Z) + dst.m_rdata[PIdx::z][i_dst] = zp; amrex::ignore_unused(xp, yp); #else amrex::ignore_unused(xp, yp, zp); @@ -195,15 +195,14 @@ struct LorentzTransformParticles * \brief BackTransform functor to select particles and Lorentz Transform them * and store in particle buffers */ -class -BackTransformParticleFunctor final : public ComputeParticleDiagFunctor +class BackTransformParticleFunctor final : public ComputeParticleDiagFunctor { public: BackTransformParticleFunctor(WarpXParticleContainer *pc_src, std::string species_name, int num_buffers); /** Computes the Lorentz transform of source particles to obtain lab-frame data in pc_dst*/ void operator () (PinnedMemoryParticleContainer& pc_dst, int &TotalParticleCounter, int i_buffer) const override; void InitData() override; - /** \brief Prepare data required to back-transform particle attribtutes for lab-frame snapshot, i_buffer + /** \brief Prepare data required to back-transform particle attributes for lab-frame snapshot, i_buffer * * \param[in] i_buffer index of the snapshot * \param[in] z_slice_in_domain if the z-slice at current_z_boost is within the bounds of diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index ce463a11ca3..8a6f0765664 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -37,10 +37,10 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti { using namespace amrex::literals; - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } m_get_position = GetParticlePosition(a_pti, a_offset); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_wpnew = attribs[PIdx::w].dataPtr(); m_uxpnew = attribs[PIdx::ux].dataPtr(); m_uypnew = attribs[PIdx::uy].dataPtr(); @@ -80,7 +80,7 @@ BackTransformParticleFunctor::BackTransformParticleFunctor ( void BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst, int &totalParticleCounter, int i_buffer) const { - if (m_perform_backtransform[i_buffer] == 0) return; + if (m_perform_backtransform[i_buffer] == 0) { return; } auto &warpx = WarpX::GetInstance(); // get particle slice const int nlevs = std::max(0, m_pc_src->finestLevel()+1); @@ -142,8 +142,9 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst amrex::ParallelFor(np, [=] AMREX_GPU_DEVICE(int i) { - if (Flag[i] == 1) GetParticleLorentzTransform(dst_data, src_data, i, + if (Flag[i] == 1) { GetParticleLorentzTransform(dst_data, src_data, i, old_size + IndexLocation[i]); + } }); amrex::Gpu::synchronize(); } @@ -171,5 +172,5 @@ BackTransformParticleFunctor::PrepareFunctorData ( int i_buffer, bool z_slice_in m_current_z_boost.at(i_buffer) = current_z_boost; m_t_lab.at(i_buffer) = t_lab; m_perform_backtransform.at(i_buffer) = 0; - if (z_slice_in_domain && (snapshot_full == 0)) m_perform_backtransform.at(i_buffer) = 1; + if (z_slice_in_domain && (snapshot_full == 0)) { m_perform_backtransform.at(i_buffer) = 1; } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H index 58823d7bb17..dd5bb239ecf 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -CellCenterFunctor final : public ComputeDiagFunctor +class CellCenterFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H index 68f9c38bc8e..c254a341122 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H @@ -13,8 +13,7 @@ * \brief Functor to compute a diagnostic and store the result in existing * MultiFab */ -class -ComputeDiagFunctor +class ComputeDiagFunctor { public: ComputeDiagFunctor( int ncomp, amrex::IntVect crse_ratio) : diff --git a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H index 3309dbed584..6856c4c90a2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ComputeParticleDiagFunctor.H @@ -15,8 +15,7 @@ /** * \brief Functor to compute a diagnostic and store the result in existing ParticleContainer. */ -class -ComputeParticleDiagFunctor +class ComputeParticleDiagFunctor { public: @@ -30,7 +29,7 @@ public: ComputeParticleDiagFunctor(ComputeParticleDiagFunctor&&) = default; ComputeParticleDiagFunctor& operator=(ComputeParticleDiagFunctor&&) = default; - /** \brief Prepare data required to back-transform particle attribtutes for + /** \brief Prepare data required to back-transform particle attributes for * lab-frame snapshot, with index i_buffer. * Note that this function has parameters that are specific to * back-transformed diagnostics, that are unused for regular diagnostics. diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H index ebf890f6545..3d04a56742b 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivBFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divB into mf_out. */ -class -DivBFunctor final : public ComputeDiagFunctor +class DivBFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H index 76e5270b972..312ccaa5cd6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/DivEFunctor.H @@ -10,8 +10,7 @@ /** * \brief Functor to compute divE into mf_out. */ -class -DivEFunctor final : public ComputeDiagFunctor +class DivEFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index c42f63f1369..3080b500712 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF for current density and store result in mf_out. */ -class -JFunctor final : public ComputeDiagFunctor +class JFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 4663a5e1a70..9938cf095bd 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -36,22 +36,28 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); auto& mypc = warpx.GetPartContainer(); mypc.DepositCurrent(current_fp_temp, warpx.getdt(m_lev), 0.0); + + // sum values in guard cells - note that this does not filter the + // current density. + for (int idim = 0; idim < 3; ++idim) { + current_fp_temp[0][idim]->FillBoundary(warpx.Geom(m_lev).periodicity()); + } } InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H index af218d2667b..1b8785af7b7 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerCellFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerCellFunctor final : public ComputeDiagFunctor +class PartPerCellFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H index b4d64117dd7..9718c9c7163 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/PartPerGridFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to cell-center MF and store result in mf_out. */ -class -PartPerGridFunctor final : public ComputeDiagFunctor +class PartPerGridFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 1939a185c97..520951218f9 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -12,8 +12,7 @@ /** * \brief Functor to calculate per-cell averages of particle properties. */ -class -ParticleReductionFunctor final : public ComputeDiagFunctor +class ParticleReductionFunctor final : public ComputeDiagFunctor { public: /** Constructor. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 8ced83d3594..2be9d2ffb36 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -144,8 +144,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const amrex::ParticleReal uz = p.rdata(PIdx::uz) / PhysConst::c; const amrex::ParticleReal upstream = ptd.m_runtime_rdata[iupstream][pind]; amrex::Real filter; - if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz, upstream) == 0._rt)) filter = 0._rt; - else filter = 1._rt; + if ((do_filter) && (filter_fn(xw, yw, zw, ux, uy, uz, upstream) == 0._rt)) {filter = 0._rt; + } else { filter = 1._rt; } amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(p.rdata(PIdx::w) * filter)); }); // Divide value by number of particles for average. Set average to zero if there are no particles @@ -156,8 +156,8 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, amrex::Array4 const& a_ppc = ppc_mf.const_array(mfi); amrex::ParallelFor(box, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - if (a_ppc(i,j,k,0) == 0) a_red(i,j,k,0) = 0; - else a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); + if (a_ppc(i,j,k,0) == 0) { a_red(i,j,k,0) = 0; + } else { a_red(i,j,k,0) = a_red(i,j,k,0) / a_ppc(i,j,k,0); } }); } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H index 46f8a3be251..bc0c8b9f270 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.H @@ -8,8 +8,7 @@ /** * \brief Functor to compute charge density rho into mf_out */ -class -RhoFunctor final : public ComputeDiagFunctor +class RhoFunctor final : public ComputeDiagFunctor { public: @@ -28,6 +27,7 @@ public: */ RhoFunctor (int lev, amrex::IntVect crse_ratio, + bool apply_rz_psatd_filter = false, int species_index = -1, bool convertRZmodes2cartesian = true, int ncomp = 1); @@ -45,6 +45,9 @@ private: // Level on which source MultiFab mf_src is defined in RZ geometry int const m_lev; + // Whether to apply k-space filtering of charge density in the diagnostics output in RZ PSATD + bool m_apply_rz_psatd_filter; + // Species index to dump rho per species const int m_species_index; diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index c36e31c0f93..340cc635fea 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -20,11 +20,13 @@ RhoFunctor::RhoFunctor (const int lev, const amrex::IntVect crse_ratio, + bool apply_rz_psatd_filter, const int species_index, bool convertRZmodes2cartesian, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), + m_apply_rz_psatd_filter(apply_rz_psatd_filter), m_species_index(species_index), m_convertRZmodes2cartesian(convertRZmodes2cartesian) {} @@ -62,7 +64,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i // Apply k-space filtering when using the PSATD solver if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - if (WarpX::use_kspace_filter) { + if (WarpX::use_kspace_filter && m_apply_rz_psatd_filter) { auto & solver = warpx.get_spectral_solver_fp(m_lev); const SpectralFieldIndex& Idx = solver.m_spectral_index; solver.ForwardTransform(m_lev, *rho, Idx.rho_new); @@ -70,6 +72,8 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i solver.BackwardTransform(m_lev, *rho, Idx.rho_new); } } +#else + amrex::ignore_unused(m_apply_rz_psatd_filter); #endif InterpolateMFForDiag(mf_dst, *rho, dcomp, warpx.DistributionMap(m_lev), diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 5d5a300dea9..c0d2a9f0d53 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -75,7 +75,7 @@ public: * * Derived classes MUST implement this function, and it must allocate m_all_field_functors * and fill it with ComputeDiagFunctor objects. - * The functions is called at intiailization and when the domain is decomposed + * The functions is called at initialization and when the domain is decomposed * during the simulation to load-balance. * \param[in] lev level on which the vector of unique_ptrs to field functors is initialized. */ @@ -114,7 +114,7 @@ public: /** Whether the last timestep is always dumped */ [[nodiscard]] bool DoDumpLastTimestep () const {return m_dump_last_timestep;} /** Returns the number of snapshots used in BTD. For Full-Diagnostics, the value is 1*/ - int getnumbuffers() {return m_num_buffers;} + [[nodiscard]] int getnumbuffers() const {return m_num_buffers;} /** Time in lab-frame associated with the ith snapshot * \param[in] i_buffer index of the buffer */ @@ -239,13 +239,13 @@ protected: amrex::Vector< std::string > m_pfield_varnames; /** Names of species for which to output particle field diagnostics */ std::vector< std::string > m_pfield_species; - /** Whether to do averaging for each of the particle fied diagnostics */ + /** Whether to do averaging for each of the particle field diagnostics */ std::vector< bool > m_pfield_do_average; /** Species indices corresponding to elements of m_pfield_varnames */ std::vector< int > m_pfield_species_index; /** List of the parser strings for the particle field diagnostics */ std::vector< std::string > m_pfield_strings; - /** Whether to use a filter function on particles before calculating particle fied diagnostics */ + /** Whether to use a filter function on particles before calculating particle field diagnostics */ std::vector< bool> m_pfield_dofilter; /** List of parser strings for pre-average filtering for the particle field diagnostics */ std::vector< std::string > m_pfield_filter_strings; @@ -309,11 +309,6 @@ protected: /** Vector of pointers to functors to compute particle output per species*/ amrex::Vector< std::unique_ptr > m_all_particle_functors; - /** Vector of total number of particles previously flushed, per species, per snapshot. - * The first vector is for total number of snapshots and second vector loops - * over the total number of species selected for diagnostics. - */ - amrex::Vector< amrex::Vector > m_totalParticles_flushed_already; /** Vector of total number of particles in the buffer, per species, per snapshot. * The first vector is for total number of snapshots and second vector loops * over the total number of species selected for diagnostics. diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index dec9c83b5b4..cb2b0981c94 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -574,9 +574,7 @@ Diagnostics::FilterComputePackFlush (int step, bool force_flush) } for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - if ( !DoDump (step, i_buffer, force_flush) ) continue; + if ( !DoDump (step, i_buffer, force_flush) ) { continue; } Flush(i_buffer, force_flush); } - - } diff --git a/Source/Diagnostics/FieldIO.H b/Source/Diagnostics/FieldIO.H index 1f8bfe5346a..138a1317e08 100644 --- a/Source/Diagnostics/FieldIO.H +++ b/Source/Diagnostics/FieldIO.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_FielIO_H_ -#define WARPX_FielIO_H_ +#ifndef WARPX_FieldIO_H_ +#define WARPX_FieldIO_H_ #include @@ -53,4 +53,4 @@ getReversedVec( const amrex::IntVect& v ); std::vector getReversedVec( const amrex::Real* v ); -#endif // WARPX_FielIO_H_ +#endif // WARPX_FieldIO_H_ diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index 403e9df7857..65741e4ff20 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -24,8 +24,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const = 0; + bool isLastBTDFlush = false) const = 0; FlushFormat () = default; virtual ~FlushFormat() = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index 228e4bc5cf6..9d8d3fcd7d2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -41,8 +41,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; #ifdef AMREX_USE_ASCENT /** \brief Do in-situ visualization for particle data. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 980047e3b46..0024122ebf9 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -21,7 +21,7 @@ FlushFormatAscent::WriteToFile ( const bool /*use_pinned_pc*/, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { #ifdef AMREX_USE_ASCENT WARPX_PROFILE("FlushFormatAscent::WriteToFile()"); @@ -94,9 +94,6 @@ FlushFormatAscent::WriteParticles(const amrex::Vector& particle_di // get names of real comps std::map real_comps_map = pc->getParticleComps(); - // WarpXParticleContainer compile-time extra AoS attributes (Real): 0 - // WarpXParticleContainer compile-time extra AoS attributes (int): 0 - // WarpXParticleContainer compile-time extra SoA attributes (Real): PIdx::nattribs // not an efficient search, but N is small... for(int j = 0; j < PIdx::nattribs; ++j) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index f6aad226d75..5c26ac97f61 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -28,8 +28,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const final; + bool isLastBTDFlush = false) const final; void CheckpointParticles (const std::string& dir, const amrex::Vector& particle_diags) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 2247a7eaadf..b083e60529f 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -39,7 +39,7 @@ FlushFormatCheckpoint::WriteToFile ( bool /*isBTD*/, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()"); @@ -172,14 +172,14 @@ FlushFormatCheckpoint::CheckpointParticles ( const std::string& dir, const amrex::Vector& particle_diags) const { - for (auto& part_diag: particle_diags) { + for (const auto& part_diag: particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); Vector real_names; Vector int_names; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -189,9 +189,12 @@ FlushFormatCheckpoint::CheckpointParticles ( #endif // get the names of the real comps - real_names.resize(pc->NumRealComps()); + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = pc->getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // and the int comps int_names.resize(pc->NumIntComps()); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index 88380407f5e..141760ac2a3 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -40,8 +40,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; ~FlushFormatOpenPMD () override = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index e15a7065f3a..e0c8c4ef2d6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -27,8 +27,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) std::string openpmd_backend {"default"}; pp_diag_name.query("openpmd_backend", openpmd_backend); // pick first available backend if default is chosen - if( openpmd_backend == "default" ) + if( openpmd_backend == "default" ) { openpmd_backend = WarpXOpenPMDFileType(); + } pp_diag_name.add("openpmd_backend", openpmd_backend); @@ -37,12 +38,13 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) const bool encodingDefined = pp_diag_name.query("openpmd_encoding", openpmd_encoding); openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; - if ( openpmd_encoding == "v" ) + if ( openpmd_encoding == "v" ) { encoding = openPMD::IterationEncoding::variableBased; - else if ( openpmd_encoding == "g" ) + } else if ( openpmd_encoding == "g" ) { encoding = openPMD::IterationEncoding::groupBased; - else if ( openpmd_encoding == "f" ) + } else if ( openpmd_encoding == "f" ) { encoding = openPMD::IterationEncoding::fileBased; + } std::string diag_type_str; pp_diag_name.get("diag_type", diag_type_str); @@ -65,8 +67,9 @@ FlushFormatOpenPMD::FlushFormatOpenPMD (const std::string& diag_name) { bool openpmd_tspf = false; const bool tspfDefined = pp_diag_name.query("openpmd_tspf", openpmd_tspf); - if ( tspfDefined && openpmd_tspf ) + if ( tspfDefined && openpmd_tspf ) { encoding = openPMD::IterationEncoding::fileBased; + } } // ADIOS2 operator type & parameters @@ -123,7 +126,7 @@ FlushFormatOpenPMD::WriteToFile ( const bool use_pinned_pc, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& full_BTD_snapshot, - bool isLastBTDFlush, const amrex::Vector& totalParticlesFlushedAlready) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()"); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); @@ -148,8 +151,9 @@ FlushFormatOpenPMD::WriteToFile ( int output_iteration = iteration[0]; // in backtransformed diagnostics (BTD), we dump into a series of labframe // snapshots - if( isBTD ) + if( isBTD ) { output_iteration = snapshotID; + } // Set step and output directory name. m_OpenPMDPlotWriter->SetStep(output_iteration, prefix, file_min_digits, isBTD); @@ -160,7 +164,7 @@ FlushFormatOpenPMD::WriteToFile ( // particles: all (reside only on locally finest level) m_OpenPMDPlotWriter->WriteOpenPMDParticles( - particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready); + particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush); // signal that no further updates will be written to this iteration m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 5e7d9b7b8e5..c62056b8907 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -35,8 +35,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** Write general info of the run into the plotfile */ void WriteJobInfo(const std::string& dir) const; @@ -46,12 +45,12 @@ public: const std::string& plotfilename, bool plot_raw_fields_guards) const; /** \brief Write particles data to file. - * \param[in] filename name of output directory + * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level * \param[in] isBTD whether this is a back-transformed diagnostic */ - void WriteParticles(const std::string& filename, + void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, bool isBTD = false) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 37a5e6cd79b..bf9073f05e9 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -65,7 +65,7 @@ FlushFormatPlotfile::WriteToFile ( const bool /*use_pinned_pc*/, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& /*full_BTD_snapshot*/, - bool isLastBTDFlush, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()"); auto & warpx = WarpX::GetInstance(); @@ -86,7 +86,7 @@ FlushFormatPlotfile::WriteToFile ( Vector rfs; const VisMF::Header::Version current_version = VisMF::GetHeaderVersion(); VisMF::SetHeaderVersion(amrex::VisMF::Header::Version_v1); - if (plot_raw_fields) rfs.emplace_back("raw_fields"); + if (plot_raw_fields) { rfs.emplace_back("raw_fields"); } amrex::WriteMultiLevelPlotfile(filename, nlev, amrex::GetVecOfConstPtrs(mf), varnames, geom, @@ -245,8 +245,9 @@ FlushFormatPlotfile::WriteWarpXHeader( HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | std::ofstream::trunc | std::ofstream::binary); - if( ! HeaderFile.good()) + if( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); + } HeaderFile.precision(17); @@ -339,10 +340,10 @@ FlushFormatPlotfile::WriteWarpXHeader( void FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, - const amrex::Real time, bool isBTD) const + const amrex::Real time, + bool isBTD) const { - - for (auto& part_diag : particle_diags) { + for (const auto& part_diag : particle_diags) { WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? @@ -354,8 +355,8 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, Vector int_flags; Vector real_flags; + // note: positions skipped here, since we reconstruct a plotfile SoA from them real_names.push_back("weight"); - real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); @@ -365,14 +366,21 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, #endif // get the names of the real comps - real_names.resize(tmp.NumRealComps()); + + // note: skips the mandatory AMREX_SPACEDIM positions for pure SoA + real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM); auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) { real_names[x.second+PIdx::nattribs] = x.first; } + for (auto const& x : runtime_rnames) { + real_names[x.second + PIdx::nattribs - AMREX_SPACEDIM] = x.first; + } // plot any "extra" fields by default real_flags = part_diag.m_plot_flags; real_flags.resize(tmp.NumRealComps(), 1); + // note: skip the mandatory AMREX_SPACEDIM positions for pure SoA + real_flags.erase(real_flags.begin(), real_flags.begin() + AMREX_SPACEDIM); + // and the names int_names.resize(tmp.NumIntComps()); auto runtime_inames = tmp.getParticleRuntimeiComps(); @@ -486,7 +494,7 @@ WriteCoarseVector( const std::string field_name, const int lev, const bool plot_guards ) { IntVect ng(0); - if (plot_guards) ng = Fx_fp->nGrowVect(); + if (plot_guards) { ng = Fx_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -523,7 +531,7 @@ WriteCoarseScalar( const std::string field_name, const int icomp ) { IntVect ng(0); - if (plot_guards) ng = F_fp->nGrowVect(); + if (plot_guards) { ng = F_fp->nGrowVect(); } if (lev == 0) { // No coarse field for level 0: instead write a MultiFab @@ -546,7 +554,7 @@ FlushFormatPlotfile::WriteAllRawFields( const bool plot_raw_fields, const int nlevels, const std::string& plotfilename, const bool plot_raw_fields_guards) const { - if (!plot_raw_fields) return; + if (!plot_raw_fields) { return; } auto & warpx = WarpX::GetInstance(); for (int lev = 0; lev < nlevels; ++lev) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 54eb7099ba4..d2ec9a5a4e0 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -61,8 +61,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** \brief Do in-situ visualization for particle data. * \param[in] particle_diags Each element of this vector handles output of 1 species. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index e162b8b3121..348e1da4a00 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -53,14 +53,13 @@ FlushFormatSensei::WriteToFile ( bool plot_raw_fields, bool plot_raw_fields_guards, const bool use_pinned_pc, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, - const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/, - const amrex::Vector& totalParticlesFlushedAlready) const + const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { amrex::ignore_unused( geom, nlev, prefix, file_min_digits, plot_raw_fields, plot_raw_fields_guards, - use_pinned_pc, - totalParticlesFlushedAlready); + use_pinned_pc + ); #ifndef AMREX_USE_SENSEI_INSITU amrex::ignore_unused(varnames, mf, iteration, time, particle_diags, diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 4464d46ebac..42b60c213f2 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -6,8 +6,7 @@ #include -class -FullDiagnostics final : public Diagnostics +class FullDiagnostics final : public Diagnostics { public: FullDiagnostics (int i, std::string name); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index c471beb05b3..a011d1c1ecf 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,5 +1,4 @@ #include "FullDiagnostics.H" - #include "ComputeDiagFunctors/CellCenterFunctor.H" #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" @@ -133,8 +132,9 @@ FullDiagnostics::Flush ( int i_buffer, bool /* force_flush */ ) auto & warpx = WarpX::GetInstance(); m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, m_file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); FlushRaw(); @@ -147,7 +147,7 @@ FullDiagnostics::FlushRaw () {} bool FullDiagnostics::DoDump (int step, int /*i_buffer*/, bool force_flush) { - if (m_already_done) return false; + if (m_already_done) { return false; } if ( force_flush || (m_intervals.contains(step+1)) ){ m_already_done = true; return true; @@ -268,14 +268,14 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } } else if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("rho"), ncomp); } } else if ( m_varnames_fields[comp].rfind("rho_", 0) == 0 ){ // Initialize rho functor to dump rho per species - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_rho_per_species_index[i], + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i], false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("rho_") + m_all_species_names[m_rho_per_species_index[i]], ncomp); @@ -360,7 +360,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) { #ifdef WARPX_DIM_RZ - if (!m_dump_rz_modes) return; + if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); @@ -443,7 +443,7 @@ FullDiagnostics::AddRZModesToDiags (int lev) } // rho if (rho_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp_multimodefab); + m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp_multimodefab); icomp += 1; AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); } @@ -507,11 +507,13 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { diag_dom.setLo(idim, std::max(m_lo[idim],warpx.Geom(lev).ProbLo(idim)) ); diag_dom.setHi(idim, std::min(m_hi[idim],warpx.Geom(lev).ProbHi(idim)) ); if ( std::fabs(warpx.Geom(lev).ProbLo(idim) - diag_dom.lo(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } if ( std::fabs(warpx.Geom(lev).ProbHi(idim) - diag_dom.hi(idim)) - > warpx.Geom(lev).CellSize(idim) ) + > warpx.Geom(lev).CellSize(idim) ) { use_warpxba = false; + } // User-defined value for coarsening should be an integer divisor of // blocking factor at level, lev. This assert is not relevant and thus @@ -573,7 +575,7 @@ FullDiagnostics::InitializeBufferData (int i_buffer, int lev, bool restart ) { ba.coarsen(m_crse_ratio); // Generate a new distribution map if the physical m_lo and m_hi for the output // is different from the lo and hi physical co-ordinates of the simulation domain. - if (!use_warpxba) dmap = amrex::DistributionMapping{ba}; + if (!use_warpxba) { dmap = amrex::DistributionMapping{ba}; } // Allocate output MultiFab for diagnostics. The data will be stored at cell-centers. const int ngrow = (m_format == "sensei" || m_format == "ascent") ? 1 : 0; int const ncomp = static_cast(m_varnames.size()); @@ -638,10 +640,10 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); } else if ( m_varnames[comp].rfind("rho_", 0) == 0 ){ // Initialize rho functor to dump rho per species - m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_rho_per_species_index[i]); + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); diff --git a/Source/Diagnostics/MultiDiagnostics.H b/Source/Diagnostics/MultiDiagnostics.H index f9907b8bdc6..d220396ed12 100644 --- a/Source/Diagnostics/MultiDiagnostics.H +++ b/Source/Diagnostics/MultiDiagnostics.H @@ -38,7 +38,7 @@ public: /** Start a new iteration, i.e., dump has not been done yet. */ void NewIteration (); Diagnostics& GetDiag(int idiag) {return *alldiags[idiag]; } - int GetTotalDiags() {return ndiags;} + [[nodiscard]] int GetTotalDiags() const {return ndiags;} DiagTypes diagstypes(int idiag) {return diags_types[idiag];} private: /** Vector of pointers to all diagnostics */ diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 5a50dfb6565..ea14919c713 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -70,9 +70,9 @@ MultiDiagnostics::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diag_type_str == "Full" || diag_type_str == "BackTransformed" || diag_type_str == "BoundaryScraping", ".diag_type must be Full or BackTransformed or BoundaryScraping"); - if (diag_type_str == "Full") diags_types[i] = DiagTypes::Full; - if (diag_type_str == "BackTransformed") diags_types[i] = DiagTypes::BackTransformed; - if (diag_type_str == "BoundaryScraping") diags_types[i] = DiagTypes::BoundaryScraping; + if (diag_type_str == "Full") { diags_types[i] = DiagTypes::Full; } + if (diag_type_str == "BackTransformed") { diags_types[i] = DiagTypes::BackTransformed; } + if (diag_type_str == "BoundaryScraping") { diags_types[i] = DiagTypes::BoundaryScraping; } } } @@ -82,11 +82,13 @@ MultiDiagnostics::FilterComputePackFlush (int step, bool force_flush, bool BackT int i = 0; for (auto& diag : alldiags){ if (BackTransform) { - if (diags_types[i] == DiagTypes::BackTransformed) + if (diags_types[i] == DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } else { - if (diags_types[i] != DiagTypes::BackTransformed) + if (diags_types[i] != DiagTypes::BackTransformed) { diag->FilterComputePackFlush (step, force_flush); + } } ++i; } diff --git a/Source/Diagnostics/OpenPMDHelpFunction.H b/Source/Diagnostics/OpenPMDHelpFunction.H index 9db4b9fb194..d2f2c4f9f9d 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.H +++ b/Source/Diagnostics/OpenPMDHelpFunction.H @@ -14,7 +14,25 @@ #include +/** Determine the preferred file ending if unspecified + * + * @return file ending without the "." + */ std::string WarpXOpenPMDFileType (); +#ifdef WARPX_USE_OPENPMD +/** Determine how many particles were already written in this species and step + * + * This checks for a particle species the current size of the id attribute, if it exists, + * and if it does it takes its extent as the number of particles already on disk. + * + * Note that this checks declared size, not necessarily written size. + * + * @return exisitng extent of the "id" attribute or zero. + */ +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies); +#endif + #endif // WARPX_OPENPMDHELPFUNCTION_H_ diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index a898c97b6b4..6170249b52b 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -27,3 +27,23 @@ WarpXOpenPMDFileType () #endif // WARPX_USE_OPENPMD return openPMDFileType; } + +#ifdef WARPX_USE_OPENPMD +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies) +{ + const auto *const scalar = openPMD::RecordComponent::SCALAR; + + unsigned long ParticleFlushOffset = 0; + + if (currSpecies.contains("id")) { + if (currSpecies["id"].contains(scalar)) { + if (!currSpecies["id"][scalar].empty()) { + ParticleFlushOffset = currSpecies["id"][scalar].getExtent().at(0); + } + } + } + + return ParticleFlushOffset; +} +#endif diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 7ca5e6541d7..a8bb9303fe1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -160,7 +160,7 @@ MultiParticleContainer::Restart (const std::string& dir) ); } - for (int j = PIdx::nattribs; j < nr; ++j) { + for (int j = PIdx::nattribs-AMREX_SPACEDIM; j < nr; ++j) { const auto& comp_name = real_comp_names[j]; auto current_comp_names = pc->getParticleComps(); auto search = current_comp_names.find(comp_name); diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index f5037e37bc0..018ddef5463 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -29,7 +29,7 @@ public: std::string m_beam_name; /** - * This function computes beam relevant quantites. + * This function computes beam relevant quantities. * * @param[in] step current time step */ diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index 2a29748b3e1..fbe67ba6fc5 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -214,14 +214,21 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_RZ) +#if defined(WARPX_DIM_3D) + const ParticleReal p_pos1 = p.pos(1); + const ParticleReal p_x_mean = p_pos0*p_w; + const ParticleReal p_y_mean = p_pos1*p_w; +#elif defined(WARPX_DIM_RZ) const ParticleReal p_theta = p.rdata(PIdx::theta); const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#else - const ParticleReal p_pos1 = p.pos(1); +#elif defined(WARPX_DIM_XZ) const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; + const ParticleReal p_y_mean = 0; +#elif defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(p_pos0); + const ParticleReal p_x_mean = 0; + const ParticleReal p_y_mean = 0; #endif const ParticleReal p_z_mean = p.pos(index_z)*p_w; @@ -263,7 +270,7 @@ void BeamRelevant::ComputeDiags (int step) if (w_sum < std::numeric_limits::min() ) { - for (auto& item: m_data) item = 0.0_rt; + for (auto& item: m_data) { item = 0.0_rt; } return; } diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index b9698d5ffd2..fe2040e50d2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -94,7 +94,7 @@ FieldMomentum::FieldMomentum (std::string rd_name) void FieldMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 9d35c1896b3..24ad0e64ea8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -340,8 +340,8 @@ bool FieldProbe::ProbeInDomain () const auto & warpx = WarpX::GetInstance(); int const lev = 0; const amrex::Geometry& gm = warpx.Geom(lev); - const auto prob_lo = gm.ProbLo(); - const auto prob_hi = gm.ProbHi(); + const auto *const prob_lo = gm.ProbLo(); + const auto *const prob_hi = gm.ProbHi(); /* * Determine if probe exists within simulation boundaries. During 2D simulations, @@ -431,8 +431,6 @@ void FieldProbe::ComputeDiags (int step) { const auto getPosition = GetParticlePosition(pti); auto setPosition = SetParticlePosition(pti); - const auto& aos = pti.GetArrayOfStructs(); - const auto* AMREX_RESTRICT m_structs = aos().dataPtr(); auto const np = pti.numParticles(); if (update_particles_moving_window) @@ -482,6 +480,8 @@ void FieldProbe::ComputeDiags (int step) ParticleReal* const AMREX_RESTRICT part_Bz = attribs[FieldProbePIdx::Bz].dataPtr(); ParticleReal* const AMREX_RESTRICT part_S = attribs[FieldProbePIdx::S].dataPtr(); + auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); + const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const std::array &dx = WarpX::CellSize(lev); @@ -556,7 +556,7 @@ void FieldProbe::ComputeDiags (int step) amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); long idx = ip*noutputs; - dvp[idx++] = m_structs[ip].id(); + dvp[idx++] = amrex::ParticleIDWrapper{idcpu[ip]}; // all particles created on IO cpu dvp[idx++] = xp; dvp[idx++] = yp; dvp[idx++] = zp; @@ -600,7 +600,7 @@ void FieldProbe::ComputeDiags (int step) // IO processor sums values from length_array to get size of total output array. /* displs records the size of each m_data as well as previous displs. This array - * tells Gatherv where in the m_data_out array allocation to write incomming data. */ + * tells Gatherv where in the m_data_out array allocation to write incoming data. */ long total_data_size = 0; amrex::Vector displs_vector; if (amrex::ParallelDescriptor::IOProcessor()) { @@ -628,14 +628,15 @@ void FieldProbe::ComputeDiags (int step) void FieldProbe::WriteToFile (int step) const { - if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) return; + if (!(ProbeInDomain() && amrex::ParallelDescriptor::IOProcessor())) { return; } // loop over num valid particles to find the lowest particle ID for later sorting auto first_id = static_cast(m_data_out[0]); for (long int i = 0; i < m_valid_particles; i++) { - if (m_data_out[i*noutputs] < first_id) + if (m_data_out[i*noutputs] < first_id) { first_id = static_cast(m_data_out[i*noutputs]); + } } // Create a new array to store probe data ordered by id, which will be printed to file. diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H index c85bf8fd541..7d59ade5dc6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H @@ -24,7 +24,14 @@ struct FieldProbePIdx { enum { - Ex = 0, Ey, Ez, +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + Ex, Ey, Ez, Bx, By, Bz, S, //!< the Poynting vector #ifdef WARPX_DIM_RZ @@ -40,9 +47,14 @@ struct FieldProbePIdx * nattribs tells the particle container to allot 7 SOA values. */ class FieldProbeParticleContainer - : public amrex::ParticleContainer<0, 0, FieldProbePIdx::nattribs> + : public amrex::ParticleContainerPureSoA { public: + static constexpr int NStructReal = 0; + static constexpr int NStructInt = 0; + static constexpr int NReal = FieldProbePIdx::nattribs; + static constexpr int NInt = 0; + FieldProbeParticleContainer (amrex::AmrCore* amr_core); ~FieldProbeParticleContainer() override = default; @@ -52,9 +64,9 @@ public: FieldProbeParticleContainer& operator= ( FieldProbeParticleContainer&& ) = default; //! amrex iterator for our number of attributes - using iterator = amrex::ParIter<0, 0, FieldProbePIdx::nattribs, 0>; + using iterator = amrex::ParIterSoA; //! amrex iterator for our number of attributes (read-only) - using const_iterator = amrex::ParConstIter<0, 0, FieldProbePIdx::nattribs, 0>; + using const_iterator = amrex::ParConstIterSoA; //! similar to WarpXParticleContainer::AddNParticles but does not include u(x,y,z) void AddNParticles (int lev, amrex::Vector const & x, amrex::Vector const & y, amrex::Vector const & z); diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp index 9a943759542..7e7aecb9167 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp @@ -59,7 +59,7 @@ using namespace amrex; FieldProbeParticleContainer::FieldProbeParticleContainer (AmrCore* amr_core) - : ParticleContainer<0, 0, FieldProbePIdx::nattribs>(amr_core->GetParGDB()) + : ParticleContainerPureSoA(amr_core->GetParGDB()) { SetParticleSize(); } @@ -89,33 +89,15 @@ FieldProbeParticleContainer::AddNParticles (int lev, * is then coppied to the permament tile which is stored on the particle * (particle_tile). */ + using PinnedTile = typename ContainerLike::ParticleTileType; - using PinnedTile = ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); for (int i = 0; i < np; i++) { - ParticleType p; - p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); -#if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(y); - p.pos(0) = x[i]; - p.pos(1) = z[i]; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(x, y); - p.pos(0) = z[i]; -#endif - - // write position, cpu id, and particle id to particle - pinned_tile.push_back(p); + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + idcpu_data.push_back(amrex::SetParticleIDandCPU(ParticleType::NextID(), ParallelDescriptor::MyProc())); } // write Real attributes (SoA) to particle initialized zero @@ -125,7 +107,13 @@ FieldProbeParticleContainer::AddNParticles (int lev, #ifdef WARPX_DIM_RZ pinned_tile.push_back_real(FieldProbePIdx::theta, np, 0.0); #endif - +#if !defined (WARPX_DIM_1D_Z) + pinned_tile.push_back_real(FieldProbePIdx::x, x); +#endif +#if defined (WARPX_DIM_3D) + pinned_tile.push_back_real(FieldProbePIdx::y, y); +#endif + pinned_tile.push_back_real(FieldProbePIdx::z, z); pinned_tile.push_back_real(FieldProbePIdx::Ex, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ey, np, 0.0); pinned_tile.push_back_real(FieldProbePIdx::Ez, np, 0.0); @@ -142,7 +130,7 @@ FieldProbeParticleContainer::AddNParticles (int lev, /* * Redistributes particles to their appropriate tiles if the box - * structure of the simulation changes to accomodate data more + * structure of the simulation changes to accommodate data more * efficiently. */ Redistribute(); diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 7843527cbf5..b4e07b51982 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -56,8 +56,7 @@ namespace auto const & plev = pc.GetParticles(lev); auto const & ptile = plev.at(box_index); - auto const & aos = ptile.GetArrayOfStructs(); - auto const np = aos.numParticles(); + auto const np = ptile.numParticles(); num_macro_particles += np; } @@ -87,7 +86,7 @@ void LoadBalanceCosts::ComputeDiags (int step) int nBoxes = 0; for (int lev = 0; lev < nLevels; ++lev) { - const auto cost = WarpX::getCosts(lev); + auto *const cost = WarpX::getCosts(lev); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( cost, "ERROR: costs are not initialized on level " + std::to_string(lev) + " !"); nBoxes += cost->size(); @@ -282,7 +281,7 @@ void LoadBalanceCosts::WriteToFile (int step) const // get a reference to WarpX instance auto& warpx = WarpX::GetInstance(); - if (!ParallelDescriptor::IOProcessor()) return; + if (!ParallelDescriptor::IOProcessor()) { return; } // final step is a special case, fill jagged array with NaN if (m_intervals.nextContains(step+1) > warpx.maxStep()) @@ -347,7 +346,7 @@ void LoadBalanceCosts::WriteToFile (int step) const while (std::getline(ss, token, m_sep[0])) { cnt += 1; - if (ss.peek() == m_sep[0]) ss.ignore(); + if (ss.peek() == m_sep[0]) { ss.ignore(); } } // 2 columns for step, time; then nBoxes*nDatafields columns for data; diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index a132c135471..157b4bfb368 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -88,7 +88,7 @@ ParticleEnergy::ParticleEnergy (std::string rd_name) void ParticleEnergy::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index 51bc1a600f9..ed5cce1d7e0 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -31,7 +31,7 @@ public: std::string m_species_name; /** - * This funciton computes the particle extrema + * This function computes the particle extrema * * @param[in] step current time step */ diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index a0fa821e580..7acde30761c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -158,7 +158,7 @@ ParticleHistogram::ParticleHistogram (std::string rd_name): void ParticleHistogram::ComputeDiags (int step) { // Judge if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // get a reference to WarpX instance auto & warpx = WarpX::GetInstance(); @@ -228,14 +228,16 @@ void ParticleHistogram::ComputeDiags (int step) auto const upstream = d_ups[i]; // don't count a particle if it is filtered out - if (do_parser_filter) - if (fun_filterparser(t, x, y, z, ux, uy, uz, upstream) == 0._rt) + if (do_parser_filter) { + if (fun_filterparser(t, x, y, z, ux, uy, uz, upstream) == 0._rt) { return; + } + } // continue function if particle is not filtered out auto const f = fun_partparser(t, x, y, z, ux, uy, uz,upstream); // determine particle bin int const bin = int(Math::floor((f-bin_min)/bin_size)); - if ( bin<0 || bin>=num_bins ) return; // discard if out-of-range + if ( bin<0 || bin>=num_bins ) { return; } // discard if out-of-range // add particle to histogram bin //! @todo performance: on CPU, we are probably faster by @@ -265,11 +267,11 @@ void ParticleHistogram::ComputeDiags (int step) Real f_max = 0.0_rt; for ( int i = 0; i < m_bin_num; ++i ) { - if ( m_data[i] > f_max ) f_max = m_data[i]; + if ( m_data[i] > f_max ) { f_max = m_data[i]; } } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_max > std::numeric_limits::min() ) m_data[i] /= f_max; + if ( f_max > std::numeric_limits::min() ) { m_data[i] /= f_max; } } return; } @@ -284,7 +286,7 @@ void ParticleHistogram::ComputeDiags (int step) } for ( int i = 0; i < m_bin_num; ++i ) { - if ( f_area > std::numeric_limits::min() ) m_data[i] /= f_area; + if ( f_area > std::numeric_limits::min() ) { m_data[i] /= f_area; } } return; } diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 98697380a4a..6ba85502819 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -35,6 +35,9 @@ public: /// File type std::string m_openpmd_backend {"default"}; + /// minimum number of digits for file suffix (file-based only supported now for now) */ + int m_file_min_digits = 6; + /// number of bins on the abscissa and ordinate int m_bin_num_abs; int m_bin_num_ord; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index dfb53b7e2c3..473a53715e6 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -60,9 +60,11 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) ParmParse pp_rd_name(rd_name); pp_rd_name.query("openpmd_backend", m_openpmd_backend); + pp_rd_name.query("file_min_digits", m_file_min_digits); // pick first available backend if default is chosen - if( m_openpmd_backend == "default" ) + if( m_openpmd_backend == "default" ) { m_openpmd_backend = WarpXOpenPMDFileType(); + } pp_rd_name.add("openpmd_backend", m_openpmd_backend); // read species @@ -130,7 +132,7 @@ ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) void ParticleHistogram2D::ComputeDiags (int step) { // Judge if the diags should be done at this step - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // resize data array Array tlo{0,0}; // lower bounds @@ -215,9 +217,11 @@ void ParticleHistogram2D::ComputeDiags (int step) auto const uz = d_uz[i] / PhysConst::c; // don't count a particle if it is filtered out - if (do_parser_filter) - if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) + if (do_parser_filter) { + if(!static_cast(fun_filterparser(t, x, y, z, ux, uy, uz, w))) { return; + } + } // continue function if particle is not filtered out auto const f_abs = fun_partparser_abs(t, x, y, z, ux, uy, uz, w); @@ -226,10 +230,10 @@ void ParticleHistogram2D::ComputeDiags (int step) // determine particle bin int const bin_abs = int(Math::floor((f_abs-bin_min_abs)/bin_size_abs)); - if ( bin_abs<0 || bin_abs>=num_bins_abs ) return; // discard if out-of-range + if ( bin_abs<0 || bin_abs>=num_bins_abs ) { return; } // discard if out-of-range int const bin_ord = int(Math::floor((f_ord-bin_min_ord)/bin_size_ord)); - if ( bin_ord<0 || bin_ord>=num_bins_ord ) return; // discard if out-of-range + if ( bin_ord<0 || bin_ord>=num_bins_ord ) { return; } // discard if out-of-range amrex::Real &data = d_table(bin_abs, bin_ord); amrex::HostDevice::Atomic::Add(&data, weight); @@ -257,10 +261,22 @@ void ParticleHistogram2D::WriteToFile (int step) const // only IO processor writes if ( !ParallelDescriptor::IOProcessor() ) { return; } + // TODO: support different filename templates + std::string filename = "openpmd"; + // TODO: support also group-based encoding + const std::string fileSuffix = std::string("_%0") + std::to_string(m_file_min_digits) + std::string("T"); + filename = filename.append(fileSuffix).append(".").append(m_openpmd_backend); + + std::string filepath = m_path + m_rd_name + "/" + filename; + // transform paths for Windows + #ifdef _WIN32 + filepath = openPMD::auxiliary::replace_all(filepath, "/", "\\"); + #endif + // Create the OpenPMD series auto series = io::Series( - m_path + m_rd_name + "/openpmd_%06T." + m_openpmd_backend, - io::Access::APPEND); + filepath, + io::Access::CREATE); auto i = series.iterations[step + 1]; // record auto f_mesh = i.meshes["data"]; diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5b370d29979..5f278d48f14 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -116,7 +116,7 @@ ParticleMomentum::ParticleMomentum (std::string rd_name) void ParticleMomentum::ComputeDiags (int step) { // Check if the diags should be done - if (!m_intervals.contains(step+1)) return; + if (!m_intervals.contains(step+1)) { return; } // Get MultiParticleContainer class object const auto & mypc = WarpX::GetInstance().GetPartContainer(); diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 01847a1ff12..09aa85586be 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -94,7 +94,7 @@ public: * This function queries deprecated input parameters and aborts * the run if one of them is specified. */ - void BackwardCompatibility (); + void BackwardCompatibility () const; }; diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 758b48b7262..ae6dae25a71 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -86,7 +86,7 @@ void ReducedDiags::LoadBalance () // load balancing operations } -void ReducedDiags::BackwardCompatibility () +void ReducedDiags::BackwardCompatibility () const { const amrex::ParmParse pp_rd_name(m_rd_name); std::vector backward_strings; @@ -116,7 +116,7 @@ void ReducedDiags::WriteToFile (int step) const ofs << WarpX::GetInstance().gett_new(0); // loop over data size and write - for (const auto& item : m_data) ofs << m_sep << item; + for (const auto& item : m_data) { ofs << m_sep << item; } // end loop over data size diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 787a95ce412..4613a70b3b0 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -76,7 +76,7 @@ RhoMaximum::RhoMaximum (std::string rd_name) for (int lev = 0; lev < nLevel; ++lev) { // Initialize functors for the charge density of each charged species - m_rho_functors[lev].push_back(std::make_unique(lev, crse_ratio, i)); + m_rho_functors[lev].push_back(std::make_unique(lev, crse_ratio, false, i)); } } } diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index ddadc70d343..71a585aea35 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -44,7 +44,7 @@ using namespace amrex; /* \brief * The functions creates the slice for diagnostics based on the user-input. - * The slice can be 1D, 2D, or 3D and it inherts the index type of the underlying data. + * The slice can be 1D, 2D, or 3D and it inherits the index type of the underlying data. * The implementation assumes that the slice is aligned with the coordinate axes. * The input parameters are modified if the user-input does not comply with requirements of coarsenability or if the slice extent is not contained within the simulation domain. * First a slice multifab (smf) with cell size equal to that of the simulation grid is created such that it extends from slice.dim_lo to slice.dim_hi and shares the same index space as the source multifab (mf) @@ -151,7 +151,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, const amrex::IntVect nghost_vect(AMREX_D_DECL(nghost, nghost, nghost)); ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); - // inteprolate if required on refined slice // + // interpolate if required on refined slice // if (interpolate == 1 ) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); @@ -309,7 +309,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, slice_cc_nd_box.setLo( idim, slice_realbox.lo(idim) ); slice_cc_nd_box.setHi( idim, slice_realbox.hi(idim) ); - if ( slice_cr_ratio[idim] > 1) slice_cr_ratio[idim] = 1; + if ( slice_cr_ratio[idim] > 1) { slice_cr_ratio[idim] = 1; } // check for interpolation -- compute index lo with floor and ceil if ( slice_cc_nd_box.lo(idim) - real_box.lo(idim) >= fac ) { @@ -349,7 +349,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, } else { - // moving realbox.lo and reabox.hi to nearest coarsenable grid point // + // moving realbox.lo and realbox.hi to nearest coarsenable grid point // auto index_lo = static_cast(floor(((slice_realbox.lo(idim) + very_small_number - (real_box.lo(idim))) / dom_geom[0].CellSize(idim))) ); auto index_hi = static_cast(ceil(((slice_realbox.hi(idim) - very_small_number diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 2ae990851e2..91409bc294d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -66,7 +66,7 @@ WarpX::GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, in ParallelDescriptor::ReadAndBcastFile(DMFileName, fileCharPtr); const std::string fileCharPtrString(fileCharPtr.dataPtr()); std::istringstream DMFile(fileCharPtrString, std::istringstream::in); - if ( ! DMFile.good()) amrex::FileOpenFailed(DMFileName); + if ( ! DMFile.good()) { amrex::FileOpenFailed(DMFileName); } DMFile.exceptions(std::ios_base::failbit | std::ios_base::badbit); int nprocs_in_checkpoint; @@ -382,11 +382,13 @@ WarpX::InitFromCheckpoint () if (do_pml) { for (int lev = 0; lev < nlevs; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); + } #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) - if (pml_rz[lev]) + if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); + } #endif } } diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 4675e219fdd..6c904790e15 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -41,10 +41,10 @@ class WarpXParticleCounter { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParIterSoA; WarpXParticleCounter (ParticleContainer* pc); - unsigned long GetTotalNumParticles () {return m_Total;} + [[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;} std::vector m_ParticleOffsetAtRank; std::vector m_ParticleSizeAtRank; @@ -77,7 +77,7 @@ class WarpXOpenPMDPlot { public: using ParticleContainer = typename WarpXParticleContainer::ContainerLike; - using ParticleIter = typename amrex::ParConstIter<0, 0, PIdx::nattribs, 0, amrex::PinnedArenaAllocator>; + using ParticleIter = typename amrex::ParConstIterSoA; /** Initialize openPMD I/O routines * @@ -125,8 +125,7 @@ public: amrex::Real time, bool use_pinned_pc = false, bool isBTD = false, - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector()); + bool isLastBTDFlush = false); /** Write out all openPMD fields for all active MR levels * @@ -205,15 +204,15 @@ private: * @param[in] varname name from WarpX * @param[out] field_name field name for openPMD-api output * @param[in] comp_name comp name for openPMD-api output - * @param[in] is_theta_mode indicate if this field will be output with theta - * modes (instead of a reconstructed 2D slice) + * @param[in] var_in_theta_mode indicate if this field will be output with theta + * modes (instead of a reconstructed 2D slice) */ void GetMeshCompNames ( int meshLevel, const std::string& varname, std::string& field_name, std::string& comp_name, - bool is_theta_mode + bool var_in_theta_mode ) const; /** This function sets up the entries for storing the particle positions and global IDs @@ -290,9 +289,9 @@ private: * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles + * @param[inout] ParticleFlushOffset previously flushed number of particles in BTD * @param[in] isBTD is this a backtransformed diagnostics (BTD) write? * @param[in] isLastBTDFlush is this the last time we will flush this BTD station? - * @param[in] ParticleFlushOffset previously flushed number of particles in BTD */ void DumpToFile (ParticleContainer* pc, const std::string& name, @@ -304,8 +303,7 @@ private: amrex::ParticleReal charge, amrex::ParticleReal mass, bool isBTD = false, - bool isLastBTDFlush = false, - int ParticleFlushOffset = 0); + bool isLastBTDFlush = false); /** Get the openPMD-api filename for openPMD::Series * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 01a0f14e9ef..057f38c853e 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -10,6 +10,7 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "FieldIO.H" #include "Particles/Filter/FilterFunctors.H" +#include "Particles/NamedComponentParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/RelativeCellPosition.H" @@ -18,11 +19,9 @@ #include "WarpX.H" #include "OpenPMDHelpFunction.H" -#include #include #include -#include #include #include #include @@ -100,8 +99,9 @@ namespace detail std::string const & engine_type, std::map< std::string, std::string > const & engine_parameters) { - if (operator_type.empty() && engine_type.empty()) + if (operator_type.empty() && engine_type.empty()) { return "{}"; + } std::string options; std::string top_block; @@ -111,7 +111,7 @@ namespace detail std::string op_parameters; for (const auto& kv : operator_parameters) { - if (!op_parameters.empty()) op_parameters.append(",\n"); + if (!op_parameters.empty()) { op_parameters.append(",\n"); } op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -119,7 +119,7 @@ namespace detail std::string en_parameters; for (const auto& kv : engine_parameters) { - if (!en_parameters.empty()) en_parameters.append(",\n"); + if (!en_parameters.empty()) { en_parameters.append(",\n"); } en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ .append("\"").append(kv.first).append("\": ") /* key */ .append("\"").append(kv.second).append("\""); /* value (as string) */ @@ -154,8 +154,9 @@ namespace detail } ] })END"; - if (!engine_type.empty() || !en_parameters.empty()) + if (!engine_type.empty() || !en_parameters.empty()) { op_block += ","; + } } // end operator string block // add the engine string block @@ -170,8 +171,9 @@ namespace detail "type": ")END"; en_block += engine_type + "\""; - if(!en_parameters.empty()) + if(!en_parameters.empty()) { en_block += ","; + } } // non-default engine parameters @@ -304,33 +306,37 @@ namespace detail getUnitDimension ( std::string const & record_name ) { - if( (record_name == "position") || (record_name == "positionOffset") ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "momentum" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -1.} - }; - else if( record_name == "charge" ) return { - {openPMD::UnitDimension::T, 1.}, - {openPMD::UnitDimension::I, 1.} - }; - else if( record_name == "mass" ) return { - {openPMD::UnitDimension::M, 1.} - }; - else if( record_name == "E" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -3.}, - {openPMD::UnitDimension::I, -1.}, - }; - else if( record_name == "B" ) return { - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::I, -1.}, - {openPMD::UnitDimension::T, -2.} - }; - else return {}; + if( (record_name == "position") || (record_name == "positionOffset") ) { + return {{openPMD::UnitDimension::L, 1.}}; + } else if( record_name == "momentum" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -1.}}; + } else if( record_name == "charge" ) { + return {{openPMD::UnitDimension::T, 1.}, + {openPMD::UnitDimension::I, 1.}}; + } else if( record_name == "mass" ) { + return {{openPMD::UnitDimension::M, 1.}}; + } else if( record_name == "weighting" ) { // NOLINT(bugprone-branch-clone) +#if defined(WARPX_DIM_1D_Z) + return {{openPMD::UnitDimension::L, -2.}}; +#elif defined(WARPX_DIM_XZ) + return {{openPMD::UnitDimension::L, -1.}}; +#else // 3D and RZ + return {}; +#endif + } else if( record_name == "E" ) { + return {{openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -3.}, + {openPMD::UnitDimension::I, -1.}}; + } else if( record_name == "B" ) { + return {{openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::I, -1.}, + {openPMD::UnitDimension::T, -2.}}; + } else { // NOLINT(bugprone-branch-clone) + return {}; + } } /** \brief For a given field that is to be written to an openPMD file, @@ -449,7 +455,7 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) // default close is true bool callClose = true; // close BTD file only when isLastBTDFlush is true - if (isBTD and !isLastBTDFlush) callClose = false; + if (isBTD and !isLastBTDFlush) { callClose = false; } if (callClose) { if (m_Series) { GetIteration(m_CurrentStep, isBTD).close(); @@ -472,8 +478,9 @@ void WarpXOpenPMDPlot::CloseStep (bool isBTD, bool isLastBTDFlush) void WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) { - if( isBTD && m_Series != nullptr ) + if( isBTD && m_Series != nullptr ) { return; // already open for this snapshot (aka timestep in lab frame) + } // either for the next ts file, // or init a single file for all ts @@ -482,10 +489,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) // close a previously open series before creating a new one // see ADIOS1 limitation: https://github.com/openPMD/openPMD-api/pull/686 - if ( m_Encoding == openPMD::IterationEncoding::fileBased ) + if ( m_Encoding == openPMD::IterationEncoding::fileBased ) { m_Series = nullptr; - else if ( m_Series != nullptr ) + } else if ( m_Series != nullptr ) { return; + } if (amrex::ParallelDescriptor::NProcs() > 1) { #if defined(AMREX_USE_MPI) @@ -504,8 +512,9 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) m_Series->setIterationEncoding( m_Encoding ); // input file / simulation setup author - if( !m_authors.empty()) + if( !m_authors.empty()) { m_Series->setAuthor( m_authors ); + } // more natural naming for PIC m_Series->setMeshesPath( "fields" ); // conform to ED-PIC extension of openPMD @@ -517,9 +526,11 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) void WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& particle_diags, - const amrex::Real time, const bool use_pinned_pc, - const bool isBTD, const bool isLastBTDFlush, - const amrex::Vector& totalParticlesFlushedAlready) + const amrex::Real time, + const bool use_pinned_pc, + const bool isBTD, + const bool isLastBTDFlush +) { WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); @@ -527,9 +538,11 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { WarpXParticleContainer* pc = particle_diags[i].getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = particle_diags[i].getPinnedParticleContainer(); - if (isBTD || use_pinned_pc) - if (!pinned_pc->isDefined()) + if (isBTD || use_pinned_pc) { + if (!pinned_pc->isDefined()) { continue; // Skip to the next particle container + } + } PinnedMemoryParticleContainer tmp = (isBTD || use_pinned_pc) ? pinned_pc->make_alike() : @@ -544,13 +557,19 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // see openPMD ED-PIC extension for namings // note: an underscore separates the record name from its component // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); real_names.push_back("weighting"); real_names.push_back("momentum_x"); real_names.push_back("momentum_y"); real_names.push_back("momentum_z"); -#ifdef WARPX_DIM_RZ - real_names.push_back("theta"); -#endif + // get the names of the real comps real_names.resize(tmp.NumRealComps()); auto runtime_rnames = tmp.getParticleRuntimeComps(); @@ -587,50 +606,44 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { particle_diags[i].m_diag_domain); if (isBTD || use_pinned_pc) { - tmp.copyParticles(*pinned_pc, true); - particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass); + particlesConvertUnits(ConvertDirection::WarpX_to_SI, pinned_pc, mass); + using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; + tmp.copyParticles(*pinned_pc, + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(p, engine) * geometry_filter(p, engine); + }, true); + particlesConvertUnits(ConvertDirection::SI_to_WarpX, pinned_pc, mass); } else { particlesConvertUnits(ConvertDirection::WarpX_to_SI, pc, mass); using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; tmp.copyParticles(*pc, - [random_filter,uniform_filter,parser_filter,geometry_filter] - AMREX_GPU_HOST_DEVICE - (const SrcData& src, int ip, const amrex::RandomEngine& engine) - { - const SuperParticleType& p = src.getSuperParticle(ip); - return random_filter(p, engine) * uniform_filter(p, engine) - * parser_filter(src,ip, engine) * geometry_filter(p, engine); - }, true); + [random_filter,uniform_filter,parser_filter,geometry_filter] + AMREX_GPU_HOST_DEVICE + (const SrcData& src, int ip, const amrex::RandomEngine& engine) + { + const SuperParticleType& p = src.getSuperParticle(ip); + return random_filter(p, engine) * uniform_filter(p, engine) + * parser_filter(src,ip, engine) * geometry_filter(p, engine); + }, true); particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. - { - if (isBTD) { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - totalParticlesFlushedAlready[i] - ); - } else { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - 0 - ); - } - } + DumpToFile(&tmp, + particle_diags.at(i).getSpeciesName(), + m_CurrentStep, + real_flags, + int_flags, + real_names, int_names, + pc->getCharge(), pc->getMass(), + isBTD, isLastBTDFlush + ); } } @@ -645,8 +658,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, amrex::ParticleReal const charge, amrex::ParticleReal const mass, const bool isBTD, - const bool isLastBTDFlush, - int ParticleFlushOffset) { + const bool isLastBTDFlush +) +{ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized"); AMREX_ALWAYS_ASSERT(write_real_comp.size() == pc->NumRealComps()); @@ -660,6 +674,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, openPMD::Iteration currIteration = GetIteration(iteration, isBTD); openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; + // only BTD writes multiple times into the same step, zero for other methods + unsigned long ParticleFlushOffset = isBTD ? num_already_flushed(currSpecies) : 0; + // prepare data structures the first time BTD has non-zero particles // we set some of them to zero extent, so we need to time that well bool const is_first_flush_with_particles = num_dump_particles > 0 && ParticleFlushOffset == 0; @@ -684,8 +701,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // we will set up empty particles unless it's BTD, where we might add some in a following buffer dump // during this setup, we mark some particle properties as constant and potentially zero-sized bool doParticleSetup = true; - if (isBTD) + if (isBTD) { doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; + } // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 @@ -707,7 +725,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++) { auto offset = static_cast( counter.m_ParticleOffsetAtRank[currentLevel] ); // For BTD, the offset include the number of particles already flushed - if (isBTD) offset += ParticleFlushOffset; + if (isBTD) { offset += ParticleFlushOffset; } for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) { auto const numParticleOnTile = pti.numParticles(); auto const numParticleOnTile64 = static_cast( numParticleOnTile ); @@ -715,80 +733,11 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, // Do not call storeChunk() with zero-sized particle tiles: // https://github.com/openPMD/openPMD-api/issues/1147 // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - if (numParticleOnTile == 0) continue; + if (numParticleOnTile == 0) { continue; } contributed_particles = true; - // get position and particle ID from aos - // note: this implementation iterates the AoS 4x... - // if we flush late as we do now, we can also copy out the data in one go - const auto &aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile - { - // Save positions -#if defined(WARPX_DIM_RZ) - { - const std::shared_ptr z( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) - z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} - std::string const positionComponent = "z"; - currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); - } - - // reconstruct x and y from polar coordinates r, theta - auto const& soa = pti.GetStructOfArrays(); - amrex::ParticleReal const* theta = soa.GetRealData(PIdx::theta).dataPtr(); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(theta != nullptr, "openPMD: invalid theta pointer."); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, - "openPMD: theta and tile size do not match"); - { - const std::shared_ptr< amrex::ParticleReal > x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - const std::shared_ptr< amrex::ParticleReal > y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - for (auto i=0; i curr( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - curr.get()[i] = aos[i].pos(currDim); - } - std::string const positionComponent = positionComponents[currDim]; - currSpecies["position"][positionComponent].storeChunk(curr, {offset}, - {numParticleOnTile64}); - } -#endif - - // save particle ID after converting it to a globally unique ID - const std::shared_ptr ids( - new uint64_t[numParticleOnTile], - [](uint64_t const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - ids.get()[i] = ablastr::particles::localIDtoGlobal(static_cast(aos[i].id()), static_cast(aos[i].cpu())); - } - auto const scalar = openPMD::RecordComponent::SCALAR; - currSpecies["id"][scalar].storeChunk(ids, {offset}, {numParticleOnTile64}); - - } - // save "extra" particle properties in AoS and SoA + // save particle properties SaveRealProperty(pti, currSpecies, offset, @@ -812,7 +761,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") { for( auto & [record_name, record] : currSpecies ) { for( auto & [comp_name, comp] : record ) { - if (comp.constant()) continue; + if (comp.constant()) { continue; } auto dtype = comp.getDatatype(); switch (dtype) { @@ -862,7 +811,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, const unsigned long long np, bool const isBTD) const { std::string options = "{}"; - if (isBTD) options = "{ \"resizable\": true }"; + if (isBTD) { options = "{ \"resizable\": true }"; } auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto dtype_int = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); // @@ -889,32 +838,32 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, std::set< std::string > addedRecords; // add meta-data per record only once for (auto idx=0; idxNumRealComps(); idx++) { - auto ii = ParticleContainer::NStructReal + idx; // jump over extra AoS names - if (write_real_comp[ii]) { + if (write_real_comp[idx]) { // handle scalar and non-scalar records by name - const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[ii]); + const auto [record_name, component_name] = detail::name2openPMD(real_comp_names[idx]); auto currRecord = currSpecies[record_name]; // meta data for ED-PIC extension [[maybe_unused]] const auto [_, newRecord] = addedRecords.insert(record_name); if( newRecord ) { currRecord.setUnitDimension( detail::getUnitDimension(record_name) ); - if( record_name == "weighting" ) + if( record_name == "weighting" ) { currRecord.setAttribute( "macroWeighted", 1u ); - else + } else { currRecord.setAttribute( "macroWeighted", 0u ); - if( record_name == "momentum" || record_name == "weighting" ) + } + if( record_name == "momentum" || record_name == "weighting" ) { currRecord.setAttribute( "weightingPower", 1.0 ); - else + } else { currRecord.setAttribute( "weightingPower", 0.0 ); + } } } } for (auto idx=0; idx const& int_comp_names) const { - auto const numParticleOnTile = pti.numParticles(); - auto const numParticleOnTile64 = static_cast( numParticleOnTile ); - auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile - auto const& soa = pti.GetStructOfArrays(); - // first we concatinate the AoS into contiguous arrays - { - // note: WarpX does not yet use extra AoS Real attributes - for( auto idx=0; idx(numParticleOnTile); + auto const& soa = pti.GetStructOfArrays(); - const std::shared_ptr< amrex::ParticleReal > d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - - for( auto kk=0; kk x,y,z +#if defined(WARPX_DIM_RZ) + auto const * const r = soa.GetRealData(PIdx::x).data(); + auto const * const theta = soa.GetRealData(PIdx::theta).data(); + + if (write_real_comp[0]) { + std::shared_ptr const x( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (int i = 0; i < numParticleOnTile; ++i) { + x.get()[i] = r[i] * std::cos(theta[i]); + } + getComponentRecord(real_comp_names[0]).storeChunk( + x, {offset}, {numParticleOnTile64}); + } + if (write_real_comp[1]) { + std::shared_ptr const y( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (int i = 0; i < numParticleOnTile; ++i) { + y.get()[i] = r[i] * std::sin(theta[i]); + } + getComponentRecord(real_comp_names[1]).storeChunk( + y, {offset}, {numParticleOnTile64}); + } +#endif + + for (auto idx=0; idx(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); @@ -1016,7 +991,7 @@ WarpXOpenPMDPlot::SetupPos ( currSpecies["position"][comp].resetDataset( realType ); } - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].resetDataset( idType ); } @@ -1028,7 +1003,7 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( amrex::ParticleReal const mass) { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}); - auto const scalar = openPMD::RecordComponent::SCALAR; + const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles auto const positionComponents = detail::getParticlePositionComponentLabels(true); @@ -1122,6 +1097,8 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( return "Esirkepov"; case CurrentDepositionAlgo::Vay : return "Vay"; + case CurrentDepositionAlgo::Villasenor : + return "Villasenor"; default: return "directMorseNielson"; } @@ -1149,17 +1126,20 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, const auto HalfFieldBoundarySize = static_cast(fieldBoundary.size() / 2u); - for (auto i = 0; i < HalfFieldBoundarySize; ++i) - if (m_fieldPMLdirections.at(i)) + for (auto i = 0; i < HalfFieldBoundarySize; ++i) { + if (m_fieldPMLdirections.at(i)) { fieldBoundary.at(i) = "open"; + } + } - for (int i = 0; i < HalfFieldBoundarySize; ++i) + for (int i = 0; i < HalfFieldBoundarySize; ++i) { if (period.isPeriodic(i)) { fieldBoundary.at(2u * i) = "periodic"; fieldBoundary.at(2u * i + 1u) = "periodic"; particleBoundary.at(2u * i) = "periodic"; particleBoundary.at(2u * i + 1u) = "periodic"; } + } meshes.setAttribute("fieldSolver", []() { switch (WarpX::electromagnetic_solver_id) { @@ -1176,10 +1156,10 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, meshes.setAttribute("fieldBoundary", fieldBoundary); meshes.setAttribute("particleBoundary", particleBoundary); meshes.setAttribute("currentSmoothing", []() { - if (WarpX::use_filter) return "Binomial"; - else return "none"; + if (WarpX::use_filter) { return "Binomial"; } + else { return "none"; } }()); - if (WarpX::use_filter) + if (WarpX::use_filter) { meshes.setAttribute("currentSmoothingParameters", []() { std::stringstream ss; ss << "period=1;compensator=false"; @@ -1197,12 +1177,14 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, std::string currentSmoothingParameters = ss.str(); return currentSmoothingParameters; }()); + } meshes.setAttribute("chargeCorrection", []() { - if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check - else return "none"; + if (WarpX::do_dive_cleaning) { return "hyperbolic"; // TODO or "spectral" or something? double-check + } else { return "none"; } }()); - if (WarpX::do_dive_cleaning) + if (WarpX::do_dive_cleaning) { meshes.setAttribute("chargeCorrectionParameters", "period=1"); + } } @@ -1281,8 +1263,9 @@ WarpXOpenPMDPlot::GetMeshCompNames (int meshLevel, } } - if ( 0 == meshLevel ) + if ( 0 == meshLevel ) { return; + } field_name += std::string("_lvl").append(std::to_string(meshLevel)); } @@ -1365,19 +1348,21 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename, } // If there are no fields to be written, interrupt the function here - if ( varnames.empty() ) return; + if ( varnames.empty() ) { return; } // loop over levels up to output_levels // note: this is usually the finestLevel, not the maxLevel for (int lev=0; lev < output_levels; lev++) { amrex::Geometry full_geom = geom[lev]; - if( isBTD ) + if( isBTD ) { full_geom = full_BTD_snapshot; + } // setup is called once. So it uses property "period" from first // geometry for field levels. - if ( (0 == lev) && first_write_to_iteration ) + if ( (0 == lev) && first_write_to_iteration ) { SetupFields(meshes, full_geom); + } amrex::Box const & global_box = full_geom.Domain(); @@ -1509,8 +1494,9 @@ WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc): m_ParticleSizeAtRank[currentLevel] = numParticles; // adjust offset, it should be numbered after particles from previous levels - for (auto lv=0; lv(result.size()); for (int i=0; i const& phi, amrex::GpuArray const& dxi) noexcept { + #if (defined WARPX_DIM_3D) amrex::RealVect normal{0.0, 0.0, 0.0}; + for (int iic = 0; iic < 2; ++iic) { + for (int kk = 0; kk < 2; ++kk) { + for (int jj=0; jj< 2; ++jj) { + for (int ii = 0; ii < 2; ++ii) { + int icstart = ic + iic; + amrex::Real sign = (ii%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(jj%2); + int w2comp = static_cast(kk%2); + normal[0] += sign * phi(icstart + ii, j + jj, k + kk) * dxi[0] * Wc[0][wccomp] * W[1][w1comp] * W[2][w2comp]; + } + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int kk = 0; kk < 2; ++kk) { + for (int ii=0; ii< 2; ++ii) { + for (int jj = 0; jj < 2; ++jj) { + int jcstart = jc + iic; + amrex::Real sign = (jj%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + int w2comp = static_cast(kk%2); + normal[1] += sign * phi(i + ii, jcstart + jj, k + kk) * dxi[1] * W[0][w1comp] * Wc[1][wccomp] * W[2][w2comp]; + } + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int jj = 0; jj < 2; ++jj) { + for (int ii=0; ii< 2; ++ii) { + for (int kk = 0; kk < 2; ++kk) { + int kcstart = kc + iic; + amrex::Real sign = (kk%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + int w2comp = static_cast(jj%2); + normal[2] += sign * phi(i + ii, j + jj, kcstart + kk) * dxi[2] * W[0][w1comp] * W[1][w2comp] * Wc[2][wccomp]; + } + } + } + } - normal[0] -= phi(i, j , k ) * dxi[0] * W[1][0] * W[2][0]; - normal[0] += phi(i+1, j , k ) * dxi[0] * W[1][0] * W[2][0]; - normal[0] -= phi(i, j+1, k ) * dxi[0] * W[1][1] * W[2][0]; - normal[0] += phi(i+1, j+1, k ) * dxi[0] * W[1][1] * W[2][0]; - normal[0] -= phi(i, j , k+1) * dxi[0] * W[1][0] * W[2][1]; - normal[0] += phi(i+1, j , k+1) * dxi[0] * W[1][0] * W[2][1]; - normal[0] -= phi(i , j+1, k+1) * dxi[0] * W[1][1] * W[2][1]; - normal[0] += phi(i+1, j+1, k+1) * dxi[0] * W[1][1] * W[2][1]; - - normal[1] -= phi(i, j , k ) * dxi[1] * W[0][0] * W[2][0]; - normal[1] += phi(i , j+1, k ) * dxi[1] * W[0][0] * W[2][0]; - normal[1] -= phi(i+1, j , k ) * dxi[1] * W[0][1] * W[2][0]; - normal[1] += phi(i+1, j+1, k ) * dxi[1] * W[0][1] * W[2][0]; - normal[1] -= phi(i, j , k+1) * dxi[1] * W[0][0] * W[2][1]; - normal[1] += phi(i , j+1, k+1) * dxi[1] * W[0][0] * W[2][1]; - normal[1] -= phi(i+1, j , k+1) * dxi[1] * W[0][1] * W[2][1]; - normal[1] += phi(i+1, j+1, k+1) * dxi[1] * W[0][1] * W[2][1]; - - normal[2] -= phi(i , j , k ) * dxi[2] * W[0][0] * W[1][0]; - normal[2] += phi(i , j , k+1) * dxi[2] * W[0][0] * W[1][0]; - normal[2] -= phi(i+1, j , k ) * dxi[2] * W[0][1] * W[1][0]; - normal[2] += phi(i+1, j , k+1) * dxi[2] * W[0][1] * W[1][0]; - normal[2] -= phi(i, j+1, k ) * dxi[2] * W[0][0] * W[1][1]; - normal[2] += phi(i , j+1, k+1) * dxi[2] * W[0][0] * W[1][1]; - normal[2] -= phi(i+1, j+1, k ) * dxi[2] * W[0][1] * W[1][1]; - normal[2] += phi(i+1, j+1, k+1) * dxi[2] * W[0][1] * W[1][1]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::RealVect normal{0.0, 0.0}; + for (int iic = 0; iic < 2; ++iic) { + for (int jj=0; jj< 2; ++jj) { + for (int ii = 0; ii < 2; ++ii) { + int icstart = ic + iic; + amrex::Real sign = (ii%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(jj%2); + normal[0] += sign * phi(icstart + ii, j + jj, k) * dxi[0] * Wc[0][wccomp] * W[1][w1comp]; + } + } + } + for (int iic = 0; iic < 2; ++iic) { + for (int ii=0; ii< 2; ++ii) { + for (int jj = 0; jj < 2; ++jj) { + int jcstart = jc + iic; + amrex::Real sign = (jj%2)*2. - 1.; + int wccomp = static_cast(iic%2); + int w1comp = static_cast(ii%2); + normal[1] += sign * phi(i + ii, jcstart + jj, k) * dxi[1] * W[0][w1comp] * Wc[1][wccomp]; + } + } + } + amrex::ignore_unused(kc); - normal[0] -= phi(i, j , k) * dxi[0] * W[1][0]; - normal[0] += phi(i+1, j , k) * dxi[0] * W[1][0]; - normal[0] -= phi(i, j+1, k) * dxi[0] * W[1][1]; - normal[0] += phi(i+1, j+1, k) * dxi[0] * W[1][1]; - - normal[1] -= phi(i, j , k) * dxi[1] * W[0][0]; - normal[1] += phi(i , j+1, k) * dxi[1] * W[0][0]; - normal[1] -= phi(i+1, j , k) * dxi[1] * W[0][1]; - normal[1] += phi(i+1, j+1, k) * dxi[1] * W[0][1]; #else + amrex::ignore_unused(i, j, k, ic, jc, kc, W, Wc, phi, dxi); amrex::RealVect normal{0.0, 0.0}; - amrex::ignore_unused(i, j, k, W, phi, dxi); WARPX_ABORT_WITH_MESSAGE("Error: interp_distance not yet implemented in 1D"); + #endif return normal; } diff --git a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H index a78aa19458a..3099f5df43d 100644 --- a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H +++ b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H @@ -7,10 +7,12 @@ #ifndef PARTICLEBOUNDARYPROCESS_H_ #define PARTICLEBOUNDARYPROCESS_H_ +#include #include #include #include + namespace ParticleBoundaryProcess { struct NoOp { @@ -25,12 +27,11 @@ struct NoOp { struct Absorb { template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator() (const PData& ptd, int i, + void operator() (PData& ptd, int i, const amrex::RealVect& /*pos*/, const amrex::RealVect& /*normal*/, amrex::RandomEngine const& /*engine*/) const noexcept { - auto& p = ptd.m_aos[i]; - p.id() = -p.id(); + amrex::ParticleIDWrapper{ptd.m_idcpu[i]}.make_invalid(); } }; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index d6196c35f44..7d6934cbb9c 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -10,6 +10,7 @@ #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/Pusher/GetAndSetPosition.H" + #include #include @@ -38,7 +39,7 @@ * passed in to this function as an argument. This function can access the * position at which the particle hit the boundary, and also the associated * normal vector. Particles can be `absorbed` by setting their ids to negative - * to flag them for removal. Likewise, the can be reflected back into the domain + * to flag them for removal. Likewise, they can be reflected back into the domain * by modifying their data appropriately and leaving their ids alone. * * This version operates only at the specified level. @@ -82,7 +83,7 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t * passed in to this function as an argument. This function can access the * position at which the particle hit the boundary, and also the associated * normal vector. Particles can be `absorbed` by setting their ids to negative - * to flag them for removal. Likewise, the can be reflected back into the domain + * to flag them for removal. Likewise, they can be reflected back into the domain * by modifying their data appropriately and leaving their ids alone. * * This version operates over all the levels in the pc. @@ -170,45 +171,45 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t auto& tile = pti.GetParticleTile(); auto ptd = tile.getParticleTileData(); const auto np = tile.numParticles(); - amrex::Particle<0,0> * const particles = tile.GetArrayOfStructs()().data(); auto phi = (*distance_to_eb[lev])[pti].array(); // signed distance function amrex::ParallelForRNG( np, [=] AMREX_GPU_DEVICE (const int ip, amrex::RandomEngine const& engine) noexcept { // skip particles that are already flagged for removal - if (particles[ip].id() < 0) return; + if (!amrex::ParticleIDWrapper{ptd.m_idcpu[ip]}.is_valid()) return; amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights_nodal(xp, yp, zp, plo, dxi, i, j, k, W); - + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { - amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, phi, dxi); - - // the closest point on the surface to pos is pos - grad phi(pos) * phi(pos) + int ic, jc, kc; // Cell-centered indices + int nodal; + amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); + amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); + DistanceToEB::normalize(normal); amrex::RealVect pos; #if (defined WARPX_DIM_3D) - pos[0] = xp - normal[0]*phi_value; - pos[1] = yp - normal[1]*phi_value; - pos[2] = zp - normal[2]*phi_value; + pos[0] = xp; + pos[1] = yp; + pos[2] = zp; #elif (defined WARPX_DIM_XZ) - pos[0] = xp - normal[0]*phi_value; - pos[1] = zp - normal[1]*phi_value; + pos[0] = xp; + pos[1] = zp; #elif (defined WARPX_DIM_RZ) - pos[0] = std::sqrt(xp*xp + yp*yp) - normal[0]*phi_value; - pos[1] = zp - normal[1]*phi_value; + pos[0] = std::sqrt(xp*xp + yp*yp); + pos[1] = zp; #elif (defined WARPX_DIM_1D_Z) - pos[0] = zp - normal[0]*phi_value; + pos[0] = zp; #endif - DistanceToEB::normalize(normal); - f(ptd, ip, pos, normal, engine); + } }); } diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index 3ef0dae5e8e..a84d6e1e42b 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,5 +4,6 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp + WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index 275b8bfde5a..a9a877475b9 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,4 +1,5 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp +CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXComputeDt.cpp b/Source/Evolve/WarpXComputeDt.cpp index e52f92c4acb..c1a87166920 100644 --- a/Source/Evolve/WarpXComputeDt.cpp +++ b/Source/Evolve/WarpXComputeDt.cpp @@ -37,12 +37,13 @@ WarpX::ComputeDt () electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { std::stringstream errorMsg; - if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) + if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { errorMsg << "warpx.const_dt must be specified with the electrostatic solver."; - else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) + } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { errorMsg << "warpx.const_dt must be specified with the hybrid-PIC solver."; - else + } else { errorMsg << "warpx.const_dt must be specified when not using a field solver."; + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), errorMsg.str()); for (int lev=0; lev<=max_level; lev++) { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index b6f6ed83136..c215311cd47 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -118,54 +118,57 @@ WarpX::Evolve (int numsteps) } } - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - if (is_synchronized) { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + if (evolve_scheme == EvolveScheme::Explicit) { + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + + if (is_synchronized) { // Not called at each iteration, so exchange all guard cells FillBoundaryE(guard_cells.ng_alloc_EB); FillBoundaryB(guard_cells.ng_alloc_EB); - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - } else { - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) + { + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { // Beyond one step, we have E^{n} and B^{n}. // Particles have p^{n-1/2} and x^{n}. + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. // E and B are up-to-date inside the domain only FillBoundaryE(guard_cells.ng_FieldGather); FillBoundaryB(guard_cells.ng_FieldGather); - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) - FillBoundaryAux(guard_cells.ng_UpdateAux); + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) HybridPICDepositInitialRhoAndJ(); + ) { HybridPICDepositInitialRhoAndJ(); } // Run multi-physics modules: // ionization, Coulomb collisions, QED @@ -182,13 +185,18 @@ WarpX::Evolve (int numsteps) // gather fields, push particles, deposit sources, update fields ExecutePythonCallback("particleinjection"); - // Electrostatic or hybrid-PIC case: only gather fields and push - // particles, deposition and calculation of fields done further below - if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + OneStep_ImplicitPicard(cur_time); + } + else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) { + // Electrostatic or hybrid-PIC case: only gather fields and push + // particles, deposition and calculation of fields done further below const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); } // Electromagnetic case: multi-J algorithm else if (do_multi_J) @@ -220,31 +228,33 @@ WarpX::Evolve (int numsteps) // value of step in code (first step is 0) mypc->doResampling(istep[0]+1, verbose); - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } - - if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); + if (evolve_scheme == EvolveScheme::Explicit) { + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); + + if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { + // At the end of last step, push p by 0.5*dt to synchronize + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP(lev, 0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); + } + is_synchronized = true; } - is_synchronized = true; } for (int lev = 0; lev <= max_level; ++lev) { @@ -404,7 +414,7 @@ WarpX::Evolve (int numsteps) if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) || exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) ExecutePythonCallback("onbreaksignal"); + if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -425,7 +435,7 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("particlescraper"); ExecutePythonCallback("beforedeposition"); - PushParticlesandDepose(cur_time); + PushParticlesandDeposit(cur_time); ExecutePythonCallback("afterdeposition"); @@ -439,8 +449,8 @@ WarpX::OneStep_nosub (Real cur_time) // solve. // For extended PML: copy J from regular grid to PML, and damp J in PML - if (do_pml && pml_has_particles) CopyJPML(); - if (do_pml && do_pml_j_damping) DampJPML(); + if (do_pml && pml_has_particles) { CopyJPML(); } + if (do_pml && do_pml_j_damping) { DampJPML(); } ExecutePythonCallback("beforeEsolve"); @@ -467,10 +477,12 @@ WarpX::OneStep_nosub (Real cur_time) else { FillBoundaryE(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_afterPushPSATD, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } } } else { EvolveF(0.5_rt * dt[0], DtType::FirstHalf); @@ -506,8 +518,9 @@ WarpX::OneStep_nosub (Real cur_time) // E and B are up-to-date in the domain, but all guard cells are // outdated. - if (safe_guard_cells) + if (safe_guard_cells) { FillBoundaryB(guard_cells.ng_alloc_EB); + } } // !PSATD ExecutePythonCallback("afterEsolve"); @@ -549,7 +562,7 @@ void WarpX::SyncCurrentAndRho () { // TODO This works only without mesh refinement const int lev = 0; - if (use_filter) ApplyFilterJ(current_fp_vay, lev); + if (use_filter) { ApplyFilterJ(current_fp_vay, lev); } } } } @@ -597,17 +610,17 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Push particle from x^{n} to x^{n+1} // from p^{n-1/2} to p^{n+1/2} const bool skip_deposition = true; - PushParticlesandDepose(cur_time, skip_deposition); + PushParticlesandDeposit(cur_time, skip_deposition); // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // 2) Set the averaged fields to zero - if (WarpX::fft_do_time_averaging) PSATDEraseAverageFields(); + if (WarpX::fft_do_time_averaging) { PSATDEraseAverageFields(); } // 3) Deposit rho (in rho_new, since it will be moved during the loop) // (after checking that pointer to rho_fp on MR level 0 is not null) @@ -639,29 +652,29 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) } // Number of depositions for multi-J scheme - const int n_depose = WarpX::do_multi_J_n_depositions; + const int n_deposit = WarpX::do_multi_J_n_depositions; // Time sub-step for each multi-J deposition - const amrex::Real sub_dt = dt[0] / static_cast(n_depose); + const amrex::Real sub_dt = dt[0] / static_cast(n_deposit); // Whether to perform multi-J depositions on a time interval that spans // one or two full time steps (from n*dt to (n+1)*dt, or from n*dt to (n+2)*dt) - const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_depose : n_depose; + const int n_loop = (WarpX::fft_do_time_averaging) ? 2*n_deposit : n_deposit; // Loop over multi-J depositions - for (int i_depose = 0; i_depose < n_loop; i_depose++) + for (int i_deposit = 0; i_deposit < n_loop; i_deposit++) { // Move J from new to old if J is linear in time - if (J_in_time == JInTime::Linear) PSATDMoveJNewToJOld(); + if (J_in_time == JInTime::Linear) { PSATDMoveJNewToJOld(); } - const amrex::Real t_depose_current = (J_in_time == JInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_current = (J_in_time == JInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - const amrex::Real t_depose_charge = (rho_in_time == RhoInTime::Linear) ? - (i_depose-n_depose+1)*sub_dt : (i_depose-n_depose+0.5_rt)*sub_dt; + const amrex::Real t_deposit_charge = (rho_in_time == RhoInTime::Linear) ? + (i_deposit-n_deposit+1)*sub_dt : (i_deposit-n_deposit+0.5_rt)*sub_dt; - // Deposit new J at relative time t_depose_current with time step dt + // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; - mypc->DepositCurrent(current, dt[0], t_depose_current); + mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', // namely 'current_fp_nodal': SyncCurrent stores the result of its centering @@ -676,10 +689,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) if (rho_fp[0]) { // Move rho from new to old if rho is linear in time - if (rho_in_time == RhoInTime::Linear) PSATDMoveRhoNewToRhoOld(); + if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } - // Deposit rho at relative time t_depose_charge - mypc->DepositCharge(rho_fp, t_depose_charge); + // Deposit rho at relative time t_deposit_charge + mypc->DepositCharge(rho_fp, t_deposit_charge); // Filter, exchange boundary, and interpolate across levels SyncRho(rho_fp, rho_cp, charge_buf); // Forward FFT of rho @@ -696,13 +709,13 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Advance E,B,F,G fields in time and update the average fields PSATDPushSpectralFields(); - // Transform non-average fields E,B,F,G after n_depose pushes + // Transform non-average fields E,B,F,G after n_deposit pushes // (the relative time reached here coincides with an integer full time step) - if (i_depose == n_depose-1) + if (i_deposit == n_deposit-1) { PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } } @@ -722,9 +735,9 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } // Damp fields in PML before exchanging guard cells @@ -736,10 +749,12 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Exchange guard cells and synchronize nodal points FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) + if (WarpX::do_dive_cleaning || WarpX::do_pml_dive_cleaning) { FillBoundaryF(guard_cells.ng_alloc_F, WarpX::sync_nodal_points); - if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) + } + if (WarpX::do_divb_cleaning || WarpX::do_pml_divb_cleaning) { FillBoundaryG(guard_cells.ng_alloc_G, WarpX::sync_nodal_points); + } #else amrex::ignore_unused(cur_time); @@ -764,7 +779,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) * */ void -WarpX::OneStep_sub1 (Real curtime) +WarpX::OneStep_sub1 (Real cur_time) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electrostatic_solver_id == ElectrostaticSolverAlgo::None, @@ -778,10 +793,10 @@ WarpX::OneStep_sub1 (Real curtime) const int coarse_lev = 0; // i) Push particles and fields on the fine patch (first fine step) - PushParticlesandDepose(fine_lev, curtime, DtType::FirstHalf); + PushParticlesandDeposit(fine_lev, cur_time, DtType::FirstHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, 2*ncomps); @@ -809,7 +824,7 @@ WarpX::OneStep_sub1 (Real curtime) // ii) Push particles on the coarse patch and mother grid. // Push the fields on the coarse patch and mother grid // by only half a coarse step (first half) - PushParticlesandDepose(coarse_lev, curtime, DtType::Full); + PushParticlesandDeposit(coarse_lev, cur_time, DtType::Full); StoreCurrent(coarse_lev); AddCurrentFromFineLevelandSumBoundary(current_fp, current_cp, current_buf, coarse_lev); AddRhoFromFineLevelandSumBoundary(rho_fp, rho_cp, charge_buf, coarse_lev, 0, ncomps); @@ -839,10 +854,10 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryAux(guard_cells.ng_UpdateAux); // iv) Push particles and fields on the fine patch (second fine step) - PushParticlesandDepose(fine_lev, curtime+dt[fine_lev], DtType::SecondHalf); + PushParticlesandDeposit(fine_lev, cur_time + dt[fine_lev], DtType::SecondHalf); RestrictCurrentFromFineToCoarsePatch(current_fp, current_cp, fine_lev); RestrictRhoFromFineToCoarsePatch(rho_fp, rho_cp, fine_lev); - if (use_filter) ApplyFilterJ(current_fp, fine_lev); + if (use_filter) { ApplyFilterJ(current_fp, fine_lev); } SumBoundaryJ(current_fp, fine_lev, Geom(fine_lev).periodicity()); ApplyFilterandSumBoundaryRho(rho_fp, rho_cp, fine_lev, PatchType::fine, 0, ncomps); @@ -863,8 +878,9 @@ WarpX::OneStep_sub1 (Real curtime) FillBoundaryE(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryF(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); + } FillBoundaryB(fine_lev, PatchType::fine, guard_cells.ng_FieldSolver); // v) Push the fields on the coarse patch and mother grid @@ -910,13 +926,15 @@ WarpX::OneStep_sub1 (Real curtime) WarpX::sync_nodal_points); } DampPML(coarse_lev, PatchType::fine); - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryE(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } - if ( safe_guard_cells ) + if ( safe_guard_cells ) { FillBoundaryB(coarse_lev, PatchType::fine, guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); + } } void @@ -954,17 +972,18 @@ WarpX::doQEDEvents (int lev) #endif void -WarpX::PushParticlesandDepose (amrex::Real cur_time, bool skip_deposition) +WarpX::PushParticlesandDeposit (amrex::Real cur_time, bool skip_current, PushType push_type) { // Evolve particles to p^{n+1/2} and x^{n+1} - // Depose current, j^{n+1/2} + // Deposit current, j^{n+1/2} for (int lev = 0; lev <= finest_level; ++lev) { - PushParticlesandDepose(lev, cur_time, DtType::Full, skip_deposition); + PushParticlesandDeposit(lev, cur_time, DtType::Full, skip_current, push_type); } } void -WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_deposition) +WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, bool skip_current, + PushType push_type) { amrex::MultiFab* current_x = nullptr; amrex::MultiFab* current_y = nullptr; @@ -991,15 +1010,15 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, } mypc->Evolve(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], *current_x, *current_y, *current_z, current_buf[lev][0].get(), current_buf[lev][1].get(), current_buf[lev][2].get(), rho_fp[lev].get(), charge_buf[lev].get(), Efield_cax[lev][0].get(), Efield_cax[lev][1].get(), Efield_cax[lev][2].get(), Bfield_cax[lev][0].get(), Bfield_cax[lev][1].get(), Bfield_cax[lev][2].get(), - cur_time, dt[lev], a_dt_type, skip_deposition); - if (! skip_deposition) { + cur_time, dt[lev], a_dt_type, skip_current, push_type); + if (! skip_current) { #ifdef WARPX_DIM_RZ // This is called after all particles have deposited their current and charge. ApplyInverseVolumeScalingToCurrentDensity(current_fp[lev][0].get(), current_fp[lev][1].get(), current_fp[lev][2].get(), lev); @@ -1029,9 +1048,9 @@ WarpX::PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type, if (do_fluid_species) { myfl->Evolve(lev, - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2], - rho_fp[lev].get(),*current_x, *current_y, *current_z, cur_time, skip_deposition); + *Efield_aux[lev][0], *Efield_aux[lev][1], *Efield_aux[lev][2], + *Bfield_aux[lev][0], *Bfield_aux[lev][1], *Bfield_aux[lev][2], + rho_fp[lev].get(), *current_x, *current_y, *current_z, cur_time, skip_current); } } } @@ -1082,8 +1101,8 @@ WarpX::applyMirrors(Real time) NullifyMF(Bz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_fp[lev]) NullifyMF(*F_fp[lev], lev, z_min, z_max); - if (G_fp[lev]) NullifyMF(*G_fp[lev], lev, z_min, z_max); + if (F_fp[lev]) { NullifyMF(*F_fp[lev], lev, z_min, z_max); } + if (G_fp[lev]) { NullifyMF(*G_fp[lev], lev, z_min, z_max); } if (lev>0) { @@ -1104,8 +1123,8 @@ WarpX::applyMirrors(Real time) NullifyMF(cBz, lev, z_min, z_max); // If div(E)/div(B) cleaning are used, set F/G field to zero - if (F_cp[lev]) NullifyMF(*F_cp[lev], lev, z_min, z_max); - if (G_cp[lev]) NullifyMF(*G_cp[lev], lev, z_min, z_max); + if (F_cp[lev]) { NullifyMF(*F_cp[lev], lev, z_min, z_max); } + if (G_cp[lev]) { NullifyMF(*G_cp[lev], lev, z_min, z_max); } } } } diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp new file mode 100644 index 00000000000..56ed26db1e2 --- /dev/null +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -0,0 +1,423 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::EvolveImplicitPicardInit (int lev) +{ + + if (lev == 0) { + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + } + + // Initialize MultiFabs to hold the E and B fields at the start of the time steps + // Only one refinement level is supported + const int nlevs_max = maxLevel() + 1; + Efield_n.resize(nlevs_max); + Efield_save.resize(nlevs_max); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_n.resize(nlevs_max); + Bfield_save.resize(nlevs_max); + } + + // The Efield_n and Bfield_n will hold the fields at the start of the time step. + // This is needed since in each iteration the fields are advanced from the values + // at the start of the step. + // The Efield_save and Bfield_save will hold the fields from the previous iteration, + // to check the change in the fields after the iterations to check for convergence. + // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then + // advance to the n+1 time level after the iterations complete. + AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); + AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); + AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); + AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); + AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); + AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); + AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); + AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); + AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); + AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); + AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); + } + +} + +void +WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) +{ + using namespace amrex::literals; + + // We have E^{n}. + // Particles have p^{n} and x^{n}. + // With full implicit, B^{n} + // With semi-implicit, B^{n-1/2} + + // Save the values at the start of the time step, + // copying particle data to x_n etc. + for (auto const& pc : *mypc) { + SaveParticlesAtImplicitStepStart (*pc, 0); + } + + // Save the fields at the start of the step + amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); + } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // Start the iterations + amrex::Real deltaE = 1._rt; + amrex::Real deltaB = 1._rt; + int iteration_count = 0; + while (iteration_count < max_picard_iterations && + (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { + iteration_count++; + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + bool skip_current = false; + PushType push_type = PushType::Implicit; + PushParticlesandDeposit(cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the E at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place + amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); + + // Updates Efield_fp so it holds the new E at n+1/2 + EvolveE(0.5_rt*dt[0]); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); + + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Save the B at n+1/2 from the previous iteration so that the change + // in this iteration can be calculated + amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); + amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); + } + + // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place + amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); + + // This updates Bfield_fp so it holds the new B at n+1/2 + EvolveB(0.5_rt*dt[0], DtType::Full); + // WarpX::sync_nodal_points is used to avoid instability + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); + } + + // The B field update needs + if (num_mirrors>0){ + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date from the mirrors + // B : guard cells are NOT up-to-date from the mirrors + } + + if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { + // Calculate the change in E and B from this iteration + // deltaE = abs(Enew - Eold)/max(abs(Enew)) + Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); + Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); + Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); + amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); + amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); + amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; + amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; + amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; + deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); + Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); + Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); + amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); + amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); + amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); + amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; + amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; + amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; + deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); + } else { + deltaB = 0.; + } + amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; + } + + // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold + // the new values at n+1/2 + } + + amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; + if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { + std::stringstream convergenceMsg; + convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; + if (require_picard_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + + // Advance particles to step n+1 + for (auto const& pc : *mypc) { + FinishImplicitParticleUpdate(*pc, 0); + } + + // Advance fields to step n+1 + // WarpX::sync_nodal_points is used to avoid instability + FinishImplicitFieldUpdate(Efield_fp, Efield_n); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + if (evolve_scheme == EvolveScheme::ImplicitPicard) { + FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + } + +} + +void +WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) +{ + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } +} + +void +WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) +{ + using namespace amrex::literals; + +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc.getParticleComps(); + + for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } +} + +void +WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, + amrex::Vector, 3 > >& Field_n) +{ + using namespace amrex::literals; + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); + amrex::Box tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); + amrex::Box tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); + }); + } + } +} diff --git a/Source/Evolve/WarpXPushType.H b/Source/Evolve/WarpXPushType.H new file mode 100644 index 00000000000..dbca64a398c --- /dev/null +++ b/Source/Evolve/WarpXPushType.H @@ -0,0 +1,18 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_PUSHTYPE_H_ +#define WARPX_PUSHTYPE_H_ + +// Specify which scheme to use for the particle advance +enum struct PushType : int +{ + Explicit = 0, // Use the standard leap-frog scheme + Implicit // Use the Crank-Nicolson scheme. + // See for example Eqs. 15-18 in Chen, JCP 407 (2020) 109228 +}; + +#endif // WARPX_PUSHTYPE_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 7bd982bb4e0..7fc6c9bf6da 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -52,7 +52,7 @@ namespace ElectrostaticSolver { void buildParsers (); void buildParsersEB (); - /* \brief Sets the EB potential string and updates the parsers + /* \brief Sets the EB potential string and updates the function parser * * \param [in] potential The string value of the potential */ diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 73dac03508b..bd3086c0438 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -105,8 +105,9 @@ WarpX::AddBoundaryField () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } // Allocate fields for charge and potential const int num_levels = max_level + 1; @@ -146,8 +147,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -181,7 +183,9 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); std::array beta; - for (int i=0 ; i < static_cast(beta.size()) ; i++) beta[i] = beta_pr[i]/PhysConst::c; // Normalize + for (int i=0 ; i < static_cast(beta.size()) ; i++) { + beta[i] = beta_pr[i]/PhysConst::c; // Normalize + } // Compute the potential phi, by solving the Poisson equation computePhi( rho, phi, beta, pc.self_fields_required_precision, @@ -201,8 +205,9 @@ WarpX::AddSpaceChargeFieldLabFrame () // Store the boundary conditions for the field solver if they haven't been // stored yet - if (!m_poisson_boundary_handler.bcs_set) + if (!m_poisson_boundary_handler.bcs_set) { m_poisson_boundary_handler.definePhiBCs(Geom(0)); + } #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -381,7 +386,7 @@ void WarpX::setPhiBC ( amrex::Vector>& phi ) const { // check if any dimension has non-periodic boundary conditions - if (!m_poisson_boundary_handler.has_non_periodic) return; + if (!m_poisson_boundary_handler.has_non_periodic) { return; } // get the boundary potentials at the current time amrex::Array phi_bc_values_lo; @@ -417,7 +422,7 @@ WarpX::setPhiBC ( amrex::Vector>& phi ) const // loop over dimensions for (int idim=0; idim, 3> > std::array const beta ) const { // return early if beta is 0 since there will be no B-field - if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) return; + if ((beta[0] == 0._rt) && (beta[1] == 0._rt) && (beta[2] == 0._rt)) { return; } for (int lev = 0; lev <= max_level; lev++) { @@ -973,7 +978,7 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo lobc[0] = LinOpBCType::Neumann; dirichlet_flag[0] = false; - // handle the r_max boundary explicity + // handle the r_max boundary explicitly if (WarpX::field_boundary_hi[0] == FieldBoundaryType::PEC) { hibc[0] = LinOpBCType::Dirichlet; dirichlet_flag[1] = true; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index b447d4e06a3..36be09696fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -106,13 +106,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) - coef2_z*Et(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Br(i,j,0,m) = coef1_z*Br(i,j,0,m) + coef2_z*Et(i,j+1,0,m); + } } }, @@ -120,13 +122,15 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( // At the +z boundary (innermost guard cell) if ( apply_hi_z && (j==domain_box.bigEnd(1)+1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) + coef2_z*Er(i,j,0,m); + } } // At the -z boundary (innermost guard cell) if ( apply_lo_z && (j==domain_box.smallEnd(1)-1) ){ - for (int m=0; m<2*nmodes-1; m++) + for (int m=0; m<2*nmodes-1; m++) { Bt(i,j,0,m) = coef1_z*Bt(i,j,0,m) - coef2_z*Er(i,j+1,0,m); + } } // At the +r boundary (innermost guard cell) if ( apply_hi_r && (i==domain_box.bigEnd(0)+1) ){ @@ -233,31 +237,39 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) + coef2_y * Ez(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_y * Bx(i,j,k) - coef2_y * Ez(i,j+1,k); + } // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) - coef2_z * Ey(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { Bx(i,j,k) = coef1_z * Bx(i,j,k) + coef2_z * Ey(i+1,j,k); + } #endif }, @@ -266,33 +278,41 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) - coef2_x * Ez(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_x * By(i,j,k) + coef2_x * Ez(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) + if ( apply_hi_z && ( k==domain_box.bigEnd(2)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) + if ( apply_lo_z && ( k==domain_box.smallEnd(2)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j,k+1); + } #elif WARPX_DIM_XZ // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_z && ( j==domain_box.bigEnd(1)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_z && ( j==domain_box.smallEnd(1)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z // At the +z boundary (innermost guard cell) - if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_z && ( i==domain_box.bigEnd(0)+1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) + coef2_z * Ex(i,j,k); + } // At the -z boundary (innermost guard cell) - if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_z && ( i==domain_box.smallEnd(0)-1 ) ) { By(i,j,k) = coef1_z * By(i,j,k) - coef2_z * Ex(i+1,j,k); + } #endif }, @@ -301,19 +321,23 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( #if (defined WARPX_DIM_3D || WARPX_DIM_XZ) // At the +x boundary (innermost guard cell) - if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) + if ( apply_hi_x && ( i==domain_box.bigEnd(0)+1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) + coef2_x * Ey(i,j,k); + } // At the -x boundary (innermost guard cell) - if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) + if ( apply_lo_x && ( i==domain_box.smallEnd(0)-1 ) ) { Bz(i,j,k) = coef1_x * Bz(i,j,k) - coef2_x * Ey(i+1,j,k); + } #endif #ifdef WARPX_DIM_3D // At the +y boundary (innermost guard cell) - if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) + if ( apply_hi_y && ( j==domain_box.bigEnd(1)+1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) - coef2_y * Ex(i,j,k); + } // At the -y boundary (innermost guard cell) - if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) + if ( apply_lo_y && ( j==domain_box.smallEnd(1)-1 ) ) { Bz(i,j,k) = coef1_y * Bz(i,j,k) + coef2_y * Ex(i,j+1,k); + } #elif WARPX_DIM_1D_Z ignore_unused(i,j,k); #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H index b6c5c0a8ce8..e49995557a0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H @@ -21,7 +21,7 @@ /** * This struct contains only static functions to initialize the stencil coefficients - * and to compute finite-difference derivatives for the Cartesian Yee algorithm. + * and to compute finite-difference derivatives for the Cylindrical Yee algorithm. */ struct CylindricalYeeAlgorithm { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d877db5253d..861b2648c1e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -139,22 +139,24 @@ class FiniteDifferenceSolver * \param[out] Efield vector of electric field MultiFabs updated at a given level * \param[in] Jfield vector of total current MultiFabs at a given level * \param[in] Jifield vector of ion current density MultiFabs at a given level + * \param[in] Jextfield vector of external current density MultiFabs at a given level * \param[in] Bfield vector of magnetic field MultiFabs at a given level * \param[in] rhofield scalar ion charge density Multifab at a given level * \param[in] Pefield scalar electron pressure MultiFab at a given level * \param[in] edge_lengths length of edges along embedded boundaries * \param[in] lev level number for the calculation - * \param[in] hybrid_pic_model instance of the hybrid-PIC model + * \param[in] hybrid_model instance of the hybrid-PIC model * \param[in] include_resistivity_term boolean flag for whether to include resistivity */ void HybridPICSolveE ( std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3>& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); /** @@ -233,11 +235,12 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template @@ -337,11 +340,12 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3>& Efield, std::array< std::unique_ptr, 3> const& Jfield, std::array< std::unique_ptr, 3> const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3> const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, std::array< std::unique_ptr, 3 > const& edge_lengths, - int lev, HybridPICModel const* hybrid_pic_model, + int lev, HybridPICModel const* hybrid_model, bool include_resistivity_term ); template diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 8923ec1f6a5..1bbc6c9b337 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -36,8 +36,9 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( m_grid_type{grid_type} { // return if not FDTD - if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) + if (fdtd_algo == ElectromagneticSolverAlgo::None || fdtd_algo == ElectromagneticSolverAlgo::PSATD) { return; + } // Calculate coefficients of finite-difference stencil #ifdef WARPX_DIM_RZ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 399177c82bb..23ef49b58cb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -46,6 +46,20 @@ public: void InitData (); + /** + * \brief + * Function to evaluate the external current expressions and populate the + * external current multifab. Note the external current can be a function + * of time and therefore this should be re-evaluated at every step. + */ + void GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths + ); + void GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev + ); + /** * \brief * Function to calculate the total current based on Ampere's law while @@ -93,6 +107,33 @@ public: std::array< std::unique_ptr, 3> const& edge_lengths, int lev, PatchType patch_type, bool include_resistivity_term); + void BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType a_dt_type, + amrex::IntVect ng, std::optional nodal_sync); + + void BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, int lev, DtType dt_type, + amrex::IntVect ng, std::optional nodal_sync); + + void FieldPush ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + amrex::IntVect ng, std::optional nodal_sync); + /** * \brief * Function to calculate the electron pressure at a given timestep type @@ -107,16 +148,16 @@ public: * charge density (and assumption of quasi-neutrality) using the user * specified electron equation of state. * - * \param[out] Pe scalar electron pressure MultiFab at a given level - * \param[in] rhofield scalar ion chrge density Multifab at a given level + * \param[out] Pe_filed scalar electron pressure MultiFab at a given level + * \param[in] rho_field scalar ion charge density Multifab at a given level */ void FillElectronPressureMF ( - std::unique_ptr const& Pe, - amrex::MultiFab* const& rhofield ); + std::unique_ptr const& Pe_field, + amrex::MultiFab* const& rho_field ) const; // Declare variables to hold hybrid-PIC model parameters /** Number of substeps to take when evolving B */ - int m_substeps = 100; + int m_substeps = 10; /** Electron temperature in eV */ amrex::Real m_elec_temp; @@ -131,12 +172,22 @@ public: /** Plasma resistivity */ std::string m_eta_expression = "0.0"; std::unique_ptr m_resistivity_parser; - amrex::ParserExecutor<1> m_eta; + amrex::ParserExecutor<2> m_eta; + bool m_resistivity_has_J_dependence = false; + + /** External current */ + std::string m_Jx_ext_grid_function = "0.0"; + std::string m_Jy_ext_grid_function = "0.0"; + std::string m_Jz_ext_grid_function = "0.0"; + std::array< std::unique_ptr, 3> m_J_external_parser; + std::array< amrex::ParserExecutor<4>, 3> m_J_external; + bool m_external_field_has_time_dependence = false; // Declare multifabs specifically needed for the hybrid-PIC model amrex::Vector< std::unique_ptr > rho_fp_temp; amrex::Vector, 3 > > current_fp_temp; amrex::Vector, 3 > > current_fp_ampere; + amrex::Vector, 3 > > current_fp_external; amrex::Vector< std::unique_ptr > electron_pressure_fp; // Helper functions to retrieve hybrid-PIC multifabs diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index a94593ded83..034bb71efbc 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -37,11 +37,16 @@ void HybridPICModel::ReadParameters () Abort("hybrid_pic_model.n0_ref should be specified if hybrid_pic_model.gamma != 1"); } - pp_hybrid.query("plasma_resistivity(rho)", m_eta_expression); + pp_hybrid.query("plasma_resistivity(rho,J)", m_eta_expression); utils::parser::queryWithParser(pp_hybrid, "n_floor", m_n_floor); // convert electron temperature from eV to J m_elec_temp *= PhysConst::q_e; + + // external currents + pp_hybrid.query("Jx_external_grid_function(x,y,z,t)", m_Jx_ext_grid_function); + pp_hybrid.query("Jy_external_grid_function(x,y,z,t)", m_Jy_ext_grid_function); + pp_hybrid.query("Jz_external_grid_function(x,y,z,t)", m_Jz_ext_grid_function); } void HybridPICModel::AllocateMFs (int nlevs_max) @@ -50,6 +55,7 @@ void HybridPICModel::AllocateMFs (int nlevs_max) rho_fp_temp.resize(nlevs_max); current_fp_temp.resize(nlevs_max); current_fp_ampere.resize(nlevs_max); + current_fp_external.resize(nlevs_max); } void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm, @@ -87,6 +93,15 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri WarpX::AllocInitMultiFab(current_fp_ampere[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp_ampere[z]", 0.0_rt); + // the external current density multifab is made nodal to avoid needing to interpolate + // to a nodal grid as has to be done for the ion and total current density multifabs + WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), + dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (ncomps == 1), @@ -101,14 +116,33 @@ void HybridPICModel::ClearLevel (int lev) for (int i = 0; i < 3; ++i) { current_fp_temp[lev][i].reset(); current_fp_ampere[lev][i].reset(); + current_fp_external[lev][i].reset(); } } void HybridPICModel::InitData () { m_resistivity_parser = std::make_unique( - utils::parser::makeParser(m_eta_expression, {"rho"})); - m_eta = m_resistivity_parser->compile<1>(); + utils::parser::makeParser(m_eta_expression, {"rho","J"})); + m_eta = m_resistivity_parser->compile<2>(); + const std::set resistivity_symbols = m_resistivity_parser->symbols(); + m_resistivity_has_J_dependence += resistivity_symbols.count("J"); + + m_J_external_parser[0] = std::make_unique( + utils::parser::makeParser(m_Jx_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[1] = std::make_unique( + utils::parser::makeParser(m_Jy_ext_grid_function,{"x","y","z","t"})); + m_J_external_parser[2] = std::make_unique( + utils::parser::makeParser(m_Jz_ext_grid_function,{"x","y","z","t"})); + m_J_external[0] = m_J_external_parser[0]->compile<4>(); + m_J_external[1] = m_J_external_parser[1]->compile<4>(); + m_J_external[2] = m_J_external_parser[2]->compile<4>(); + + // check if the external current parsers depend on time + for (int i=0; i<3; i++) { + const std::set J_ext_symbols = m_J_external_parser[i]->symbols(); + m_external_field_has_time_dependence += J_ext_symbols.count("t"); + } auto & warpx = WarpX::GetInstance(); @@ -181,6 +215,178 @@ void HybridPICModel::InitData () Ey_IndexType[1] = 1; Ez_IndexType[1] = 1; #endif + + // Initialize external current - note that this approach skips the check + // if the current is time dependent which is what needs to be done to + // write time independent fields on the first step. + std::array< std::unique_ptr, 3 > edge_lengths; + + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { +#ifdef AMREX_USE_EB + auto& edge_lengths_x = warpx.getedgelengths(lev, 0); + edge_lengths[0] = std::make_unique( + edge_lengths_x, amrex::make_alias, 0, edge_lengths_x.nComp() + ); + auto& edge_lengths_y = warpx.getedgelengths(lev, 1); + edge_lengths[1] = std::make_unique( + edge_lengths_y, amrex::make_alias, 0, edge_lengths_y.nComp() + ); + auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + edge_lengths[2] = std::make_unique( + edge_lengths_z, amrex::make_alias, 0, edge_lengths_z.nComp() + ); +#endif + GetCurrentExternal(edge_lengths, lev); + } +} + +void HybridPICModel::GetCurrentExternal ( + amrex::Vector, 3>> const& edge_lengths) +{ + if (!m_external_field_has_time_dependence) { return; } + + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { + GetCurrentExternal(edge_lengths[lev], lev); + } +} + + +void HybridPICModel::GetCurrentExternal ( + std::array< std::unique_ptr, 3> const& edge_lengths, + int lev) +{ + // This logic matches closely to WarpX::InitializeExternalFieldsOnGridUsingParser + // except that the parsers include time dependence. + auto & warpx = WarpX::GetInstance(); + + auto t = warpx.gett_new(lev); + + auto dx_lev = warpx.Geom(lev).CellSizeArray(); + const RealBox& real_box = warpx.Geom(lev).ProbDomain(); + + auto& mfx = current_fp_external[lev][0]; + auto& mfy = current_fp_external[lev][1]; + auto& mfz = current_fp_external[lev][2]; + + const amrex::IntVect x_nodal_flag = mfx->ixType().toIntVect(); + const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect(); + const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect(); + + // avoid implicit lambda capture + auto Jx_external = m_J_external[0]; + auto Jy_external = m_J_external[1]; + auto Jz_external = m_J_external[2]; + + for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& tbx = mfi.tilebox( x_nodal_flag, mfx->nGrowVect() ); + const amrex::Box& tby = mfi.tilebox( y_nodal_flag, mfy->nGrowVect() ); + const amrex::Box& tbz = mfi.tilebox( z_nodal_flag, mfz->nGrowVect() ); + + auto const& mfxfab = mfx->array(mfi); + auto const& mfyfab = mfy->array(mfi); + auto const& mfzfab = mfz->array(mfi); + +#ifdef AMREX_USE_EB + amrex::Array4 const& lx = edge_lengths[0]->array(mfi); + amrex::Array4 const& ly = edge_lengths[1]->array(mfi); + amrex::Array4 const& lz = edge_lengths[2]->array(mfi); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(ly); +#endif +#else + amrex::ignore_unused(edge_lengths); +#endif + + amrex::ParallelFor (tbx, tby, tbz, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lx(i, j, k) <= 0) return; +#endif + // Shift required in the x-, y-, or z- position + // depending on the index type of the multifab +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#else + const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - x_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - x_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the x-component of the field. + mfxfab(i,j,k) = Jx_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (ly(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - y_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - y_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the y-component of the field. + mfyfab(i,j,k) = Jy_external(x,y,z,t); + }, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + // skip if node is covered by an embedded boundary +#ifdef AMREX_USE_EB + if (lz(i, j, k) <= 0) return; +#endif +#if defined(WARPX_DIM_1D_Z) + const amrex::Real x = 0._rt; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real y = 0._rt; + const amrex::Real fac_z = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real z = j*dx_lev[1] + real_box.lo(1) + fac_z; +#elif defined(WARPX_DIM_3D) + const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; + const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; + const amrex::Real fac_y = (1._rt - z_nodal_flag[1]) * dx_lev[1] * 0.5_rt; + const amrex::Real y = j*dx_lev[1] + real_box.lo(1) + fac_y; + const amrex::Real fac_z = (1._rt - z_nodal_flag[2]) * dx_lev[2] * 0.5_rt; + const amrex::Real z = k*dx_lev[2] + real_box.lo(2) + fac_z; +#endif + // Initialize the z-component of the field. + mfzfab(i,j,k) = Jz_external(x,y,z,t); + } + ); + } } void HybridPICModel::CalculateCurrentAmpere ( @@ -210,7 +416,7 @@ void HybridPICModel::CalculateCurrentAmpere ( // the boundary correction was already applied to J_i and the B-field // boundary ensures that J itself complies with the boundary conditions, right? // ApplyJfieldBoundary(lev, Jfield[0].get(), Jfield[1].get(), Jfield[2].get()); - for (int i=0; i<3; i++) current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); + for (int i=0; i<3; i++) { current_fp_ampere[lev][i]->FillBoundary(warpx.Geom(lev).periodicity()); } } void HybridPICModel::HybridPICSolveE ( @@ -265,7 +471,8 @@ void HybridPICModel::HybridPICSolveE ( // Solve E field in regular cells warpx.get_pointer_fdtd_solver_fp(lev)->HybridPICSolveE( - Efield, current_fp_ampere[lev], Jfield, Bfield, rhofield, + Efield, current_fp_ampere[lev], Jfield, current_fp_external[lev], + Bfield, rhofield, electron_pressure_fp[lev], edge_lengths, lev, this, include_resistivity_term ); @@ -303,7 +510,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) void HybridPICModel::FillElectronPressureMF ( std::unique_ptr const& Pe_field, - amrex::MultiFab* const& rho_field ) + amrex::MultiFab* const& rho_field ) const { const auto n0_ref = m_n0_ref; const auto elec_temp = m_elec_temp; @@ -329,3 +536,156 @@ void HybridPICModel::FillElectronPressureMF ( }); } } + +void HybridPICModel::BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + auto& warpx = WarpX::GetInstance(); + for (int lev = 0; lev <= warpx.finestLevel(); ++lev) + { + BfieldEvolveRK( + Bfield, Efield, Jfield, rhofield, edge_lengths, dt, lev, dt_type, + ng, nodal_sync + ); + } +} + +void HybridPICModel::BfieldEvolveRK ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, int lev, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + // Make copies of the B-field multifabs at t = n and create multifabs for + // each direction to store the Runge-Kutta intermediate terms. Each + // multifab has 2 components for the different terms that need to be stored. + std::array< MultiFab, 3 > B_old; + std::array< MultiFab, 3 > K; + for (int ii = 0; ii < 3; ii++) + { + B_old[ii] = MultiFab( + Bfield[lev][ii]->boxArray(), Bfield[lev][ii]->DistributionMap(), 1, + Bfield[lev][ii]->nGrowVect() + ); + MultiFab::Copy(B_old[ii], *Bfield[lev][ii], 0, 0, 1, ng); + + K[ii] = MultiFab( + Bfield[lev][ii]->boxArray(), Bfield[lev][ii]->DistributionMap(), 2, + Bfield[lev][ii]->nGrowVect() + ); + K[ii].setVal(0.0); + } + + // The Runge-Kutta scheme begins here. + // Step 1: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * [-curl x E(B_old)] = B_old + 0.5 * dt * K0. + for (int ii = 0; ii < 3; ii++) + { + // Extract 0.5 * dt * K0 for each direction into index 0 of K. + MultiFab::LinComb( + K[ii], 1._rt, *Bfield[lev][ii], 0, -1._rt, B_old[ii], 0, 0, 1, ng + ); + } + + // Step 2: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * K0 + 0.5 * dt * [-curl x E(B_old + 0.5 * dt * K1)] + // = B_old + 0.5 * dt * K0 + 0.5 * dt * K1 + for (int ii = 0; ii < 3; ii++) + { + // Subtract 0.5 * dt * K0 from the Bfield for each direction, to get + // B_new = B_old + 0.5 * dt * K1. + MultiFab::Subtract(*Bfield[lev][ii], K[ii], 0, 0, 1, ng); + // Extract 0.5 * dt * K1 for each direction into index 1 of K. + MultiFab::LinComb( + K[ii], 1._rt, *Bfield[lev][ii], 0, -1._rt, B_old[ii], 0, 1, 1, ng + ); + } + + // Step 3: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + 0.5 * dt * K1 + dt * [-curl x E(B_old + 0.5 * dt * K1)] + // = B_old + 0.5 * dt * K1 + dt * K2 + for (int ii = 0; ii < 3; ii++) + { + // Subtract 0.5 * dt * K1 from the Bfield for each direction to get + // B_new = B_old + dt * K2. + MultiFab::Subtract(*Bfield[lev][ii], K[ii], 1, 0, 1, ng); + } + + // Step 4: + FieldPush( + Bfield, Efield, Jfield, rhofield, edge_lengths, + 0.5_rt*dt, dt_type, ng, nodal_sync + ); + + // The Bfield is now given by: + // B_new = B_old + dt * K2 + 0.5 * dt * [-curl x E(B_old + dt * K2)] + // = B_old + dt * K2 + 0.5 * dt * K3 + for (int ii = 0; ii < 3; ii++) + { + // Subtract B_old from the Bfield for each direction, to get + // B = dt * K2 + 0.5 * dt * K3. + MultiFab::Subtract(*Bfield[lev][ii], B_old[ii], 0, 0, 1, ng); + + // Add dt * K2 + 0.5 * dt * K3 to index 0 of K (= 0.5 * dt * K0). + MultiFab::Add(K[ii], *Bfield[lev][ii], 0, 0, 1, ng); + + // Add 2 * 0.5 * dt * K1 to index 0 of K. + MultiFab::LinComb( + K[ii], 1.0, K[ii], 0, 2.0, K[ii], 1, 0, 1, ng + ); + + // Overwrite the Bfield with the Runge-Kutta sum: + // B_new = B_old + 1/3 * dt * (0.5 * K0 + K1 + K2 + 0.5 * K3). + MultiFab::LinComb( + *Bfield[lev][ii], 1.0, B_old[ii], 0, 1.0/3.0, K[ii], 0, 0, 1, ng + ); + } +} + +void HybridPICModel::FieldPush ( + amrex::Vector, 3>>& Bfield, + amrex::Vector, 3>>& Efield, + amrex::Vector, 3>> const& Jfield, + amrex::Vector> const& rhofield, + amrex::Vector, 3>> const& edge_lengths, + amrex::Real dt, DtType dt_type, + IntVect ng, std::optional nodal_sync ) +{ + auto& warpx = WarpX::GetInstance(); + + // Calculate J = curl x B / mu0 + CalculateCurrentAmpere(Bfield, edge_lengths); + // Calculate the E-field from Ohm's law + HybridPICSolveE(Efield, Jfield, Bfield, rhofield, edge_lengths, true); + warpx.FillBoundaryE(ng, nodal_sync); + // Push forward the B-field using Faraday's law + warpx.EvolveB(dt, dt_type); + warpx.FillBoundaryB(ng, nodal_sync); +} diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp index ff220887099..5100eed0df3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICSolveE.cpp @@ -369,6 +369,7 @@ void FiniteDifferenceSolver::HybridPICSolveE ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 >& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -382,14 +383,14 @@ void FiniteDifferenceSolver::HybridPICSolveE ( #ifdef WARPX_DIM_RZ HybridPICSolveECylindrical ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); #else HybridPICSolveECartesian ( - Efield, Jfield, Jifield, Bfield, rhofield, Pefield, + Efield, Jfield, Jifield, Jextfield, Bfield, rhofield, Pefield, edge_lengths, lev, hybrid_model, include_resistivity_term ); @@ -406,6 +407,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -431,6 +433,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; + const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -484,6 +487,9 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Array4 const& Jir = Jifield[0]->const_array(mfi); Array4 const& Jit = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextr = Jextfield[0]->const_array(mfi); + Array4 const& Jextt = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Br = Bfield[0]->const_array(mfi); Array4 const& Bt = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); @@ -508,16 +514,16 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // calculate enE = (J - Ji) x B enE_nodal(i, j, 0, 0) = ( - (jt_interp - jit_interp) * Bz_interp - - (jz_interp - jiz_interp) * Bt_interp + (jt_interp - jit_interp - Jextt(i, j, 0)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, 0)) * Bt_interp ); enE_nodal(i, j, 0, 1) = ( - (jz_interp - jiz_interp) * Br_interp - - (jr_interp - jir_interp) * Bz_interp + (jz_interp - jiz_interp - Jextz(i, j, 0)) * Br_interp + - (jr_interp - jir_interp - Jextr(i, j, 0)) * Bz_interp ); enE_nodal(i, j, 0, 2) = ( - (jr_interp - jir_interp) * Bt_interp - - (jt_interp - jit_interp) * Br_interp + (jr_interp - jir_interp - Jextr(i, j, 0)) * Bt_interp + - (jt_interp - jit_interp - Jextt(i, j, 0)) * Br_interp ); }); @@ -584,8 +590,17 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Er_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Er_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Er_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDr(Pe, coefs_r, n_coefs_r, i, j, 0, 0); @@ -596,7 +611,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Er(i, j, 0) = (enE_r - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Er(i, j, 0) += eta(rho_val) * Jr(i, j, 0); + if (include_resistivity_term) { Er(i, j, 0) += eta(rho_val, jtot_val) * Jr(i, j, 0); } }, // Et calculation @@ -617,8 +632,17 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Er_stag, coarsen, i, j, 0, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Et_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Et_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Et_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure // -> d/dt = 0 for m = 0 @@ -630,20 +654,29 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Et(i, j, 0) = (enE_t - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Et(i, j, 0) += eta(rho_val) * Jt(i, j, 0); + if (include_resistivity_term) { Et(i, j, 0) += eta(rho_val, jtot_val) * Jt(i, j, 0); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,0) <= 0) return; + if (lz(i,j,0) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jr_val = Interp(Jr, Jr_stag, Ez_stag, coarsen, i, j, 0, 0); + Real jt_val = Interp(Jt, Jt_stag, Ez_stag, coarsen, i, j, 0, 0); + Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, 0, 0); + jtot_val = std::sqrt(jr_val*jr_val + jt_val*jt_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k, 0); @@ -654,7 +687,7 @@ void FiniteDifferenceSolver::HybridPICSolveECylindrical ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } } ); @@ -674,6 +707,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( std::array< std::unique_ptr, 3 >& Efield, std::array< std::unique_ptr, 3 > const& Jfield, std::array< std::unique_ptr, 3 > const& Jifield, + std::array< std::unique_ptr, 3 > const& Jextfield, std::array< std::unique_ptr, 3 > const& Bfield, std::unique_ptr const& rhofield, std::unique_ptr const& Pefield, @@ -693,6 +727,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // get hybrid model parameters const auto eta = hybrid_model->m_eta; const auto rho_floor = hybrid_model->m_n_floor * PhysConst::q_e; + const auto resistivity_has_J_dependence = hybrid_model->m_resistivity_has_J_dependence; // Index type required for interpolating fields from their respective // staggering to the Ex, Ey, Ez locations @@ -746,6 +781,9 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Array4 const& Jix = Jifield[0]->const_array(mfi); Array4 const& Jiy = Jifield[1]->const_array(mfi); Array4 const& Jiz = Jifield[2]->const_array(mfi); + Array4 const& Jextx = Jextfield[0]->const_array(mfi); + Array4 const& Jexty = Jextfield[1]->const_array(mfi); + Array4 const& Jextz = Jextfield[2]->const_array(mfi); Array4 const& Bx = Bfield[0]->const_array(mfi); Array4 const& By = Bfield[1]->const_array(mfi); Array4 const& Bz = Bfield[2]->const_array(mfi); @@ -770,16 +808,16 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // calculate enE = (J - Ji) x B enE_nodal(i, j, k, 0) = ( - (jy_interp - jiy_interp) * Bz_interp - - (jz_interp - jiz_interp) * By_interp + (jy_interp - jiy_interp - Jexty(i, j, k)) * Bz_interp + - (jz_interp - jiz_interp - Jextz(i, j, k)) * By_interp ); enE_nodal(i, j, k, 1) = ( - (jz_interp - jiz_interp) * Bx_interp - - (jx_interp - jix_interp) * Bz_interp + (jz_interp - jiz_interp - Jextz(i, j, k)) * Bx_interp + - (jx_interp - jix_interp - Jextx(i, j, k)) * Bz_interp ); enE_nodal(i, j, k, 2) = ( - (jx_interp - jix_interp) * By_interp - - (jy_interp - jiy_interp) * Bx_interp + (jx_interp - jix_interp - Jextx(i, j, k)) * By_interp + - (jy_interp - jiy_interp - Jexty(i, j, k)) * Bx_interp ); }); @@ -844,8 +882,17 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ex_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ex_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ex_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ex_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDx(Pe, coefs_x, n_coefs_x, i, j, k); @@ -856,7 +903,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ex(i, j, k) = (enE_x - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ex(i, j, k) += eta(rho_val) * Jx(i, j, k); + if (include_resistivity_term) { Ex(i, j, k) += eta(rho_val, jtot_val) * Jx(i, j, k); } }, // Ey calculation @@ -864,18 +911,27 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries #ifdef WARPX_DIM_3D - if (ly(i,j,k) <= 0) return; + if (ly(i,j,k) <= 0) { return; } #elif defined(WARPX_DIM_XZ) //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered amrex::ignore_unused(ly); - if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) return; + if (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j-1, k)<=0 || lz(i, j, k)<=0) { return; } #endif #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ey_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ey_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ey_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ey_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDy(Pe, coefs_y, n_coefs_y, i, j, k); @@ -886,20 +942,29 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ey(i, j, k) = (enE_y - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ey(i, j, k) += eta(rho_val) * Jy(i, j, k); + if (include_resistivity_term) { Ey(i, j, k) += eta(rho_val, jtot_val) * Jy(i, j, k); } }, // Ez calculation [=] AMREX_GPU_DEVICE (int i, int j, int k){ #ifdef AMREX_USE_EB // Skip field solve if this cell is fully covered by embedded boundaries - if (lz(i,j,k) <= 0) return; + if (lz(i,j,k) <= 0) { return; } #endif // Interpolate to get the appropriate charge density in space Real rho_val = Interp(rho, nodal, Ez_stag, coarsen, i, j, k, 0); + // Interpolate current to appropriate staggering to match E field + Real jtot_val = 0._rt; + if (include_resistivity_term && resistivity_has_J_dependence) { + Real jx_val = Interp(Jx, Jx_stag, Ez_stag, coarsen, i, j, k, 0); + Real jy_val = Interp(Jy, Jy_stag, Ez_stag, coarsen, i, j, k, 0); + Real jz_val = Interp(Jz, Jz_stag, Ez_stag, coarsen, i, j, k, 0); + jtot_val = std::sqrt(jx_val*jx_val + jy_val*jy_val + jz_val*jz_val); + } + // safety condition since we divide by rho_val later - if (rho_val < rho_floor) rho_val = rho_floor; + if (rho_val < rho_floor) { rho_val = rho_floor; } // Get the gradient of the electron pressure auto grad_Pe = T_Algo::UpwardDz(Pe, coefs_z, n_coefs_z, i, j, k); @@ -910,7 +975,7 @@ void FiniteDifferenceSolver::HybridPICSolveECartesian ( Ez(i, j, k) = (enE_z - grad_Pe) / rho_val; // Add resistivity only if E field value is used to update B - if (include_resistivity_term) Ez(i, j, k) += eta(rho_val) * Jz(i, j, k); + if (include_resistivity_term) { Ez(i, j, k) += eta(rho_val, jtot_val) * Jz(i, j, k); } } ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index b26a918c9ae..205b009b2eb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -27,8 +27,7 @@ * \brief This class contains the macroscopic properties of the medium needed to * evaluate macroscopic Maxwell equation. */ -class -MacroscopicProperties +class MacroscopicProperties { public: MacroscopicProperties (); // constructor @@ -102,7 +101,7 @@ private: /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * Lax-Wendroff scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma), permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update @@ -134,7 +133,7 @@ struct LaxWendroffAlgo { /** * \brief - * This struct contains only static functions to compute the co-efficients for the + * This struct contains only static functions to compute the coefficients for the * BackwardEuler scheme of macroscopic Maxwells equations using * macroscopic properties, namely, conductivity (sigma) and permittivity (epsilon). * Permeability of the material, mu, is used as (beta/mu) for the E-update diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 32c07e0b9fe..965db68a558 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -66,6 +66,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_sigma_function,{"x","y","z"})); } + // Query input for material permittivity, epsilon. bool epsilon_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "epsilon", m_epsilon)) { m_epsilon_s = "constant"; @@ -91,7 +92,7 @@ MacroscopicProperties::ReadParameters () utils::parser::makeParser(m_str_epsilon_function,{"x","y","z"})); } - // Query input for material permittivity, epsilon. + // Query input for material permeability, mu. bool mu_specified = false; if (utils::parser::queryWithParser(pp_macroscopic, "mu", m_mu)) { m_mu_s = "constant"; diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp index 6b77a559b8c..192017656ce 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp @@ -206,7 +206,7 @@ void WarpX::setVectorPotentialBC ( amrex::Vector,3>>& A ) const { // check if any dimension has non-periodic boundary conditions - if (!m_vector_poisson_boundary_handler.has_non_periodic) return; + if (!m_vector_poisson_boundary_handler.has_non_periodic) { return; } auto dirichlet_flag = m_vector_poisson_boundary_handler.dirichlet_flag; @@ -228,7 +228,7 @@ WarpX::setVectorPotentialBC ( amrex::Vector= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into the right index fields_arr(i,j,k,field_index) = spectral_field_value; @@ -390,15 +390,15 @@ SpectralFieldData::BackwardTransform (const int lev, Complex spectral_field_value = field_arr(i,j,k,field_index); // Apply proper shift in each dimension #if (AMREX_SPACEDIM >= 2) - if (!is_nodal_x) spectral_field_value *= xshift_arr[i]; + if (!is_nodal_x) { spectral_field_value *= xshift_arr[i]; } #endif #if defined(WARPX_DIM_3D) - if (!is_nodal_y) spectral_field_value *= yshift_arr[j]; - if (!is_nodal_z) spectral_field_value *= zshift_arr[k]; + if (!is_nodal_y) { spectral_field_value *= yshift_arr[j]; } + if (!is_nodal_z) { spectral_field_value *= zshift_arr[k]; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } #elif defined(WARPX_DIM_1D_Z) - if (!is_nodal_z) spectral_field_value *= zshift_arr[i]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[i]; } #endif // Copy field into temporary array tmp_arr(i,j,k) = spectral_field_value; @@ -447,7 +447,7 @@ SpectralFieldData::BackwardTransform (const int lev, { for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - if ((fill_guards[dir]) == 0) mf_box.grow(dir, -mf_ng[dir]); + if ((fill_guards[dir]) == 0) { mf_box.grow(dir, -mf_ng[dir]); } } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index 236fe3fbb29..c4a693bf25a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -52,7 +52,7 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, tmpSpectralField = SpectralField(spectralspace_ba, dm, n_rz_azimuthal_modes, 0); // By default, we assume the z FFT is done from/to a nodal grid in real space. - // It the FFT is performed from/to a cell-centered grid in real space, + // If the FFT is performed from/to a cell-centered grid in real space, // a correcting "shift" factor must be applied in spectral space. zshift_FFTfromCell = k_space.getSpectralShiftFactor(dm, 1, ShiftType::TransformFromCellCentered); @@ -331,7 +331,7 @@ SpectralFieldDataRZ::FABZForwardTransform (amrex::MFIter const & mfi, amrex::Box [=] AMREX_GPU_DEVICE(int i, int j, int k, int mode) noexcept { Complex spectral_field_value = tmp_arr(i,j,k,mode); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the correct index. int const ic = field_index + mode*n_fields; fields_arr(i,j,k,ic) = spectral_field_value*inv_nz; @@ -369,7 +369,7 @@ SpectralFieldDataRZ::FABZBackwardTransform (amrex::MFIter const & mfi, amrex::Bo int const ic = field_index + mode*n_fields; Complex spectral_field_value = fields_arr(i,j,k,ic); // Apply proper shift. - if (!is_nodal_z) spectral_field_value *= zshift_arr[j]; + if (!is_nodal_z) { spectral_field_value *= zshift_arr[j]; } // Copy field into the right index. tmp_arr(i,j,k,mode) = spectral_field_value; }); diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp index c2309512596..39782408eb0 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/BesselRoots.cpp @@ -58,7 +58,7 @@ namespace{ q0 = static_cast(jn(n, p0)); q1 = static_cast(jn(n, p1)); for (int it=1; it <= nitmx; it++) { - if (q1 == q0) break; + if (q1 == q0) { break; } p = p1 - q1*(p1 - p0)/(q1 - q0); dp = p - p1; if (it > 1 && std::abs(dp) < tol) { @@ -142,7 +142,7 @@ void GetBesselRoots(int n, int nk, amrex::Vector& roots, amrex::Vec const auto errj = static_cast(std::abs(jn(n, zeroj))); // improve solution using procedure SecantRootFinder - if (errj > tol) ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); + if (errj > tol) { ::SecantRootFinder(n, nitmx, tol, &zeroj, &ierror); } roots[ik] = zeroj; ier[ik] = ierror; diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index aff5ad25fcc..a8930e1d48b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -122,7 +122,7 @@ HankelTransform::HankelTransform (int const hankel_order, // Calculate the matrix M by inverting invM if (azimuthal_mode !=0 && hankel_order != azimuthal_mode-1) { // In this case, invM is singular, thus we calculate the pseudo-inverse. - // The Moore-Penrose psuedo-inverse is calculated using the SVD method. + // The Moore-Penrose pseudo-inverse is calculated using the SVD method. M.resize(m_nk*m_nr, 0.); amrex::Vector invMcopy(invM); @@ -132,7 +132,7 @@ HankelTransform::HankelTransform (int const hankel_order, amrex::Vector sp((m_nr)*(m_nk-1), 0.); amrex::Vector temp((m_nr)*(m_nk-1), 0.); - // Calculate the singlular-value-decomposition of invM (leaving out the first row). + // Calculate the singular-value-decomposition of invM (leaving out the first row). // invM = u*sdiag*vt // Note that invMcopy.dataPtr()+1 is passed in so that the first ik row is skipped // A copy is passed in since the matrix is destroyed diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 6d35234f0ed..91fe2953668 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -137,7 +137,7 @@ SpectralKSpace::getKComponent( const DistributionMapping& dm, * specified by `i_dim`. * * (By default, we assume the FFT is done from/to a collocated grid in real space - * It the FFT is performed from/to a cell-centered grid in real space, + * If the FFT is performed from/to a cell-centered grid in real space, * a correcting "shift" factor must be applied in spectral space.) */ SpectralShiftFactor diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6e69dc8a635..53b8e67412d 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -183,11 +183,11 @@ WarpX::PSATDForwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { - if (F_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *F_fp[lev], Idx.F); } if (spectral_solver_cp[lev]) { - if (F_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *F_cp[lev], Idx.F); } } } } @@ -200,17 +200,17 @@ WarpX::PSATDBackwardTransformF () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F); } #else - if (F_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); + if (F_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *F_fp[lev], Idx.F, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F); } #else - if (F_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); + if (F_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *F_cp[lev], Idx.F, m_fill_guards_fields); } #endif } } @@ -229,11 +229,11 @@ WarpX::PSATDForwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { - if (G_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *G_fp[lev], Idx.G); } if (spectral_solver_cp[lev]) { - if (G_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *G_cp[lev], Idx.G); } } } } @@ -246,17 +246,17 @@ WarpX::PSATDBackwardTransformG () for (int lev = 0; lev <= finest_level; ++lev) { #ifdef WARPX_DIM_RZ - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G); } #else - if (G_fp[lev]) spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); + if (G_fp[lev]) { spectral_solver_fp[lev]->BackwardTransform(lev, *G_fp[lev], Idx.G, m_fill_guards_fields); } #endif if (spectral_solver_cp[lev]) { #ifdef WARPX_DIM_RZ - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G); } #else - if (G_cp[lev]) spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); + if (G_cp[lev]) { spectral_solver_cp[lev]->BackwardTransform(lev, *G_cp[lev], Idx.G, m_fill_guards_fields); } #endif } } @@ -370,15 +370,15 @@ void WarpX::PSATDForwardTransformRho ( const amrex::Vector>& charge_cp, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) return; + if (charge_fp[0] == nullptr) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); + if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); + if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } } } @@ -759,22 +759,23 @@ WarpX::PushPSATD () PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); #ifdef WARPX_DIM_RZ - if (pml_rz[0]) pml_rz[0]->PushPSATD(0); + if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } #endif // FFT of F and G - if (WarpX::do_dive_cleaning) PSATDForwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDForwardTransformG(); + if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } // Update E, B, F, and G in k-space PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G PSATDBackwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp); - if (WarpX::fft_do_time_averaging) + if (WarpX::fft_do_time_averaging) { PSATDBackwardTransformEBavg(Efield_avg_fp, Bfield_avg_fp, Efield_avg_cp, Bfield_avg_cp); - if (WarpX::do_dive_cleaning) PSATDBackwardTransformF(); - if (WarpX::do_divb_cleaning) PSATDBackwardTransformG(); + } + if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } + if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } // Evolve the fields in the PML boxes for (int lev = 0; lev <= finest_level; ++lev) @@ -784,9 +785,9 @@ WarpX::PushPSATD () pml[lev]->PushPSATD(lev); } ApplyEfieldBoundary(lev, PatchType::fine); - if (lev > 0) ApplyEfieldBoundary(lev, PatchType::coarse); + if (lev > 0) { ApplyEfieldBoundary(lev, PatchType::coarse); } ApplyBfieldBoundary(lev, PatchType::fine, DtType::FirstHalf); - if (lev > 0) ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); + if (lev > 0) { ApplyBfieldBoundary(lev, PatchType::coarse, DtType::FirstHalf); } } #endif } @@ -916,7 +917,7 @@ WarpX::EvolveE (int lev, PatchType patch_type, amrex::Real a_dt) void WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -927,16 +928,16 @@ WarpX::EvolveF (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveF (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } EvolveF(lev, PatchType::fine, a_dt, a_dt_type); - if (lev > 0) EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); + if (lev > 0) { EvolveF(lev, PatchType::coarse, a_dt, a_dt_type); } } void WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_type) { - if (!do_dive_cleaning) return; + if (!do_dive_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveF()"); @@ -966,7 +967,7 @@ WarpX::EvolveF (int lev, PatchType patch_type, amrex::Real a_dt, DtType a_dt_typ void WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } for (int lev = 0; lev <= finest_level; ++lev) { @@ -977,7 +978,7 @@ WarpX::EvolveG (amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } EvolveG(lev, PatchType::fine, a_dt, a_dt_type); @@ -990,7 +991,7 @@ WarpX::EvolveG (int lev, amrex::Real a_dt, DtType a_dt_type) void WarpX::EvolveG (int lev, PatchType patch_type, amrex::Real a_dt, DtType /*a_dt_type*/) { - if (!do_divb_cleaning) return; + if (!do_divb_cleaning) { return; } WARPX_PROFILE("WarpX::EvolveG()"); @@ -1076,8 +1077,8 @@ WarpX::DampFieldsInGuards(const int lev, { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for ( amrex::MFIter mfi(*Efield[0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) { @@ -1170,8 +1171,8 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& for (int iside = 0; iside < 2; iside++) { // Only apply to damped boundaries - if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) continue; - if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) continue; + if (iside == 0 && WarpX::field_boundary_lo[dampdir] != FieldBoundaryType::Damped) { continue; } + if (iside == 1 && WarpX::field_boundary_hi[dampdir] != FieldBoundaryType::Damped) { continue; } for (amrex::MFIter mfi(*mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { @@ -1207,7 +1208,7 @@ void WarpX::DampFieldsInGuards(const int lev, std::unique_ptr& } #ifdef WARPX_DIM_RZ -// This scales the current by the inverse volume and wraps around the depostion at negative radius. +// This scales the current by the inverse volume and wraps around the deposition at negative radius. // It is faster to apply this on the grid than to do it particle by particle. // It is put here since there isn't another nice place for it. void diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 9a5449dfef8..50a85a8cdee 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -47,12 +47,17 @@ void WarpX::HybridPICEvolveFields () // SyncCurrent does not include a call to FillBoundary, but it is needed // for the hybrid-PIC solver since current values are interpolated to // a nodal grid - for (int lev = 0; lev <= finest_level; ++lev) - for (int idim = 0; idim < 3; ++idim) + for (int lev = 0; lev <= finest_level; ++lev) { + for (int idim = 0; idim < 3; ++idim) { current_fp[lev][idim]->FillBoundary(Geom(lev).periodicity()); + } + } // Get requested number of substeps to use - int sub_steps = m_hybrid_pic_model->m_substeps / 2; + int sub_steps = m_hybrid_pic_model->m_substeps; + + // Get the external current + m_hybrid_pic_model->GetCurrentExternal(m_edge_lengths); // Reference hybrid-PIC multifabs auto& rho_fp_temp = m_hybrid_pic_model->rho_fp_temp; @@ -63,10 +68,6 @@ void WarpX::HybridPICEvolveFields () // 0'th index of `rho_fp`, J_i^{n-1/2} in `current_fp_temp` and J_i^{n+1/2} // in `current_fp`. - // TODO: To speed up the algorithm insert Runge-Kutta integration logic - // for B update instead of the substep update used here - can test with - // small timestep using this simpler implementation - // Note: E^{n} is recalculated with the accurate J_i^{n} since at the end // of the last step we had to "guess" it. It also needs to be // recalculated to include the resistivity before evolving B. @@ -95,14 +96,12 @@ void WarpX::HybridPICEvolveFields () // momentum equation for (int sub_step = 0; sub_step < sub_steps; sub_step++) { - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); - m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp_temp, Bfield_fp, rho_fp_temp, m_edge_lengths, - true + m_hybrid_pic_model->BfieldEvolveRK( + Bfield_fp, Efield_fp, current_fp_temp, rho_fp_temp, + m_edge_lengths, 0.5_rt/sub_steps*dt[0], + DtType::FirstHalf, guard_cells.ng_FieldSolver, + WarpX::sync_nodal_points ); - FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5_rt / sub_steps * dt[0], DtType::FirstHalf); - FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } // Average rho^{n} and rho^{n+1} to get rho^{n+1/2} in rho_fp_temp @@ -123,14 +122,12 @@ void WarpX::HybridPICEvolveFields () // Now push the B field from t=n+1/2 to t=n+1 using the n+1/2 quantities for (int sub_step = 0; sub_step < sub_steps; sub_step++) { - m_hybrid_pic_model->CalculateCurrentAmpere(Bfield_fp, m_edge_lengths); - m_hybrid_pic_model->HybridPICSolveE( - Efield_fp, current_fp, Bfield_fp, rho_fp_temp, m_edge_lengths, - true + m_hybrid_pic_model->BfieldEvolveRK( + Bfield_fp, Efield_fp, current_fp, rho_fp_temp, + m_edge_lengths, 0.5_rt/sub_steps*dt[0], + DtType::SecondHalf, guard_cells.ng_FieldSolver, + WarpX::sync_nodal_points ); - FillBoundaryE(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); - EvolveB(0.5_rt / sub_steps * dt[0], DtType::SecondHalf); - FillBoundaryB(guard_cells.ng_FieldSolver, WarpX::sync_nodal_points); } // Extrapolate the ion current density to t=n+1 using diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index 94903f3cb35..6c86bba67b6 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -18,12 +18,12 @@ /** * calc_M calculates the Magnetization field of the vacuum at a specific point and returns it as a three component vector * \param[in] arr This is teh empty array that will be filled with the components of the M-field - * \param[in] ex The x-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ey The y-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] ez The z-component of the E-field at the point at whicht the M-field is to be calculated - * \param[in] bx The x-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] by The y-component of the B-field at the point at whicht the M-field is to be calculated - * \param[in] bz The z-component of the B-field at the point at whicht the M-field is to be calculated + * \param[in] ex The x-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ey The y-component of the E-field at the point at which the M-field is to be calculated + * \param[in] ez The z-component of the E-field at the point at which the M-field is to be calculated + * \param[in] bx The x-component of the B-field at the point at which the M-field is to be calculated + * \param[in] by The y-component of the B-field at the point at which the M-field is to be calculated + * \param[in] bz The z-component of the B-field at the point at which the M-field is to be calculated * \param[in] xi_c2 The quantum parameter * c2 being used for the simulation * \param[in] c2 the speed of light squared */ @@ -72,7 +72,7 @@ void calc_M(amrex::Real arr [], amrex::Real ex, amrex::Real ey, amrex::Real ez, * \param[in] Jz the current field in z * \param[in] dx The x spatial step, used for calculating curls * \param[in] dy The y spatial step, used for calculating curls - * \param[in] dz The z spatial step, used for calulating curls + * \param[in] dz The z spatial step, used for calculating curls * \param[in] dt The temporal step, used for the half push/correction to the E-fields at the end of the function * \param[in] xi_c2 Quantum parameter * c**2 */ @@ -109,7 +109,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,l), tmpEy(j+1,k,l), tmpEz(j+1,k,l), Bx(j+1,k,l), By(j+1,k,l), Bz(j+1,k,l), xi_c2, c2); @@ -182,7 +182,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -198,7 +198,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 )+a02*( a01*a12 - a02*a11 ); - // Calcualting the rows of the inverse matrix using the general 3x3 inverse form + // Calculating the rows of the inverse matrix using the general 3x3 inverse form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -206,7 +206,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -220,7 +220,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,l) = Ex(j,k,l) + 0.5_rt*dt*dEx; @@ -239,7 +239,7 @@ const amrex::Real dyi = 1._rt/dy; amrex::Real Mpz [3] = {0._rt,0._rt,0._rt}; amrex::Real Mnz [3] = {0._rt,0._rt,0._rt}; - // Calcualting the M-field at the chosen stencil points + // Calculating the M-field at the chosen stencil points calc_M(Mpx, tmpEx(j+1,k,0), tmpEy(j+1,k,0), tmpEz(j+1,k,0), Bx(j+1,k,0), By(j+1,k,0), Bz(j+1,k,0), xi_c2, c2); @@ -308,7 +308,7 @@ const amrex::Real dyi = 1._rt/dy; + 7._rt*c2*bz*(BVxB + Bmu0J) ) }; - // Calcualting matrix values for the QED correction algorithm + // Calculating matrix values for the QED correction algorithm const amrex::Real a00 = beta + xi_c2*( 8._rt*c2i*ex*ex + 14._rt*bx*bx ); @@ -324,7 +324,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real detA = a00*( a11*a22 - a12*a12 ) - a01*( a01*a22 - a02*a12 ) + a02*( a01*a12 - a02*a11 ); - // Calcualting inverse matrix values using the general 3x3 form + // Calculating inverse matrix values using the general 3x3 form const amrex::Real invAx[3] = {a22*a11 - a12*a12, a12*a02 - a22*a01, a12*a01 - a11*a02}; @@ -332,7 +332,7 @@ const amrex::Real dyi = 1._rt/dy; const amrex::Real invAz[3] = {a12*a01 - a02*a11, a02*a01 - a12*a00, a11*a00 - a01*a01}; - // Calcualting the final QED corrections by mutliplying the Omega vector with the inverse matrix + // Calculating the final QED corrections by mutliplying the Omega vector with the inverse matrix const amrex::Real dEx = (-1._rt/detA)*(invAx[0]*Omega[0] + invAx[1]*Omega[1] + @@ -346,7 +346,7 @@ const amrex::Real dyi = 1._rt/dy; invAz[1]*Omega[1] + invAz[2]*Omega[2]); - // Adding the QED corrections to the origional fields + // Adding the QED corrections to the original fields Ex(j,k,0) = Ex(j,k,0) + 0.5_rt*dt*dEx; diff --git a/Source/Filter/BilinearFilter.cpp b/Source/Filter/BilinearFilter.cpp index ecfee2cd7c9..a095bce6ae3 100644 --- a/Source/Filter/BilinearFilter.cpp +++ b/Source/Filter/BilinearFilter.cpp @@ -35,7 +35,7 @@ namespace { for(int ipass=1; ipass< lastpass; ipass++){ // element 0 has to be treated in its own way new_s.at(0) = 0.5_rt * old_s.at(0); - if (1(el); + } stencil_length_each_dir += 1.; #if defined(WARPX_DIM_3D) // npass_each_dir = npass_x npass_y npass_z diff --git a/Source/Filter/NCIGodfreyFilter.H b/Source/Filter/NCIGodfreyFilter.H index 86446495869..91479cecd0d 100644 --- a/Source/Filter/NCIGodfreyFilter.H +++ b/Source/Filter/NCIGodfreyFilter.H @@ -16,7 +16,7 @@ enum class godfrey_coeff_set { Ex_Ey_Bz=0, Bx_By_Ez=1 }; /** - * \brief Class for Godrey's filter to suppress Numerical Cherenkov Instability + * \brief Class for Godfrey's filter to suppress Numerical Cherenkov Instability * * It derives from the base class Filter. * The filter stencil is initialized in method ComputeStencils. Computing the diff --git a/Source/Filter/NCIGodfreyFilter.cpp b/Source/Filter/NCIGodfreyFilter.cpp index 1cfd42aed03..9567bdf1bb2 100644 --- a/Source/Filter/NCIGodfreyFilter.cpp +++ b/Source/Filter/NCIGodfreyFilter.cpp @@ -45,7 +45,7 @@ void NCIGodfreyFilter::ComputeStencils() { using namespace warpx::nci_godfrey; - // Sanity checks: filter length shoulz be 5 in z + // Sanity checks: filter length should be 5 in z # if defined(WARPX_DIM_3D) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( slen.z==5,"ERROR: NCI filter requires 5 points in z"); diff --git a/Source/Fluids/MusclHancockUtils.H b/Source/Fluids/MusclHancockUtils.H index 764143a7220..2765f28f3b3 100644 --- a/Source/Fluids/MusclHancockUtils.H +++ b/Source/Fluids/MusclHancockUtils.H @@ -44,12 +44,13 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::min(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::max(a, b); - else + } else { return 0.0_rt; + } } // Min of 3 inputs AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -68,24 +69,26 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real minmod3 (amrex::Real a, amrex::Real b , amrex::Real c) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt && c > 0.0_rt) { return min3(a,b,c); - else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt && c < 0.0_rt) { return max3(a,b,c); - else + } else { return 0.0; + } } //maxmod AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real maxmod (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a > 0.0_rt && b > 0.0_rt) + if (a > 0.0_rt && b > 0.0_rt) { return std::max(a, b); - else if (a < 0.0_rt && b < 0.0_rt) + } else if (a < 0.0_rt && b < 0.0_rt) { return std::min(a, b); - else + } else { return 0.0_rt; + } } // Rusanov Flux (density) AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -132,30 +135,33 @@ amrex::Real ave_adjustable_diff (amrex::Real a, amrex::Real b) { using namespace amrex::literals; constexpr auto sigma = static_cast(2.0*0.732050807568877); - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, sigma*a, sigma*b ); - else + } else { return 0.0_rt; + } } // ave_minmod Low diffusivity AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod3( (a+b)/2.0_rt, 2.0_rt*a, 2.0_rt*b ); - else + } else { return 0.0_rt; + } } // ave_superbee AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE amrex::Real ave_superbee (amrex::Real a, amrex::Real b) { using namespace amrex::literals; - if (a*b > 0.0_rt) + if (a*b > 0.0_rt) { return minmod( maxmod(a,b), minmod(2.0_rt*a,2.0_rt*b)); - else + } else { return 0.0_rt; + } } // stage2 slope limiting AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -379,7 +385,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -396,7 +402,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -414,7 +420,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_m = 0; - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } return U - U_m; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -431,7 +437,7 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) #if defined(WARPX_DIM_3D) // U is zero if N is zero, Check positivity before dividing amrex::Real U_p = 0; - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } return U_p - U; #else amrex::ignore_unused(N, NU, U, i, j, k); @@ -450,11 +456,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k-1) > 0) U_m = NU(i,j,k-1)/N(i,j,k-1); + if (N(i,j,k-1) > 0) { U_m = NU(i,j,k-1)/N(i,j,k-1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j-1,k) > 0) U_m = NU(i,j-1,k)/N(i,j-1,k); + if (N(i,j-1,k) > 0) { U_m = NU(i,j-1,k)/N(i,j-1,k); } #else - if (N(i-1,j,k) > 0) U_m = NU(i-1,j,k)/N(i-1,j,k); + if (N(i-1,j,k) > 0) { U_m = NU(i-1,j,k)/N(i-1,j,k); } #endif // Return the difference @@ -471,11 +477,11 @@ const amrex::Array4& NU, amrex::Real& U, int i, int j, int k) // U is zero if N is zero, Check positivity before dividing #if defined(WARPX_DIM_3D) - if (N(i,j,k+1) > 0) U_p = NU(i,j,k+1)/N(i,j,k+1); + if (N(i,j,k+1) > 0) { U_p = NU(i,j,k+1)/N(i,j,k+1); } #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - if (N(i,j+1,k) > 0) U_p = NU(i,j+1,k)/N(i,j+1,k); + if (N(i,j+1,k) > 0) { U_p = NU(i,j+1,k)/N(i,j+1,k); } #else - if (N(i+1,j,k) > 0) U_p = NU(i+1,j,k)/N(i+1,j,k); + if (N(i+1,j,k) > 0) { U_p = NU(i+1,j,k)/N(i+1,j,k); } #endif // Return the difference diff --git a/Source/Fluids/WarpXFluidContainer.H b/Source/Fluids/WarpXFluidContainer.H index 172ef4c4209..04ec4d9e80d 100644 --- a/Source/Fluids/WarpXFluidContainer.H +++ b/Source/Fluids/WarpXFluidContainer.H @@ -23,7 +23,7 @@ * * WarpXFluidContainer contains the main functions for initialization, * interaction with the grid (field gather and current deposition), fluid - * source and push, advective update and updates for non-intertial terms. + * source and push, advective update and updates for non-inertial terms. */ class WarpXFluidContainer { @@ -45,7 +45,7 @@ public: void ReadParameters (); /** - * Evolve updates a single timestep (dt) of the cold relativistic fluid eqautions + * Evolve updates a single timestep (dt) of the cold relativistic fluid equations */ void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, @@ -99,12 +99,12 @@ public: * \param[in] Bx Yee magnetic field (x) * \param[in] By Yee magnetic field (y) * \param[in] Bz Yee magnetic field (z) - * \param[in] cur_time Current time + * \param[in] t Current time */ void GatherAndPush (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, - amrex::Real cur_time); + amrex::Real t); /** * DepositCurrent interpolates the fluid current density comps. onto the Yee grid and diff --git a/Source/Fluids/WarpXFluidContainer.cpp b/Source/Fluids/WarpXFluidContainer.cpp index 5b6eacf2fc7..1915e4bf772 100644 --- a/Source/Fluids/WarpXFluidContainer.cpp +++ b/Source/Fluids/WarpXFluidContainer.cpp @@ -27,8 +27,8 @@ WarpXFluidContainer::WarpXFluidContainer(int nlevs_max, int ispecies, const std: // Initialize injection objects const ParmParse pp_species_name(species_name); - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "none", h_inj_mom, + SpeciesUtils::parseDensity(species_name, "", h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, "", "none", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); if (h_inj_rho) { #ifdef AMREX_USE_GPU @@ -332,7 +332,7 @@ void WarpXFluidContainer::ApplyBcFluidsAndComms (int lev) [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - // If the cell is is first gaurd cell & the dimension is non + // If the cell is is first guard cell & the dimension is non // periodic, then copy Q_{i+1} = Q_{i-1}. // Don't check r-dir in Z: #if defined(WARPX_DIM_3D) @@ -495,7 +495,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #endif //N and NU are always defined at the nodes, the tmp_Q_* are defined - //inbetween the nodes (i.e. on the staggered Yee grid) and store the + //in between the nodes (i.e. on the staggered Yee grid) and store the //values of N and U at these points. //(i.e. the 4 components correspond to N + the 3 components of U) // Extract the temporary arrays for edge values @@ -594,7 +594,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Select the specific implementation depending on dimensionality #if defined(WARPX_DIM_3D) - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; amrex::Real JdU1x = J11x*dU1x ; amrex::Real JdU2x = J22x*dU2x ; @@ -636,7 +636,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) - // Have no RZ-intertial source for primative vars if in XZ + // Have no RZ-inertial source for primitive vars if in XZ amrex::Real N_source = 0.0; #if defined(WARPX_DIM_RZ) @@ -652,7 +652,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // NUz -> -NUz (NUz_arr(i-1,j,k) -> NUz_arr(i+1,j,k)) dU0x = ave( -UpDx_N(N_arr,i,j,k) , UpDx_N(N_arr,i,j,k) ); // First term in the ave is: U_{x,y} + U_{x,y}_p, - // which can be writen as 2*U_{x,y} + UpDx_U(U_{x,y}) + // which can be written as 2*U_{x,y} + UpDx_U(U_{x,y}) dU1x = ave( 2.0_rt*Ux + UpDx_U(N_arr,NUx_arr,Ux,i,j,k) , UpDx_U(N_arr,NUx_arr,Ux,i,j,k) ); dU2x = ave( 2.0_rt*Uy + UpDx_U(N_arr,NUy_arr,Uy,i,j,k) , UpDx_U(N_arr,NUy_arr,Uy,i,j,k) ); dU3x = ave( -UpDx_U(N_arr,NUz_arr,Uz,i,j,k) , UpDx_U(N_arr,NUz_arr,Uz,i,j,k) ); @@ -669,7 +669,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) } #endif - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0x = J00x*dU0x + J01x*dU1x + J02x*dU2x + J03x*dU3x; amrex::Real JdU1x = J11x*dU1x; amrex::Real JdU2x = J22x*dU2x; @@ -699,7 +699,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) #else - // Compute U ([ N, U]) at the halfsteps (U_tidle) using the slopes (dU) + // Compute U ([ N, U]) at the halfsteps (U_tilde) using the slopes (dU) amrex::Real JdU0z = J00z*dU0z + J01z*dU1z + J02z*dU2z + J03z*dU3z; amrex::Real JdU1z = J11z*dU1z; amrex::Real JdU2z = J22z*dU2z; @@ -734,7 +734,7 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) ); } - // Given the values of `U_minus` and `U_plus`, compute fluxes inbetween nodes, and update N, NU accordingly + // Given the values of `U_minus` and `U_plus`, compute fluxes in between nodes, and update N, NU accordingly #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -821,10 +821,12 @@ void WarpXFluidContainer::AdvectivePush_Muscl (int lev) // Radial Surfaces amrex::Real S_Ar_plus = 2.0_rt*MathConst::pi*(r + dr/2.0_rt)*dz; amrex::Real S_Ar_minus = 2.0_rt*MathConst::pi*(r - dr/2.0_rt)*dz; - if (i == domain.smallEnd(0)) + if (i == domain.smallEnd(0)) { S_Ar_minus = 0.0_rt; - if (i == domain.bigEnd(0)+1) + } + if (i == domain.bigEnd(0)+1) { S_Ar_plus = 2.0_rt*MathConst::pi*(r)*dz; + } // Impose "none" boundaries // Condition: Vx(r) = 0 at boundaries @@ -1222,7 +1224,7 @@ void WarpXFluidContainer::DepositCharge (int lev, amrex::MultiFab &rho, int icom amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - if ( owner_mask_rho_arr(i,j,k) ) rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); + if ( owner_mask_rho_arr(i,j,k) ) { rho_arr(i,j,k,icomp) += q*N_arr(i,j,k); } } ); } @@ -1333,19 +1335,19 @@ void WarpXFluidContainer::DepositCurrent( { amrex::Real jx_tmp = ablastr::coarsen::sample::Interp(tmp_jx_fluid_arr, j_nodal_type, jx_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_x_arr(i,j,k) ) jx_arr(i, j, k) += jx_tmp; + if ( owner_mask_x_arr(i,j,k) ) { jx_arr(i, j, k) += jx_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jy_tmp = ablastr::coarsen::sample::Interp(tmp_jy_fluid_arr, j_nodal_type, jy_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_y_arr(i,j,k) ) jy_arr(i, j, k) += jy_tmp; + if ( owner_mask_y_arr(i,j,k) ) { jy_arr(i, j, k) += jy_tmp; } }, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { amrex::Real jz_tmp = ablastr::coarsen::sample::Interp(tmp_jz_fluid_arr, j_nodal_type, jz_type, coarsening_ratio, i, j, k, 0); - if ( owner_mask_z_arr(i,j,k) ) jz_arr(i, j, k) += jz_tmp; + if ( owner_mask_z_arr(i,j,k) ) { jz_arr(i, j, k) += jz_tmp; } } ); } diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index f010d87d21e..34807338c4d 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -2,15 +2,16 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - WarpXAMReXInit.cpp + ExternalField.cpp + GetTemperature.cpp + GetVelocity.cpp InjectorDensity.cpp InjectorMomentum.cpp PlasmaInjector.cpp - WarpXInitData.cpp TemperatureProperties.cpp VelocityProperties.cpp - GetTemperature.cpp - GetVelocity.cpp ReconnectionPerturbation.cpp + WarpXAMReXInit.cpp + WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H new file mode 100644 index 00000000000..7b64339a042 --- /dev/null +++ b/Source/Initialization/ExternalField.H @@ -0,0 +1,69 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_H_ +#define EXTERNAL_FIELD_H_ + +#include "ExternalField_fwd.H" + +#include +#include +#include +#include + +#include +#include + +enum class ExternalFieldType +{ + default_zero, + constant, + parse_ext_grid_function, + read_from_file +}; + +/** + * \brief Struct to store data related to external electromagnetic fields + * (flags, field values, and field parsers) + */ +struct ExternalFieldParams +{ + + /** + * \brief The constructor reads and stores the parameters related to the external fields. + * "pp_warpx" must point at the "warpx" parameter group in the inputfile. + */ + ExternalFieldParams(const amrex::ParmParse& pp_warpx); + + //! Initial electric field on the grid + amrex::GpuArray E_external_grid = {0,0,0}; + //! Initial magnetic field on the grid + amrex::GpuArray B_external_grid = {0,0,0}; + + //! Initialization type for external magnetic field on the grid + ExternalFieldType B_ext_grid_type = ExternalFieldType::default_zero; + //! Initialization type for external electric field on the grid + ExternalFieldType E_ext_grid_type = ExternalFieldType::default_zero; + + //! User-defined parser to initialize x-component of the magnetic field on the grid + std::unique_ptr Bxfield_parser; + //! User-defined parser to initialize y-component of the magnetic field on the grid + std::unique_ptr Byfield_parser; + //! User-defined parser to initialize z-component of the magnetic field on the grid + std::unique_ptr Bzfield_parser; + //! User-defined parser to initialize x-component of the electric field on the grid + std::unique_ptr Exfield_parser; + //! User-defined parser to initialize y-component of the electric field on the grid + std::unique_ptr Eyfield_parser; + //! User-defined parser to initialize z-component of the electric field on the grid + std::unique_ptr Ezfield_parser; + + //! Path of the file where external fields are stored + std::string external_fields_path; +}; + +#endif //EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp new file mode 100644 index 00000000000..909e5c01977 --- /dev/null +++ b/Source/Initialization/ExternalField.cpp @@ -0,0 +1,182 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "ExternalField.H" + +#include "Utils/TextMsg.H" +#include "Utils/Parser/ParserUtils.H" + +#include + +#include +#include + +namespace +{ + + enum class EMFieldType{E, B}; + + template + ExternalFieldType string_to_external_field_type(std::string s) + { + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + + if constexpr (T == EMFieldType::E){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_b_ext_grid_function", + "parse_B_ext_grid_function can be used only for B_ext_grid_init_style"); + } + else{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(s != "parse_e_ext_grid_function", + "parse_E_ext_grid_function can be used only for E_ext_grid_init_style"); + } + + if ( s.empty() || s == "default"){ + return ExternalFieldType::default_zero; + } + else if ( s == "constant"){ + return ExternalFieldType::constant; + } + else if ( s == "parse_b_ext_grid_function" || s == "parse_e_ext_grid_function"){ + return ExternalFieldType::parse_ext_grid_function; + } + else if ( s == "read_from_file"){ + return ExternalFieldType::read_from_file; + } + else{ + WARPX_ABORT_WITH_MESSAGE( + "'" + s + "' is an unknown external field type!"); + } + + return ExternalFieldType::default_zero; + } +} + +ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) +{ + // default values of E_external_grid and B_external_grid + // are used to set the E and B field when "constant" or + // "parser" is not explicitly used in the input. + std::string B_ext_grid_s; + pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); + B_ext_grid_type = string_to_external_field_type(B_ext_grid_s); + + std::string E_ext_grid_s; + pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); + E_ext_grid_type = string_to_external_field_type(E_ext_grid_s); + + // + // Constant external field + // + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_B = std::vector(3); + if (B_ext_grid_type == ExternalFieldType::constant) { + utils::parser::getArrWithParser(pp_warpx, "B_external_grid", v_B); + } + std::copy(v_B.begin(), v_B.end(), B_external_grid.begin()); + + // if the input string is "constant", the values for the + // external grid must be provided in the input. + auto v_E = std::vector(3); + if (E_ext_grid_type == ExternalFieldType::constant) { + utils::parser::getArrWithParser(pp_warpx, "E_external_grid", v_E); + } + std::copy(v_E.begin(), v_E.end(), E_external_grid.begin()); + //___________________________________________________________________________ + + + // + // External E field with parser + // + + // if the input string for the B-field is "parse_b_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + + //! Strings storing parser function to initialize the components of the magnetic field on the grid + std::string str_Bx_ext_grid_function; + std::string str_By_ext_grid_function; + std::string str_Bz_ext_grid_function; + +#ifdef WARPX_DIM_RZ + std::stringstream warnMsg; + warnMsg << "Parser for external B (r and theta) fields does not work with RZ\n" + << "The initial Br and Bt fields are currently hardcoded to 0.\n" + << "The initial Bz field should only be a function of z.\n"; + ablastr::warn_manager::WMRecordWarning( + "Inputs", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); + str_Bx_ext_grid_function = "0"; + str_By_ext_grid_function = "0"; +#else + utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", + str_Bx_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", + str_By_ext_grid_function); +#endif + utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", + str_Bz_ext_grid_function); + + Bxfield_parser = std::make_unique( + utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); + Byfield_parser = std::make_unique( + utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); + Bzfield_parser = std::make_unique( + utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External B field with parser + // + + // if the input string for the E-field is "parse_e_ext_grid_function", + // then the analytical expression or function must be + // provided in the input file. + if (E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + +#ifdef WARPX_DIM_RZ + WARPX_ABORT_WITH_MESSAGE( + "E parser for external fields does not work with RZ -- TO DO"); +#endif + + //! Strings storing parser function to initialize the components of the electric field on the grid + std::string str_Ex_ext_grid_function; + std::string str_Ey_ext_grid_function; + std::string str_Ez_ext_grid_function; + + utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", + str_Ex_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", + str_Ey_ext_grid_function); + utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", + str_Ez_ext_grid_function); + + Exfield_parser = std::make_unique( + utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); + Eyfield_parser = std::make_unique( + utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); + Ezfield_parser = std::make_unique( + utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + } + //___________________________________________________________________________ + + + // + // External fields from file + // + + if (E_ext_grid_type == ExternalFieldType::read_from_file || + B_ext_grid_type == ExternalFieldType::read_from_file){ + std::string read_fields_from_path="./"; + pp_warpx.query("read_fields_from_path", external_fields_path); + } + //___________________________________________________________________________ +} diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H new file mode 100644 index 00000000000..f7a8547ecce --- /dev/null +++ b/Source/Initialization/ExternalField_fwd.H @@ -0,0 +1,13 @@ +/* Copyright 2023 Luca Fedeli + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef EXTERNAL_FIELD_FWD_H_ +#define EXTERNAL_FIELD_FWD_H_ + +struct ExternalFieldParams; + +#endif //EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index b0e71b52fa7..f7ee6cff138 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -130,7 +130,7 @@ namespace { u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); // Rejection method xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) reject = false; + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } } } else { // Mean velocity magnitude is greater than thermal velocity @@ -151,7 +151,7 @@ namespace { } // Rejection method const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) reject = false; + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } } } @@ -197,7 +197,7 @@ struct InjectorMomentumGaussianFlux u_th = m_uz_th; } amrex::Real u = generateGaussianFluxDist(u_m, u_th, engine); - if (m_flux_direction < 0) u = -u; + if (m_flux_direction < 0) { u = -u; } // Note: Here, in RZ geometry, the variables `ux` and `uy` actually // correspond to the radial and azimuthal component of the momentum @@ -309,7 +309,7 @@ struct InjectorMomentumBoltzmann // The following condition is equation 32 in Zenitani 2015 // (Phys. Plasmas 22, 042116) , called the flipping method. It - // transforms the intergral: d3x' -> d3x where d3x' is the volume + // transforms the integral: d3x' -> d3x where d3x' is the volume // element for positions in the boosted frame. The particle positions // and densities can be initialized in the simulation frame. // The flipping method can transform any symmetric distribution from one @@ -338,10 +338,10 @@ struct InjectorMomentumBoltzmann { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } const amrex::Real beta = velocity(x,y,z); int const dir = velocity.direction(); - const auto gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + const auto gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } @@ -351,7 +351,7 @@ private: GetTemperature temperature; }; -// struct whose getMomentum returns momentum for 1 particle with relativistc +// struct whose getMomentum returns momentum for 1 particle with relativistic // drift velocity beta, from the Maxwell-Juttner distribution. Method is from // Zenitani 2015 (Phys. Plasmas 22, 042116). struct InjectorMomentumJuttner @@ -409,8 +409,8 @@ struct InjectorMomentumJuttner // The value of dir is the boost direction to be transformed. u[dir] = u[dir]*(2._rt*x1-1._rt); x1 = amrex::Random(engine); - // The following condition is equtaion 32 in Zenitani, called - // The flipping method. It transforms the intergral: d3x' -> d3x + // The following condition is equation 32 in Zenitani, called + // The flipping method. It transforms the integral: d3x' -> d3x // where d3x' is the volume element for positions in the boosted frame. // The particle positions and densities can be initialized in the // simulation frame with this method. @@ -441,10 +441,10 @@ struct InjectorMomentumJuttner { using namespace amrex::literals; amrex::Real u[3]; - for (auto& el : u) el = 0.0_rt; + for (auto& el : u) { el = 0.0_rt; } amrex::Real const beta = velocity(x,y,z); int const dir = velocity.direction(); - auto const gamma = static_cast(1._rt/sqrt(1._rt-beta*beta)); + auto const gamma = 1._rt/std::sqrt(1._rt-beta*beta); u[dir] = gamma*beta; return amrex::XDim3 {u[0],u[1],u[2]}; } @@ -488,7 +488,7 @@ private: amrex::Real u_over_r; }; -// struct whose getMomentumm returns local momentum computed from parser. +// struct whose getMomentum returns local momentum computed from parser. struct InjectorMomentumParser { InjectorMomentumParser (amrex::ParserExecutor<3> const& a_ux_parser, @@ -517,7 +517,7 @@ struct InjectorMomentumParser amrex::ParserExecutor<3> m_ux_parser, m_uy_parser, m_uz_parser; }; -// struct whose getMomentumm returns local momentum and thermal spread computed from parser. +// struct whose getMomentum returns local momentum and thermal spread computed from parser. struct InjectorMomentumGaussianParser { InjectorMomentumGaussianParser (amrex::ParserExecutor<3> const& a_ux_m_parser, diff --git a/Source/Initialization/InjectorMomentum.cpp b/Source/Initialization/InjectorMomentum.cpp index 35513260827..4642d3a0cc6 100644 --- a/Source/Initialization/InjectorMomentum.cpp +++ b/Source/Initialization/InjectorMomentum.cpp @@ -9,7 +9,7 @@ using namespace amrex; -void InjectorMomentum::clear () +void InjectorMomentum::clear () // NOLINT(readability-make-member-function-const) { switch (type) { diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 99aba7bc88b..c86ed432bd1 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -43,19 +43,19 @@ struct InjectorPositionRandomPlane using namespace amrex::literals; #if ((defined WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) // In RZ, the 3 components of the `XDim3` vector below correspond to r, theta, z respectively - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; - else return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), amrex::Random(engine)}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, amrex::Random(engine)}; } + else { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } #elif (defined(WARPX_DIM_XZ)) // In 2D, the 2 first components of the `XDim3` vector below correspond to x and z - if (dir == 0) return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; - else return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{0._rt, amrex::Random(engine), 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), amrex::Random(engine), 0._rt}; } + else { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt }; } #elif (defined(WARPX_DIM_1D_Z)) // In 2D, the first components of the `XDim3` vector below correspond to z - if (dir == 0) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - if (dir == 1) return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; - else return amrex::XDim3{0._rt, 0._rt, 0._rt}; + if (dir == 0) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + if (dir == 1) { return amrex::XDim3{amrex::Random(engine), 0._rt, 0._rt}; } + else { return amrex::XDim3{0._rt, 0._rt, 0._rt}; } #endif } private: @@ -221,7 +221,7 @@ struct InjectorPosition z <= zmax and z >= zmin); } - // bool: whether the region defined by lo and hi overaps with the plasma region + // bool: whether the region defined by lo and hi overlaps with the plasma region [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 307f9c515dc..bf4f78a2168 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -1,12 +1,13 @@ -CEXE_sources += WarpXAMReXInit.cpp -CEXE_sources += WarpXInitData.cpp -CEXE_sources += PlasmaInjector.cpp +CEXE_sources += ExternalField.cpp +CEXE_sources += GetTemperature.cpp +CEXE_sources += GetVelocity.cpp CEXE_sources += InjectorDensity.cpp CEXE_sources += InjectorMomentum.cpp +CEXE_sources += PlasmaInjector.cpp CEXE_sources += TemperatureProperties.cpp -CEXE_sources += GetTemperature.cpp CEXE_sources += VelocityProperties.cpp -CEXE_sources += GetVelocity.cpp CEXE_sources += ReconnectionPerturbation.cpp +CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index cec84440159..2651aad455f 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -45,7 +46,8 @@ public: /** Default constructor*/ PlasmaInjector () = default; - PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom); + PlasmaInjector (int ispecies, const std::string& name, const amrex::Geometry& geom, + const std::string& src_name=""); // Default move and copy operations PlasmaInjector(const PlasmaInjector&) = delete; @@ -72,9 +74,8 @@ public: [[nodiscard]] amrex::XDim3 getMomentum ( amrex::Real x, amrex::Real y, amrex::Real z) const noexcept; - [[nodiscard]] amrex::Real getCharge () {return charge;} - [[nodiscard]] amrex::Real getMass () {return mass;} - [[nodiscard]] PhysicalSpecies getPhysicalSpecies() const {return physical_species;} + bool queryCharge (amrex::ParticleReal& a_charge) const; + bool queryMass (amrex::ParticleReal& a_mass) const; // bool: whether the initial injection of particles should be done // This routine is called during initialization of the plasma. @@ -111,6 +112,8 @@ public: long npart; int do_symmetrize = 0; int symmetrization_order = 4; + bool do_focusing = false; + amrex::Real focal_distance; bool external_file = false; //! initialize from an openPMD file amrex::Real z_shift = 0.0; //! additional z offset for particle positions @@ -138,17 +141,19 @@ public: amrex::Real density_min = std::numeric_limits::epsilon(); amrex::Real density_max = std::numeric_limits::max(); - InjectorPosition* getInjectorPosition (); - InjectorPosition* getInjectorFluxPosition (); - InjectorDensity* getInjectorDensity (); + [[nodiscard]] InjectorPosition* getInjectorPosition () const; + [[nodiscard]] InjectorPosition* getInjectorFluxPosition () const; + [[nodiscard]] InjectorDensity* getInjectorDensity () const; - InjectorFlux* getInjectorFlux (); - InjectorMomentum* getInjectorMomentumDevice (); - InjectorMomentum* getInjectorMomentumHost (); + [[nodiscard]] InjectorFlux* getInjectorFlux () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumDevice () const; + [[nodiscard]] InjectorMomentum* getInjectorMomentumHost () const; protected: - amrex::Real mass, charge; + bool mass_from_source = false; + bool charge_from_source = false; + amrex::ParticleReal mass, charge; PhysicalSpecies physical_species = PhysicalSpecies::unspecified; @@ -156,6 +161,7 @@ protected: int species_id; std::string species_name; + std::string source_name; std::unique_ptr h_inj_pos; InjectorPosition* d_inj_pos = nullptr; @@ -185,15 +191,15 @@ protected: std::unique_ptr h_mom_temp; std::unique_ptr h_mom_vel; - void setupSingleParticle (const amrex::ParmParse& pp_species_name); - void setupMultipleParticles (const amrex::ParmParse& pp_species_name); - void setupGaussianBeam (const amrex::ParmParse& pp_species_name); - void setupNRandomPerCell (const amrex::ParmParse& pp_species_name); - void setupNFluxPerCell (const amrex::ParmParse& pp_species_name); - void setupNuniformPerCell (const amrex::ParmParse& pp_species_name); - void setupExternalFile (const amrex::ParmParse& pp_species_name); + void setupSingleParticle (amrex::ParmParse const& pp_species); + void setupMultipleParticles (amrex::ParmParse const& pp_species); + void setupGaussianBeam (amrex::ParmParse const& pp_species); + void setupNRandomPerCell (amrex::ParmParse const& pp_species); + void setupNFluxPerCell (amrex::ParmParse const& pp_species); + void setupNuniformPerCell (amrex::ParmParse const& pp_species); + void setupExternalFile (amrex::ParmParse const& pp_species); - void parseFlux (const amrex::ParmParse& pp_species_name); + void parseFlux (amrex::ParmParse const& pp_species); }; #endif diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp index e5b503af786..7ae5a9db3d3 100644 --- a/Source/Initialization/PlasmaInjector.cpp +++ b/Source/Initialization/PlasmaInjector.cpp @@ -46,10 +46,9 @@ using namespace amrex::literals; PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, - const amrex::Geometry& geom): - species_id{ispecies}, species_name{name} + const amrex::Geometry& geom, const std::string& src_name): + species_id{ispecies}, species_name{name}, source_name{src_name} { - const amrex::ParmParse pp_species_name(species_name); #ifdef AMREX_USE_GPU static_assert(std::is_trivially_copyable::value, @@ -60,7 +59,9 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, "InjectorMomentum must be trivially copyable"); #endif - pp_species_name.query("radially_weighted", radially_weighted); + const amrex::ParmParse pp_species(species_name); + + utils::parser::queryWithParser(pp_species, source_name, "radially_weighted", radially_weighted); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(radially_weighted, "ERROR: Only radially_weighted=true is supported"); // Unlimited boundaries @@ -102,43 +103,41 @@ PlasmaInjector::PlasmaInjector (int ispecies, const std::string& name, } # endif - utils::parser::queryWithParser(pp_species_name, "xmin", xmin); - utils::parser::queryWithParser(pp_species_name, "ymin", ymin); - utils::parser::queryWithParser(pp_species_name, "zmin", zmin); - utils::parser::queryWithParser(pp_species_name, "xmax", xmax); - utils::parser::queryWithParser(pp_species_name, "ymax", ymax); - utils::parser::queryWithParser(pp_species_name, "zmax", zmax); + utils::parser::queryWithParser(pp_species, source_name, "xmin", xmin); + utils::parser::queryWithParser(pp_species, source_name, "ymin", ymin); + utils::parser::queryWithParser(pp_species, source_name, "zmin", zmin); + utils::parser::queryWithParser(pp_species, source_name, "xmax", xmax); + utils::parser::queryWithParser(pp_species, source_name, "ymax", ymax); + utils::parser::queryWithParser(pp_species, source_name, "zmax", zmax); - utils::parser::queryWithParser(pp_species_name, "density_min", density_min); - utils::parser::queryWithParser(pp_species_name, "density_max", density_max); + utils::parser::queryWithParser(pp_species, source_name, "density_min", density_min); + utils::parser::queryWithParser(pp_species, source_name, "density_max", density_max); std::string injection_style = "none"; - // Parse injection style - pp_species_name.query("injection_style", injection_style); + utils::parser::query(pp_species, source_name, "injection_style", injection_style); std::transform(injection_style.begin(), - injection_style.end(), - injection_style.begin(), - ::tolower); - SpeciesUtils::extractSpeciesProperties(species_name, injection_style, charge, mass, physical_species); + injection_style.end(), + injection_style.begin(), + ::tolower); num_particles_per_cell_each_dim.assign(3, 0); if (injection_style == "singleparticle") { - setupSingleParticle(pp_species_name); + setupSingleParticle(pp_species); return; } else if (injection_style == "multipleparticles") { - setupMultipleParticles(pp_species_name); + setupMultipleParticles(pp_species); return; } else if (injection_style == "gaussian_beam") { - setupGaussianBeam(pp_species_name); + setupGaussianBeam(pp_species); } else if (injection_style == "nrandompercell") { - setupNRandomPerCell(pp_species_name); + setupNRandomPerCell(pp_species); } else if (injection_style == "nfluxpercell") { - setupNFluxPerCell(pp_species_name); + setupNFluxPerCell(pp_species); } else if (injection_style == "nuniformpercell") { - setupNuniformPerCell(pp_species_name); + setupNuniformPerCell(pp_species); } else if (injection_style == "external_file") { - setupExternalFile(pp_species_name); + setupExternalFile(pp_species); } else if (injection_style != "none") { SpeciesUtils::StringParseAbortMessage("Injection style", injection_style); } @@ -184,36 +183,26 @@ PlasmaInjector::~PlasmaInjector () PlasmaInjector::~PlasmaInjector () = default; #endif -void PlasmaInjector::setupSingleParticle (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupSingleParticle (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "single_particle_pos", single_particle_pos, 0, 3); - utils::parser::getArrWithParser( - pp_species_name, "single_particle_u", single_particle_u, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_pos", single_particle_pos, 0, 3); + utils::parser::getArrWithParser(pp_species, source_name, "single_particle_u", single_particle_u, 0, 3); for (auto& x : single_particle_u) { x *= PhysConst::c; } - utils::parser::getWithParser( - pp_species_name, "single_particle_weight", single_particle_weight); + utils::parser::getWithParser(pp_species, source_name, "single_particle_weight", single_particle_weight); add_single_particle = true; } -void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupMultipleParticles (amrex::ParmParse const& pp_species) { - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_x", multiple_particles_pos_x); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_y", multiple_particles_pos_y); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_pos_z", multiple_particles_pos_z); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_ux", multiple_particles_ux); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uy", multiple_particles_uy); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_uz", multiple_particles_uz); - utils::parser::getArrWithParser( - pp_species_name, "multiple_particles_weight", multiple_particles_weight); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_x", multiple_particles_pos_x); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_y", multiple_particles_pos_y); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_pos_z", multiple_particles_pos_z); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_ux", multiple_particles_ux); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uy", multiple_particles_uy); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_uz", multiple_particles_uz); + utils::parser::getArrWithParser(pp_species, source_name, "multiple_particles_weight", multiple_particles_weight); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( ((multiple_particles_pos_x.size() == multiple_particles_pos_y.size()) && (multiple_particles_pos_x.size() == multiple_particles_pos_z.size()) && @@ -228,29 +217,35 @@ void PlasmaInjector::setupMultipleParticles (const amrex::ParmParse& pp_species_ add_multiple_particles = true; } -void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupGaussianBeam (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser(pp_species_name, "x_m", x_m); - utils::parser::getWithParser(pp_species_name, "y_m", y_m); - utils::parser::getWithParser(pp_species_name, "z_m", z_m); - utils::parser::getWithParser(pp_species_name, "x_rms", x_rms); - utils::parser::getWithParser(pp_species_name, "y_rms", y_rms); - utils::parser::getWithParser(pp_species_name, "z_rms", z_rms); - utils::parser::queryWithParser(pp_species_name, "x_cut", x_cut); - utils::parser::queryWithParser(pp_species_name, "y_cut", y_cut); - utils::parser::queryWithParser(pp_species_name, "z_cut", z_cut); - utils::parser::getWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::getWithParser(pp_species_name, "npart", npart); - pp_species_name.query("do_symmetrize", do_symmetrize); - pp_species_name.query("symmetrization_order", symmetrization_order); + utils::parser::getWithParser(pp_species, source_name, "x_m", x_m); + utils::parser::getWithParser(pp_species, source_name, "y_m", y_m); + utils::parser::getWithParser(pp_species, source_name, "z_m", z_m); + utils::parser::getWithParser(pp_species, source_name, "x_rms", x_rms); + utils::parser::getWithParser(pp_species, source_name, "y_rms", y_rms); + utils::parser::getWithParser(pp_species, source_name, "z_rms", z_rms); + utils::parser::queryWithParser(pp_species, source_name, "x_cut", x_cut); + utils::parser::queryWithParser(pp_species, source_name, "y_cut", y_cut); + utils::parser::queryWithParser(pp_species, source_name, "z_cut", z_cut); + utils::parser::getWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::getWithParser(pp_species, source_name, "npart", npart); + utils::parser::queryWithParser(pp_species, source_name, "do_symmetrize", do_symmetrize); + utils::parser::queryWithParser(pp_species, source_name, "symmetrization_order", symmetrization_order); + const bool focusing_is_specified = pp_species.contains("focal_distance"); + if(focusing_is_specified){ + do_focusing = true; + utils::parser::queryWithParser(pp_species, source_name, "focal_distance", focal_distance); + } const std::set valid_symmetries = {4,8}; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( valid_symmetries.count(symmetrization_order), "Error: Symmetrization only supported to orders 4 or 8 "); gaussian_beam = true; - SpeciesUtils::parseMomentum(species_name, "gaussian_beam", h_inj_mom, + SpeciesUtils::parseMomentum(species_name, source_name, "gaussian_beam", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); + #if defined(WARPX_DIM_XZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE( y_rms > 0._rt, "Error: Gaussian beam y_rms must be strictly greater than 0 in 2D " @@ -265,10 +260,9 @@ void PlasmaInjector::setupGaussianBeam (const amrex::ParmParse& pp_species_name) #endif } -void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNRandomPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell); #if WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -289,17 +283,17 @@ void PlasmaInjector::setupNRandomPerCell (const amrex::ParmParse& pp_species_nam #else d_inj_pos = h_inj_pos.get(); #endif - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "nrandompercell", h_inj_mom, + + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nrandompercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNFluxPerCell (amrex::ParmParse const& pp_species) { - utils::parser::getWithParser( - pp_species_name, "num_particles_per_cell", num_particles_per_cell_real); + utils::parser::getWithParser(pp_species, source_name, "num_particles_per_cell", num_particles_per_cell_real); #ifdef WARPX_DIM_RZ if (WarpX::n_rz_azimuthal_modes > 1) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -309,14 +303,11 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) "(Please visit PR#765 for more information.)"); } #endif - utils::parser::getWithParser( - pp_species_name, "surface_flux_pos", surface_flux_pos); - utils::parser::queryWithParser( - pp_species_name, "flux_tmin", flux_tmin); - utils::parser::queryWithParser( - pp_species_name, "flux_tmax", flux_tmax); + utils::parser::getWithParser(pp_species, source_name, "surface_flux_pos", surface_flux_pos); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmin", flux_tmin); + utils::parser::queryWithParser(pp_species, source_name, "flux_tmax", flux_tmax); std::string flux_normal_axis_string; - pp_species_name.get("flux_normal_axis", flux_normal_axis_string); + utils::parser::get(pp_species, source_name, "flux_normal_axis", flux_normal_axis_string); flux_normal_axis = -1; #ifdef WARPX_DIM_RZ if (flux_normal_axis_string == "r" || flux_normal_axis_string == "R") { @@ -353,7 +344,7 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) #endif WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_normal_axis >= 0, "Error: Invalid value for flux_normal_axis. It must be " + flux_normal_axis_help); - pp_species_name.get("flux_direction", flux_direction); + utils::parser::getWithParser(pp_species, source_name, "flux_direction", flux_direction); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flux_direction == +1 || flux_direction == -1, "Error: flux_direction must be -1 or +1."); // Construct InjectorPosition with InjectorPositionRandom. @@ -369,15 +360,15 @@ void PlasmaInjector::setupNFluxPerCell (const amrex::ParmParse& pp_species_name) d_flux_pos = h_flux_pos.get(); #endif - parseFlux(pp_species_name); - SpeciesUtils::parseMomentum(species_name, "nfluxpercell", h_inj_mom, + parseFlux(pp_species); + SpeciesUtils::parseMomentum(species_name, source_name, "nfluxpercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel, flux_normal_axis, flux_direction); } -void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupNuniformPerCell (amrex::ParmParse const& pp_species) { // Note that for RZ, three numbers are expected, r, theta, and z. // For 2D, only two are expected. The third is overwritten with 1. @@ -389,9 +380,8 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na #else constexpr int num_required_ppc_each_dim = 3; #endif - utils::parser::getArrWithParser( - pp_species_name, "num_particles_per_cell_each_dim", - num_particles_per_cell_each_dim, 0, num_required_ppc_each_dim); + utils::parser::getArrWithParser(pp_species, source_name, "num_particles_per_cell_each_dim", num_particles_per_cell_each_dim, + 0, num_required_ppc_each_dim); #if WARPX_DIM_XZ num_particles_per_cell_each_dim.push_back(1); #endif @@ -425,14 +415,14 @@ void PlasmaInjector::setupNuniformPerCell (const amrex::ParmParse& pp_species_na num_particles_per_cell = num_particles_per_cell_each_dim[0] * num_particles_per_cell_each_dim[1] * num_particles_per_cell_each_dim[2]; - SpeciesUtils::parseDensity(species_name, h_inj_rho, density_parser); - SpeciesUtils::parseMomentum(species_name, "nuniformpercell", h_inj_mom, + SpeciesUtils::parseDensity(species_name, source_name, h_inj_rho, density_parser); + SpeciesUtils::parseMomentum(species_name, source_name, "nuniformpercell", h_inj_mom, ux_parser, uy_parser, uz_parser, ux_th_parser, uy_th_parser, uz_th_parser, h_mom_temp, h_mom_vel); } -void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::setupExternalFile (amrex::ParmParse const& pp_species) { #ifndef WARPX_USE_OPENPMD WARPX_ABORT_WITH_MESSAGE( @@ -441,15 +431,15 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) #endif external_file = true; std::string str_injection_file; - pp_species_name.get("injection_file", str_injection_file); + utils::parser::get(pp_species, source_name, "injection_file", str_injection_file); // optional parameters - utils::parser::queryWithParser(pp_species_name, "q_tot", q_tot); - utils::parser::queryWithParser(pp_species_name, "z_shift",z_shift); + utils::parser::queryWithParser(pp_species, source_name, "q_tot", q_tot); + utils::parser::queryWithParser(pp_species, source_name, "z_shift",z_shift); #ifdef WARPX_USE_OPENPMD - const bool charge_is_specified = pp_species_name.contains("charge"); - const bool mass_is_specified = pp_species_name.contains("mass"); - const bool species_is_specified = pp_species_name.contains("species_type"); + const bool charge_is_specified = pp_species.contains("charge"); + const bool mass_is_specified = pp_species.contains("mass"); + const bool species_is_specified = pp_species.contains("species_type"); if (amrex::ParallelDescriptor::IOProcessor()) { m_openpmd_input_series = std::make_unique( @@ -465,70 +455,68 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) std::string const ps_name = it.particles.begin()->first; openPMD::ParticleSpecies ps = it.particles.begin()->second; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("charge") || charge_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'charge' species record. " - "Please specify '" + ps_name + ".charge' or " - "'" + ps_name + ".species_type' in your input file!\n"); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ps.contains("mass") || mass_is_specified || species_is_specified, - std::string("'") + ps_name + - ".injection_file' does not contain a 'mass' species record. " - "Please specify '" + ps_name + ".mass' or " - "'" + ps_name + ".species_type' in your input file!\n"); - - if (charge_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".charge' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".charge' will take precedence.\n"); - } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".species_type' will take precedence.\n"); - } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record - auto p_q_ptr = - ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_q = p_q_ptr.get()[0]; - auto const charge_unit = static_cast(ps["charge"][openPMD::RecordComponent::SCALAR].unitSI()); - charge = p_q * charge_unit; - } - if (mass_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".mass' and '" + - ps_name + ".injection_file' specify a charge.\n'" + - ps_name + ".mass' will take precedence.\n"); - } - else if (species_is_specified) { - ablastr::warn_manager::WMRecordWarning("Species", - "Both '" + ps_name + ".species_type' and '" + - ps_name + ".injection_file' specify a mass.\n'" + - ps_name + ".species_type' will take precedence.\n"); + charge_from_source = ps.contains("charge"); + mass_from_source = ps.contains("mass"); + + if (charge_from_source) { + if (charge_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".charge' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".charge' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if charge is a constant record + auto p_q_ptr = + ps["charge"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_q = p_q_ptr.get()[0]; + auto const charge_unit = static_cast(ps["charge"][openPMD::RecordComponent::SCALAR].unitSI()); + charge = p_q * charge_unit; + } } - else { - // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record - auto p_m_ptr = - ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); - m_openpmd_input_series->flush(); - amrex::ParticleReal const p_m = p_m_ptr.get()[0]; - auto const mass_unit = static_cast(ps["mass"][openPMD::RecordComponent::SCALAR].unitSI()); - mass = p_m * mass_unit; + if (mass_from_source) { + if (mass_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".mass' and '" + + ps_name + ".injection_file' specify a charge.\n'" + + ps_name + ".mass' will take precedence.\n"); + } + else if (species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + ps_name + ".species_type' and '" + + ps_name + ".injection_file' specify a mass.\n'" + + ps_name + ".species_type' will take precedence.\n"); + } + else { + // TODO: Add ASSERT_WITH_MESSAGE to test if mass is a constant record + auto p_m_ptr = + ps["mass"][openPMD::RecordComponent::SCALAR].loadChunk(); + m_openpmd_input_series->flush(); + amrex::ParticleReal const p_m = p_m_ptr.get()[0]; + auto const mass_unit = static_cast(ps["mass"][openPMD::RecordComponent::SCALAR].unitSI()); + mass = p_m * mass_unit; + } } } // IOProcessor - // Broadcast charge and mass to non-IO processors - if (!charge_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&charge, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); - if (!mass_is_specified && !species_is_specified) - amrex::ParallelDescriptor::Bcast(&mass, 1, - amrex::ParallelDescriptor::IOProcessorNumber()); + // Broadcast charge and mass to non-IO processors if read in from the file + std::array flags{charge_from_source, mass_from_source}; + amrex::ParallelDescriptor::Bcast(flags.data(), flags.size(), amrex::ParallelDescriptor::IOProcessorNumber()); + charge_from_source = flags[0]; + mass_from_source = flags[1]; + if (charge_from_source) { + amrex::ParallelDescriptor::Bcast(&charge, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } + if (mass_from_source) { + amrex::ParallelDescriptor::Bcast(&mass, 1, amrex::ParallelDescriptor::IOProcessorNumber()); + } #else WARPX_ABORT_WITH_MESSAGE( "Plasma injection via external_file requires openPMD support: " @@ -539,20 +527,19 @@ void PlasmaInjector::setupExternalFile (const amrex::ParmParse& pp_species_name) // Depending on injection type at runtime, initialize inj_flux // so that inj_flux->getFlux calls // InjectorFlux[Constant or Parser or etc.].getFlux. -void PlasmaInjector::parseFlux (const amrex::ParmParse& pp_species_name) +void PlasmaInjector::parseFlux (amrex::ParmParse const& pp_species) { // parse flux information std::string flux_prof_s; - pp_species_name.get("flux_profile", flux_prof_s); + utils::parser::get(pp_species, source_name, "flux_profile", flux_prof_s); std::transform(flux_prof_s.begin(), flux_prof_s.end(), flux_prof_s.begin(), ::tolower); if (flux_prof_s == "constant") { - utils::parser::getWithParser(pp_species_name, "flux", flux); + utils::parser::getWithParser(pp_species, source_name, "flux", flux); // Construct InjectorFlux with InjectorFluxConstant. h_inj_flux.reset(new InjectorFlux((InjectorFluxConstant*)nullptr, flux)); } else if (flux_prof_s == "parse_flux_function") { - utils::parser::Store_parserString( - pp_species_name, "flux_function(x,y,z,t)", str_flux_function); + utils::parser::Store_parserString(pp_species, source_name, "flux_function(x,y,z,t)", str_flux_function); // Construct InjectorFlux with InjectorFluxParser. flux_parser = std::make_unique( utils::parser::makeParser(str_flux_function,{"x","y","z","t"})); @@ -595,38 +582,56 @@ bool PlasmaInjector::overlapsWith (const amrex::XDim3& lo, || (zmin > hi.z) || (zmax < lo.z) ); } +bool +PlasmaInjector::queryCharge (amrex::ParticleReal& a_charge) const +{ + if (charge_from_source) { + a_charge = charge; + } + return charge_from_source; +} + +bool +PlasmaInjector::queryMass (amrex::ParticleReal& a_mass) const +{ + if (mass_from_source) { + a_mass = mass; + } + return mass_from_source; +} + InjectorPosition* -PlasmaInjector::getInjectorPosition () +PlasmaInjector::getInjectorPosition () const { return d_inj_pos; } InjectorPosition* -PlasmaInjector::getInjectorFluxPosition () +PlasmaInjector::getInjectorFluxPosition () const { return d_flux_pos; } InjectorDensity* -PlasmaInjector::getInjectorDensity () +PlasmaInjector::getInjectorDensity () const { return d_inj_rho; } InjectorFlux* -PlasmaInjector::getInjectorFlux () +PlasmaInjector::getInjectorFlux () const { return d_inj_flux; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumDevice () +PlasmaInjector::getInjectorMomentumDevice () const { return d_inj_mom; } InjectorMomentum* -PlasmaInjector::getInjectorMomentumHost () +PlasmaInjector::getInjectorMomentumHost () const { return h_inj_mom.get(); } diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index d9d9121c846..4d30b67c851 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -30,8 +30,9 @@ struct TemperatureProperties * information * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - TemperatureProperties (const amrex::ParmParse& pp); + TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of temperature initialization */ TemperatureInitType m_type; diff --git a/Source/Initialization/TemperatureProperties.cpp b/Source/Initialization/TemperatureProperties.cpp index 34e1d2b5fcc..ff73134a24e 100644 --- a/Source/Initialization/TemperatureProperties.cpp +++ b/Source/Initialization/TemperatureProperties.cpp @@ -17,17 +17,17 @@ * If temperature is a constant, store value. If a parser, make and * store the parser function */ -TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { +TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp, std::string const& source_name) { // Set defaults amrex::Real theta; std::string temp_dist_s = "constant"; std::string mom_dist_s; - pp.query("theta_distribution_type", temp_dist_s); - pp.query("momentum_distribution_type", mom_dist_s); + utils::parser::query(pp, source_name, "theta_distribution_type", temp_dist_s); + utils::parser::query(pp, source_name, "momentum_distribution_type", mom_dist_s); if (temp_dist_s == "constant") { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - utils::parser::queryWithParser(pp, "theta", theta), + utils::parser::queryWithParser(pp, source_name, "theta", theta), "Temperature parameter theta not specified"); // Do validation on theta value @@ -58,7 +58,7 @@ TemperatureProperties::TemperatureProperties (const amrex::ParmParse& pp) { } else if (temp_dist_s == "parser") { std::string str_theta_function; - utils::parser::Store_parserString(pp, "theta_function(x,y,z)", str_theta_function); + utils::parser::Store_parserString(pp, source_name, "theta_function(x,y,z)", str_theta_function); m_ptr_temperature_parser = std::make_unique( utils::parser::makeParser(str_theta_function,{"x","y","z"})); diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index a66d0fca653..453d97725e2 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -33,8 +33,9 @@ struct VelocityProperties * store the parser function * * \param[in] pp: Reference to the parameter parser object for the species being initialized + * \param[in] source_name: Optional group name of the input parameters */ - VelocityProperties (const amrex::ParmParse& pp); + VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name); /* Type of velocity initialization */ VelocityInitType m_type; diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 799b29e5be8..14b48c79524 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -11,13 +11,13 @@ #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) +VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string const& source_name) { // Set defaults std::string vel_dist_s = "constant"; std::string vel_dir_s = "x"; - pp.query("bulk_vel_dir", vel_dir_s); + utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); if(vel_dir_s[0] == '-'){ m_sign_dir = -1; } @@ -44,9 +44,9 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) " other character."); } - pp.query("beta_distribution_type", vel_dist_s); + utils::parser::query(pp, source_name, "beta_distribution_type", vel_dist_s); if (vel_dist_s == "constant") { - utils::parser::queryWithParser(pp, "beta", m_velocity); + utils::parser::queryWithParser(pp, source_name, "beta", m_velocity); m_type = VelConstantValue; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( m_velocity > -1 && m_velocity < 1, @@ -56,7 +56,7 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp) } else if (vel_dist_s == "parser") { std::string str_beta_function; - utils::parser::Store_parserString(pp, "beta_function(x,y,z)", str_beta_function); + utils::parser::Store_parserString(pp, source_name, "beta_function(x,y,z)", str_beta_function); m_ptr_velocity_parser = std::make_unique( utils::parser::makeParser(str_beta_function,{"x","y","z"})); diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 50c90d40ae6..613250b5c2f 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -7,8 +7,6 @@ #ifndef WARPX_AMREX_INIT_H_ #define WARPX_AMREX_INIT_H_ -#include - #include namespace warpx::initialization diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8502bdc4ae6..d1dd0bbe90b 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -8,6 +8,7 @@ #include "Initialization/WarpXAMReXInit.H" #include +#include #include #include @@ -29,6 +30,10 @@ namespace { bool the_arena_is_managed = false; // AMReX' default: true pp_amrex.queryAdd("the_arena_is_managed", the_arena_is_managed); + // https://amrex-codes.github.io/amrex/docs_html/InputsComputeBackends.html + std::string omp_threads = "nosmt"; // AMReX' default: system + pp_amrex.queryAdd("omp_threads", omp_threads); + // Work-around: // If warpx.numprocs is used for the domain decomposition, we will not use blocking factor // to generate grids. Nonetheless, AMReX has asserts in place that validate that the diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bf5652e2749..e32f5e50798 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -20,6 +20,7 @@ #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/LinearInterpolation.H" #include "Utils/Logo/GetLogo.H" @@ -123,7 +124,7 @@ WarpX::PostProcessBaseGrids (BoxArray& ba0) const #if defined(WARPX_DIM_3D) for (int k = 0; k < numprocs[2]; ++k) { // The first extra[2] blocks get one extra cell with a total of - // sz[2]+1. The rest get sz[2] cells. The docomposition in y + // sz[2]+1. The rest get sz[2] cells. The decomposition in y // and x directions are similar. int klo = (k < extra[2]) ? k*(sz[2]+1) : (k*sz[2]+extra[2]); int khi = (k < extra[2]) ? klo+(sz[2]+1)-1 : klo+sz[2]-1; @@ -232,6 +233,9 @@ WarpX::PrintMainPICparameters () else if (current_deposition_algo == CurrentDepositionAlgo::Esirkepov){ amrex::Print() << "Current Deposition: | Esirkepov \n"; } + else if (current_deposition_algo == CurrentDepositionAlgo::Villasenor){ + amrex::Print() << "Current Deposition: | Villasenor \n"; + } // Print type of particle pusher if (particle_pusher_algo == ParticlePusherAlgo::Vay){ amrex::Print() << "Particle Pusher: | Vay \n"; @@ -479,8 +483,9 @@ WarpX::InitData () ExecutePythonCallback("beforeInitEsolve"); ComputeSpaceChargeField(reset_fields); ExecutePythonCallback("afterInitEsolve"); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) + if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); + } // Set up an invariant condition through the rest of // execution, that any code besides the field solver that @@ -489,7 +494,7 @@ WarpX::InitData () AddExternalFields(); } - if (restart_chkfile.empty() || write_diagonstics_on_restart) { + if (restart_chkfile.empty() || write_diagnostics_on_restart) { // Write full diagnostics before the first iteration. multi_diags->FilterComputePackFlush(istep[0] - 1); @@ -510,12 +515,12 @@ void WarpX::AddExternalFields () { for (int lev = 0; lev <= finest_level; ++lev) { // FIXME: RZ multimode has more than one component for all these - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -555,7 +560,7 @@ WarpX::InitPML () do_pml_Hi[0][idim] = 1; // on level 0 } } - if (finest_level > 0) do_pml = 1; + if (finest_level > 0) { do_pml = 1; } if (do_pml) { #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) @@ -586,10 +591,12 @@ WarpX::InitPML () // Domain box at level, lev const amrex::Box DomainBox = Geom(lev).Domain(); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) + if (levelBox.smallEnd(idim) == DomainBox.smallEnd(idim)) { do_pml_Lo[lev][idim] = do_pml_Lo[0][idim]; - if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) + } + if (levelBox.bigEnd(idim) == DomainBox.bigEnd(idim)) { do_pml_Hi[lev][idim] = do_pml_Hi[0][idim]; + } } #ifdef WARPX_DIM_RZ @@ -622,8 +629,9 @@ WarpX::ComputePMLFactors () { for (int lev = 0; lev <= finest_level; ++lev) { - if (pml[lev]) + if (pml[lev]) { pml[lev]->ComputePMLFactors(dt[lev]); + } } } } @@ -727,40 +735,15 @@ void WarpX::PostRestart () { mypc->PostRestart(); + for (int lev = 0; lev <= maxLevel(); ++lev) { + LoadExternalFieldsFromFile(lev); + } } void WarpX::InitLevelData (int lev, Real /*time*/) { - - const ParmParse pp_warpx("warpx"); - - // default values of E_external_grid and B_external_grid - // are used to set the E and B field when "constant" or - // "parser" is not explicitly used in the input. - pp_warpx.query("B_ext_grid_init_style", B_ext_grid_s); - std::transform(B_ext_grid_s.begin(), - B_ext_grid_s.end(), - B_ext_grid_s.begin(), - ::tolower); - - pp_warpx.query("E_ext_grid_init_style", E_ext_grid_s); - std::transform(E_ext_grid_s.begin(), - E_ext_grid_s.end(), - E_ext_grid_s.begin(), - ::tolower); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (B_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "B_external_grid", B_external_grid); - - // if the input string is "constant", the values for the - // external grid must be provided in the input. - if (E_ext_grid_s == "constant") - utils::parser::getArrWithParser(pp_warpx, "E_external_grid", E_external_grid); - // initialize the averaged fields only if the averaged algorithm // is activated ('psatd.do_time_averaging=1') const ParmParse pp_psatd("psatd"); @@ -770,37 +753,44 @@ WarpX::InitLevelData (int lev, Real /*time*/) // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if ( ( B_ext_grid_s == "constant") && (lev <= maxlevel_extEMfield_init) ) + const auto is_B_ext_const = + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; + if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Bfield_fp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_fp[lev][i]->setVal(B_external_grid[i]); - } + Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } if (lev > 0) { - Bfield_aux[lev][i]->setVal(B_external_grid[i]); - Bfield_cp[lev][i]->setVal(B_external_grid[i]); - if (fft_do_time_averaging) { - Bfield_avg_cp[lev][i]->setVal(B_external_grid[i]); - } + Bfield_aux[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + Bfield_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + if (fft_do_time_averaging) { + Bfield_avg_cp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); + } } } + // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if ( ( E_ext_grid_s == "constant") && (lev <= maxlevel_extEMfield_init) ) + const auto is_E_ext_const = + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; + if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Efield_fp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_fp[lev][i]->setVal(E_external_grid[i]); + Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } - if (lev > 0) { - Efield_aux[lev][i]->setVal(E_external_grid[i]); - Efield_cp[lev][i]->setVal(E_external_grid[i]); - if (fft_do_time_averaging) { - Efield_avg_cp[lev][i]->setVal(E_external_grid[i]); - } - } + if (lev > 0) { + Efield_aux[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + Efield_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + if (fft_do_time_averaging) { + Efield_avg_cp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); + } + } } } @@ -813,73 +803,49 @@ WarpX::InitLevelData (int lev, Real /*time*/) // provided in the input file. // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if (B_ext_grid_s == "parse_b_ext_grid_function" && (lev <= maxlevel_extEMfield_init)) { + if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { + + // Initialize Bfield_fp with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp[lev][0].get(), + Bfield_fp[lev][1].get(), + Bfield_fp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); - //! Strings storing parser function to initialize the components of the magnetic field on the grid - std::string str_Bx_ext_grid_function; - std::string str_By_ext_grid_function; - std::string str_Bz_ext_grid_function; - -#ifdef WARPX_DIM_RZ - std::stringstream warnMsg; - warnMsg << "Parser for external B (r and theta) fields does not work with RZ\n" - << "The initial Br and Bt fields are currently hardcoded to 0.\n" - << "The initial Bz field should only be a function of z.\n"; - ablastr::warn_manager::WMRecordWarning( - "Inputs", warnMsg.str(), ablastr::warn_manager::WarnPriority::high); - str_Bx_ext_grid_function = "0"; - str_By_ext_grid_function = "0"; -#else - utils::parser::Store_parserString(pp_warpx, "Bx_external_grid_function(x,y,z)", - str_Bx_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "By_external_grid_function(x,y,z)", - str_By_ext_grid_function); -#endif - utils::parser::Store_parserString(pp_warpx, "Bz_external_grid_function(x,y,z)", - str_Bz_ext_grid_function); - - Bxfield_parser = std::make_unique( - utils::parser::makeParser(str_Bx_ext_grid_function,{"x","y","z"})); - Byfield_parser = std::make_unique( - utils::parser::makeParser(str_By_ext_grid_function,{"x","y","z"})); - Bzfield_parser = std::make_unique( - utils::parser::makeParser(str_Bz_ext_grid_function,{"x","y","z"})); - - // Initialize Bfield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + if (lev > 0) { + InitializeExternalFieldsOnGridUsingParser( + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); + } } + ParmParse pp_warpx("warpx"); int IncludeBfieldPerturbation = 0; pp_warpx.query("IncludeBfieldPerturbation",IncludeBfieldPerturbation); if ( (IncludeBfieldPerturbation == 1) && (lev <= maxlevel_extEMfield_init) ) { @@ -895,24 +861,24 @@ WarpX::InitLevelData (int lev, Real /*time*/) Reconnection_Perturbation::AddBfieldPerturbation (Bfield_fp[lev][0].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), lev, PatchType::fine); + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::fine); if (lev > 0) { Reconnection_Perturbation::AddBfieldPerturbation (Bfield_aux[lev][0].get(), Bfield_aux[lev][1].get(), Bfield_aux[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), lev, PatchType::fine); + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::fine); Reconnection_Perturbation::AddBfieldPerturbation (Bfield_cp[lev][0].get(), Bfield_cp[lev][1].get(), Bfield_cp[lev][2].get(), - Bxfield_parser->compile<3>(), - Byfield_parser->compile<3>(), - Bzfield_parser->compile<3>(), lev, PatchType::coarse); + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), lev, PatchType::coarse); } #endif @@ -923,75 +889,55 @@ WarpX::InitLevelData (int lev, Real /*time*/) // provided in the input file. // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation - if (E_ext_grid_s == "parse_e_ext_grid_function" && (lev <= maxlevel_extEMfield_init)) { - -#ifdef WARPX_DIM_RZ - WARPX_ABORT_WITH_MESSAGE( - "E and B parser for external fields does not work with RZ -- TO DO"); -#endif - - //! Strings storing parser function to initialize the components of the electric field on the grid - std::string str_Ex_ext_grid_function; - std::string str_Ey_ext_grid_function; - std::string str_Ez_ext_grid_function; - - utils::parser::Store_parserString(pp_warpx, "Ex_external_grid_function(x,y,z)", - str_Ex_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ey_external_grid_function(x,y,z)", - str_Ey_ext_grid_function); - utils::parser::Store_parserString(pp_warpx, "Ez_external_grid_function(x,y,z)", - str_Ez_ext_grid_function); - - Exfield_parser = std::make_unique( - utils::parser::makeParser(str_Ex_ext_grid_function,{"x","y","z"})); - Eyfield_parser = std::make_unique( - utils::parser::makeParser(str_Ey_ext_grid_function,{"x","y","z"})); - Ezfield_parser = std::make_unique( - utils::parser::makeParser(str_Ez_ext_grid_function,{"x","y","z"})); + if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) + && (lev <= maxlevel_extEMfield_init)) { // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser(Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); + InitializeExternalFieldsOnGridUsingParser( + Efield_fp[lev][0].get(), + Efield_fp[lev][1].get(), + Efield_fp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { - m_fdtd_solver_fp[lev]->EvolveECTRho(Efield_fp[lev], m_edge_lengths[lev], - m_face_areas[lev], ECTRhofield[lev], lev); - + m_fdtd_solver_fp[lev]->EvolveECTRho( + Efield_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], lev); } #endif if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser(Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser(Efield_cp[lev][0].get(), - Efield_cp[lev][1].get(), - Efield_cp[lev][2].get(), - Exfield_parser->compile<3>(), - Eyfield_parser->compile<3>(), - Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::coarse); + InitializeExternalFieldsOnGridUsingParser( + Efield_aux[lev][0].get(), + Efield_aux[lev][1].get(), + Efield_aux[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + + InitializeExternalFieldsOnGridUsingParser( + Efield_cp[lev][0].get(), + Efield_cp[lev][1].get(), + Efield_cp[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::coarse); #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { // We initialize ECTRhofield consistently with the Efield @@ -1003,37 +949,7 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - // Reading external fields from data file - if (add_external_B_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#else - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][0].get(), "B", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][1].get(), "B", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Bfield_fp_external[lev][2].get(), "B", "z"); -#endif - } - if (add_external_E_field) { - std::string read_fields_from_path="./"; - pp_warpx.query("read_fields_from_path", read_fields_from_path); -#if defined(WARPX_DIM_RZ) - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, - "External field reading is not implemented for more than one RZ mode (see #3829)"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "r"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "t"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); -#else - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][0].get(), "E", "x"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][1].get(), "E", "y"); - ReadExternalFieldFromFile(read_fields_from_path, Efield_fp_external[lev][2].get(), "E", "z"); -#endif - } + LoadExternalFieldsFromFile(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1429,6 +1345,37 @@ void WarpX::CheckKnownIssues() #endif } +void +WarpX::LoadExternalFieldsFromFile (int const lev) +{ + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); +#endif + } + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } +} + #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( @@ -1504,12 +1451,12 @@ WarpX::ReadExternalFieldFromFile ( auto FC_chunk_data = FC.loadChunk(chunk_offset,chunk_extent); series.flush(); - auto FC_data_host = FC_chunk_data.get(); + auto *FC_data_host = FC_chunk_data.get(); // Load data to GPU const size_t total_extent = size_t(extent[0]) * extent[1] * extent[2]; amrex::Gpu::DeviceVector FC_data_gpu(total_extent); - auto FC_data = FC_data_gpu.data(); + auto *FC_data = FC_data_gpu.data(); amrex::Gpu::copy(amrex::Gpu::hostToDevice, FC_data_host, FC_data_host + total_extent, FC_data); // Loop over boxes @@ -1548,8 +1495,8 @@ WarpX::ReadExternalFieldFromFile ( #if defined(WARPX_DIM_RZ) // Get index of the external field array - int const ir = floor( (x0-offset0)/file_dr ); - int const iz = floor( (x1-offset1)/file_dz ); + int const ir = std::floor( (x0-offset0)/file_dr ); + int const iz = std::floor( (x1-offset1)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ir * file_dr; @@ -1562,9 +1509,9 @@ WarpX::ReadExternalFieldFromFile ( else { x2 = real_box.lo(2) + k*dx[2] + 0.5_rt*dx[2]; } // Get index of the external field array - int const ix = floor( (x0-offset0)/file_dx ); - int const iy = floor( (x1-offset1)/file_dy ); - int const iz = floor( (x2-offset2)/file_dz ); + int const ix = std::floor( (x0-offset0)/file_dx ); + int const iy = std::floor( (x1-offset1)/file_dy ); + int const iz = std::floor( (x2-offset2)/file_dz ); // Get coordinates of external grid point amrex::Real const xx0 = offset0 + ix * file_dx; @@ -1611,7 +1558,7 @@ WarpX::ReadExternalFieldFromFile ( void WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) { -#if defined(WARPX_DIM_1D) +#if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); #elif defined(WARPX_DIM_XZ) WARPX_ABORT_WITH_MESSAGE("Reading from openPMD for external fields is not known to work with XZ (see #3828)"); diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index e29439c4bc0..0fdb45c64f8 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -114,8 +114,9 @@ void WarpXLaserProfiles::FromFileLaserProfile::update (amrex::Real t) { t += m_params.t_min - m_params.t_delay; - if(t >= m_params.t_max) + if(t >= m_params.t_max) { return; + } const auto idx_times = find_left_right_time_indices(t); const auto idx_t_left = idx_times.first; const auto idx_t_right = idx_times.second; @@ -186,8 +187,8 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ m_params.n_rz_azimuthal_components = static_cast(extent[0]); m_params.nt = static_cast(extent[1]); m_params.nr = static_cast(extent[2]); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); - if(m_params.nr <= 1) WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in lasy file must be >=2"); } + if(m_params.nr <= 1) { WARPX_ABORT_WITH_MESSAGE("nr in lasy file must be >=2"); } // Calculate the min and max of the grid m_params.t_min = static_cast(offset[0] + position[0]*spacing[0]); m_params.t_max = static_cast(m_params.t_min + (m_params.nt-1)*spacing[0]); @@ -240,24 +241,24 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); //Uniform grid flag char flag; inp.read(&flag, 1); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read grid type from binary file"); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(flag, "Binary files with non uniform grid are no longer supported"); //Grid points along t, x and y inp.read(reinterpret_cast(&m_params.nt), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.nx), sizeof(uint32_t)); inp.read(reinterpret_cast(&m_params.ny), sizeof(uint32_t)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); - if(m_params.nt <= 1) WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); - if(m_params.nx <= 1) WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read sizes from binary file"); } + if(m_params.nt <= 1) { WARPX_ABORT_WITH_MESSAGE("nt in binary file must be >=2"); } + if(m_params.nx <= 1) { WARPX_ABORT_WITH_MESSAGE("nx in binary file must be >=2"); } #if (defined(WARPX_DIM_3D) || (defined WARPX_DIM_RZ)) - if(m_params.ny <= 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); + if(m_params.ny <= 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be >=2 in 3D"); } #elif defined(WARPX_DIM_XZ) - if(m_params.ny != 1) WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); + if(m_params.ny != 1) { WARPX_ABORT_WITH_MESSAGE("ny in binary file must be 1 in 2D"); } #endif //Coordinates Vector dbuf_t, dbuf_x, dbuf_y; @@ -274,7 +275,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_ static_cast(dbuf_x.size()*sizeof(double))); inp.read(reinterpret_cast(dbuf_y.dataPtr()), static_cast(dbuf_y.size()*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read coords from binary file"); } m_params.t_min = static_cast(dbuf_t[0]); m_params.t_max = static_cast(dbuf_t[1]); @@ -383,7 +384,7 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, if(ParallelDescriptor::IOProcessor()){ //Read data chunk std::ifstream inp(m_params.binary_file_name, std::ios::binary); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to open binary file"); } inp.exceptions(std::ios_base::failbit | std::ios_base::badbit); #if (defined(WARPX_DIM_3D)) auto skip_amount = 1 + @@ -401,12 +402,12 @@ WarpXLaserProfiles::FromFileLaserProfile::read_binary_data_t_chunk (int t_begin, sizeof(double)*t_begin*m_params.nx*m_params.ny; #endif inp.seekg(static_cast(skip_amount)); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } const int read_size = (i_last - i_first + 1)* m_params.nx*m_params.ny; Vector buf_e(read_size); inp.read(reinterpret_cast(buf_e.dataPtr()), static_cast(read_size*sizeof(double))); - if(!inp) WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); + if(!inp) { WARPX_ABORT_WITH_MESSAGE("Failed to read field data from binary file"); } std::transform(buf_e.begin(), buf_e.end(), h_E_binary_data.begin(), [](auto x) {return static_cast(x);} ); } @@ -440,7 +441,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cartes const auto tmp_y_max = m_params.y_max; const auto tmp_nx = m_params.nx; const auto tmp_ny = m_params.ny; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -523,7 +524,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_cylind const auto tmp_r_max = m_params.r_max; const auto tmp_nr = m_params.nr; const auto tmp_n_rz_azimuthal_components = m_params.n_rz_azimuthal_components; - const auto p_E_lasy_data = m_params.E_lasy_data.dataPtr(); + const auto *const p_E_lasy_data = m_params.E_lasy_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* @@ -625,7 +626,7 @@ WarpXLaserProfiles::FromFileLaserProfile::internal_fill_amplitude_uniform_binary const auto tmp_ny = m_params.ny; #endif const auto tmp_nx = m_params.nx; - const auto p_E_binary_data = m_params.E_binary_data.dataPtr(); + const auto *const p_E_binary_data = m_params.E_binary_data.dataPtr(); const auto tmp_idx_first_time = m_params.first_time_index; const int idx_t_right = idx_t_left+1; const auto t_left = idx_t_left* diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp index ef00d95fd43..96d61d920f1 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileGaussian.cpp @@ -73,10 +73,11 @@ WarpXLaserProfiles::GaussianLaserProfile::init ( m_params.stc_direction[1]*m_common_params.p_X[1] + m_params.stc_direction[2]*m_common_params.p_X[2]; - if (arg < -1.0_rt || arg > 1.0_rt) + if (arg < -1.0_rt || arg > 1.0_rt) { m_params.theta_stc = 0._rt; - else + } else { m_params.theta_stc = std::acos(arg); + } #else m_params.theta_stc = 0.; #endif diff --git a/Source/Parallelization/CMakeLists.txt b/Source/Parallelization/CMakeLists.txt index 48f0d3a49bd..d38f8d6bbf8 100644 --- a/Source/Parallelization/CMakeLists.txt +++ b/Source/Parallelization/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) GuardCellManager.cpp WarpXComm.cpp WarpXRegrid.cpp + WarpXSumGuardCells.cpp ) endforeach() diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index ee8fd044abb..38cef54921b 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -100,7 +100,7 @@ public: amrex::IntVect ng_UpdateAux = amrex::IntVect::TheZeroVector(); // Number of guard cells of all MultiFabs that must exchanged before moving window amrex::IntVect ng_MovingWindow = amrex::IntVect::TheZeroVector(); - // Number of guard cells of E and B that are exchanged immediatly after the main PSATD push + // Number of guard cells of E and B that are exchanged immediately after the main PSATD push amrex::IntVect ng_afterPushPSATD = amrex::IntVect::TheZeroVector(); // Number of guard cells for local deposition of J and rho diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index c1ecc5db34c..ae5ab839d10 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -172,7 +172,7 @@ guardCellManager::Init ( // After pushing particle int ng_alloc_F_int = (do_moving_window) ? 2 : 0; // CKC solver requires one additional guard cell - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC) { ng_alloc_F_int = std::max( ng_alloc_F_int, 1 ); } ng_alloc_F = IntVect(AMREX_D_DECL(ng_alloc_F_int, ng_alloc_F_int, ng_alloc_F_int)); // Used if warpx.do_divb_cleaning = 1 @@ -314,8 +314,9 @@ guardCellManager::Init ( // If NCI filter, add guard cells in the z direction IntVect ng_NCIFilter = IntVect::TheZeroVector(); - if (do_fdtd_nci_corr) + if (do_fdtd_nci_corr) { ng_NCIFilter[WARPX_ZINDEX] = NCIGodfreyFilter::m_stencil_width; + } // Note: communications of guard cells for bilinear filter are handled // separately. diff --git a/Source/Parallelization/Make.package b/Source/Parallelization/Make.package index 629cfafea62..7930118a1d2 100644 --- a/Source/Parallelization/Make.package +++ b/Source/Parallelization/Make.package @@ -1,5 +1,6 @@ CEXE_sources += WarpXComm.cpp CEXE_sources += WarpXRegrid.cpp CEXE_sources += GuardCellManager.cpp +CEXE_sources += WarpXSumGuardCells.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Parallelization diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 179afe86ae2..7d5b25b1560 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -551,7 +551,7 @@ void WarpX::FillBoundaryE (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryE(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryE(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -608,7 +608,7 @@ void WarpX::FillBoundaryB (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryB(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryB(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -665,7 +665,7 @@ void WarpX::FillBoundaryE_avg(int lev, IntVect ng) { FillBoundaryE_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryE_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryE_avg(lev, PatchType::coarse, ng); } } void @@ -719,7 +719,7 @@ void WarpX::FillBoundaryB_avg (int lev, IntVect ng) { FillBoundaryB_avg(lev, PatchType::fine, ng); - if (lev > 0) FillBoundaryB_avg(lev, PatchType::coarse, ng); + if (lev > 0) { FillBoundaryB_avg(lev, PatchType::coarse, ng); } } void @@ -770,7 +770,7 @@ void WarpX::FillBoundaryF (int lev, IntVect ng, std::optional nodal_sync) { FillBoundaryF(lev, PatchType::fine, ng, nodal_sync); - if (lev > 0) FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); + if (lev > 0) { FillBoundaryF(lev, PatchType::coarse, ng, nodal_sync); } } void @@ -780,7 +780,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_fp[lev]) pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); + if (F_fp[lev]) { pml[lev]->ExchangeF(patch_type, F_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -795,7 +795,7 @@ WarpX::FillBoundaryF (int lev, PatchType patch_type, IntVect ng, std::optionalok()) { - if (F_cp[lev]) pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); + if (F_cp[lev]) { pml[lev]->ExchangeF(patch_type, F_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryF(patch_type, nodal_sync); } @@ -824,7 +824,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_fp[lev]) pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); + if (G_fp[lev]) { pml[lev]->ExchangeG(patch_type, G_fp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -839,7 +839,7 @@ void WarpX::FillBoundaryG (int lev, PatchType patch_type, IntVect ng, std::optio { if (do_pml && pml[lev] && pml[lev]->ok()) { - if (G_cp[lev]) pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); + if (G_cp[lev]) { pml[lev]->ExchangeG(patch_type, G_cp[lev].get(), do_pml_in_domain); } pml[lev]->FillBoundaryG(patch_type, nodal_sync); } @@ -904,7 +904,7 @@ WarpX::SyncCurrent ( // SumBoundary even if there is only a single process, because a process // may have multiple boxes. Furthermore, even if there is only a single // box on a single process, SumBoundary should also be called if there - // are periodic boundaries. So we always call SumBounary even if it + // are periodic boundaries. So we always call SumBoundary even if it // might be a no-op in some cases, because the function does not perform // any communication if not necessary. // @@ -923,13 +923,13 @@ WarpX::SyncCurrent ( // have all been properly filtered and summed. For the current level, if // there are levels below this, we need to process this level's cp data // just like we have done for the finer level. The iteration continues - // until we reache level 0. There are however two additional + // until we reach level 0. There are however two additional // complications. // // The first complication is that simply calling ParallelAdd to add the // finer level's cp data to the current level's fp data does not // work. Suppose there are multiple boxes on the current level (or just - // a single box with periodic bounaries). A given (i,j,k) can be present + // a single box with periodic boundaries). A given (i,j,k) can be present // in more than one box for nodal data in AMReX. // At the time of calling ParallelAdd, the current // level's fp data have not been summed. Because of how ParallelAdd @@ -1048,7 +1048,7 @@ WarpX::SyncRho ( { WARPX_PROFILE("WarpX::SyncRho()"); - if (!charge_fp[0]) return; + if (!charge_fp[0]) { return; } const int ncomp = charge_fp[0]->nComp(); // See comments in WarpX::SyncCurrent for an explanation of the algorithm. @@ -1330,7 +1330,7 @@ void WarpX::ApplyFilterandSumBoundaryRho ( const int glev = (patch_type == PatchType::fine) ? lev : lev-1; const std::unique_ptr& rho = (patch_type == PatchType::fine) ? charge_fp[lev] : charge_cp[lev]; - if (rho == nullptr) return; + if (rho == nullptr) { return; } ApplyFilterandSumBoundaryRho(lev, glev, *rho, icomp, ncomp); } @@ -1373,7 +1373,7 @@ void WarpX::AddRhoFromFineLevelandSumBoundary ( const int icomp, const int ncomp) { - if (!charge_fp[lev]) return; + if (!charge_fp[lev]) { return; } ApplyFilterandSumBoundaryRho(charge_fp, charge_cp, lev, PatchType::fine, icomp, ncomp); @@ -1452,7 +1452,7 @@ void WarpX::NodalSyncJ ( const int lev, PatchType patch_type) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine) { @@ -1478,7 +1478,7 @@ void WarpX::NodalSyncRho ( const int icomp, const int ncomp) { - if (!override_sync_intervals.contains(istep[0])) return; + if (!override_sync_intervals.contains(istep[0])) { return; } if (patch_type == PatchType::fine && charge_fp[lev]) { diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index b09f4ad8344..cfd8bf22c2b 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" #include "Particles/WarpXParticleContainer.H" @@ -152,11 +153,11 @@ template void RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, const bool redistribute, const int lev) { - if (mf == nullptr) return; + if (mf == nullptr) { return; } const IntVect& ng = mf->nGrowVect(); std::unique_ptr pmf; WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } mf = std::move(pmf); } @@ -165,17 +166,17 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi { if (ba == boxArray(lev)) { - if (ParallelDescriptor::NProcs() == 1) return; + if (ParallelDescriptor::NProcs() == 1) { return; } // Fine patch for (int idim=0; idim < 3; ++idim) { RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); } - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); } RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); @@ -336,8 +337,9 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); - if (current_buffer_masks[lev] || gather_buffer_masks[lev]) + if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); + } } // Re-initialize the lattice element finder with the new ba and dm. diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 425ce320856..260ebb3871a 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -8,10 +8,6 @@ #ifndef WARPX_SUM_GUARD_CELLS_H_ #define WARPX_SUM_GUARD_CELLS_H_ -#include "Utils/WarpXAlgorithmSelection.H" - -#include - #include /** \brief Sum the values of `mf`, where the different boxes overlap @@ -26,20 +22,10 @@ * updates both the *valid* cells and *guard* cells. (This is because a * spectral solver requires the value of the sources over a large stencil.) */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = mf.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); -} + int icomp=0, int ncomp=1); /** \brief Sum the values of `src` where the different boxes overlap * (i.e. in the guard cells) and copy them into `dst` @@ -56,23 +42,10 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. */ -inline void +void WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, - const int icomp=0, const int ncomp=1) -{ - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) - n_updated_guards = dst.nGrowVect(); - else // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - - dst.setVal(0., icomp, ncomp, n_updated_guards); -// ablastr::utils::communication::ParallelAdd(dst, src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); - dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); -} + int icomp=0, int ncomp=1); #endif // WARPX_SUM_GUARD_CELLS_H_ diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp new file mode 100644 index 00000000000..20ac73cab50 --- /dev/null +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -0,0 +1,51 @@ +/* Copyright 2019 Maxence Thevenet, Remi Lehe, Weiqun Zhang + * + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXSumGuardCells.H" + +#include "Utils/WarpXAlgorithmSelection.H" + +#include "WarpX.H" + +#include + +void +WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = mf.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); +} + + +void +WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, + const amrex::Periodicity& period, + const amrex::IntVect& src_ngrow, + const int icomp, const int ncomp) +{ + amrex::IntVect n_updated_guards; + + // Update both valid cells and guard cells + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + n_updated_guards = dst.nGrowVect(); + } else { // Update only the valid cells + n_updated_guards = amrex::IntVect::TheZeroVector(); + } + + dst.setVal(0., icomp, ncomp, n_updated_guards); + dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); +} diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index be92e42d5fc..1de6b999e0b 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -7,9 +7,9 @@ #ifndef WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ #define WARPX_PARTICLES_COLLISION_BACKGROUNDMCCCOLLISION_H_ -#include "MCCProcess.H" #include "Particles/MultiParticleContainer.H" #include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" #include #include @@ -32,7 +32,7 @@ public: BackgroundMCCCollision ( BackgroundMCCCollision&& ) = delete; BackgroundMCCCollision& operator= ( BackgroundMCCCollision&& ) = delete; - amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes); + [[nodiscard]] amrex::ParticleReal get_nu_max (amrex::Vector const& mcc_processes) const; /** Perform the collisions * @@ -70,10 +70,10 @@ public: private: - amrex::Vector m_scattering_processes; - amrex::Vector m_ionization_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; - amrex::Gpu::DeviceVector m_ionization_processes_exe; + amrex::Vector m_scattering_processes; + amrex::Vector m_ionization_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + amrex::Gpu::DeviceVector m_ionization_processes_exe; bool init_flag = false; bool ionization_flag = false; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index ebf8cb47172..4cb16f6fa50 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -90,7 +90,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name amrex::Vector scattering_process_names; pp_collision_name.queryarr("scattering_processes", scattering_process_names); - // create a vector of MCCProcess objects from each scattering + // create a vector of ScatteringProcess objects from each scattering // process name for (const auto& scattering_process : scattering_process_names) { const std::string kw_cross_section = scattering_process + "_cross_section"; @@ -107,17 +107,17 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name pp_collision_name, kw_energy.c_str(), energy); } - MCCProcess process(scattering_process, cross_section_file, energy); + ScatteringProcess process(scattering_process, cross_section_file, energy); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != MCCProcessType::INVALID, - "Cannot add an unknown MCC process type"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); // if the scattering process is ionization get the secondary species // only one ionization process is supported, the vector // m_ionization_processes is only used to make it simple to calculate // the maximum collision frequency with the same function used for // particle conserving processes - if (process.type() == MCCProcessType::IONIZATION) { + if (process.type() == ScatteringProcessType::IONIZATION) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(!ionization_flag, "Background MCC only supports a single ionization process"); ionization_flag = true; @@ -133,8 +133,8 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name } #ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - amrex::Gpu::HostVector h_ionization_processes_exe; + amrex::Gpu::HostVector h_scattering_processes_exe; + amrex::Gpu::HostVector h_ionization_processes_exe; for (auto const& p : m_scattering_processes) { h_scattering_processes_exe.push_back(p.executor()); } @@ -162,7 +162,7 @@ BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name * ranges from 1e-4 to 5000 eV in 0.2 eV increments */ amrex::ParticleReal -BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) +BackgroundMCCCollision::get_nu_max(amrex::Vector const& mcc_processes) const { using namespace amrex::literals; amrex::ParticleReal nu, nu_max = 0.0; @@ -276,7 +276,7 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto const flvl = species1.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // firstly loop over particles box by box and do all particle conserving // scattering @@ -324,7 +324,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile auto T_a_func = m_background_temperature_func; // get collision parameters - auto scattering_processes = m_scattering_processes_exe.data(); + auto *scattering_processes = m_scattering_processes_exe.data(); auto const process_count = static_cast(m_scattering_processes_exe.size()); auto const total_collision_prob = m_total_collision_prob; @@ -352,7 +352,7 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile [=] AMREX_GPU_HOST_DEVICE (long ip, amrex::RandomEngine const& engine) { // determine if this particle should collide - if (amrex::Random(engine) > total_collision_prob) return; + if (amrex::Random(engine) > total_collision_prob) { return; } amrex::ParticleReal x, y, z; GetPosition.AsStored(ip, x, y, z); @@ -398,13 +398,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile nu_i += n_a * sigma_E * v_coll / nu_max; // check if this collision should be performed - if (col_select > nu_i) continue; + if (col_select > nu_i) { continue; } // charge exchange is implemented as a simple swap of the projectile // and target velocities which doesn't require any of the Lorentz // transformations below; note that if the projectile and target // have the same mass this is identical to back scattering - if (scattering_process.m_type == MCCProcessType::CHARGE_EXCHANGE) { + if (scattering_process.m_type == ScatteringProcessType::CHARGE_EXCHANGE) { ux[ip] = ua_x; uy[ip] = ua_y; uz[ip] = ua_z; @@ -433,13 +433,13 @@ void BackgroundMCCCollision::doBackgroundCollisionsWithinTile // transform to COM frame ParticleUtils::doLorentzTransform(vx, vy, vz, uCOM_x, uCOM_y, uCOM_z); - if ((scattering_process.m_type == MCCProcessType::ELASTIC) - || (scattering_process.m_type == MCCProcessType::EXCITATION)) { + if ((scattering_process.m_type == ScatteringProcessType::ELASTIC) + || (scattering_process.m_type == ScatteringProcessType::EXCITATION)) { ParticleUtils::RandomizeVelocity( vx, vy, vz, sqrt(vx*vx + vy*vy + vz*vz), engine ); } - else if (scattering_process.m_type == MCCProcessType::BACK) { + else if (scattering_process.m_type == ScatteringProcessType::BACK) { // elastic scattering with cos(chi) = -1 (i.e. 180 degrees) vx *= -1.0_prt; vy *= -1.0_prt; @@ -501,7 +501,7 @@ void BackgroundMCCCollision::doBackgroundIonization m_mass1, sqrt_kb_m, m_background_temperature_func, t ); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(species1, species2, elec_tile, ion_tile, elec_tile, np_elec, np_ion, Filter, CopyElec, CopyIon, Transform ); diff --git a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt index 15742af465f..ca8db0c53ec 100644 --- a/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt +++ b/Source/Particles/Collision/BackgroundMCC/CMakeLists.txt @@ -3,6 +3,5 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BackgroundMCCCollision.cpp - MCCProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H index cd9b74c72b8..73ed60d0ad7 100644 --- a/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H +++ b/Source/Particles/Collision/BackgroundMCC/ImpactIonization.H @@ -4,10 +4,10 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ -#define WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#ifndef WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ +#define WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ -#include "MCCProcess.H" +#include "Particles/Collision/ScatteringProcess.H" #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -39,7 +39,7 @@ public: * ensure accurate energy calculations (otherwise errors occur with single * or mixed precision builds of WarpX). * - * @param[in] mcc_process an MCCProcess object associated with the ionization + * @param[in] mcc_process an ScatteringProcess object associated with the ionization * @param[in] mass colliding particle's mass (could also assume electron) * @param[in] total_collision_prob total probability for a collision to occur * @param[in] nu_max maximum collision frequency @@ -48,7 +48,7 @@ public: * @param[in] t the current simulation time */ ImpactIonizationFilterFunc( - MCCProcess const& mcc_process, + ScatteringProcess const& mcc_process, double const mass, amrex::ParticleReal const total_collision_prob, amrex::ParticleReal const nu_max, @@ -77,7 +77,7 @@ public: using std::sqrt; // determine if this particle should collide - if (Random(engine) > m_total_collision_prob) return false; + if (Random(engine) > m_total_collision_prob) { return false; } // get references to the particle to get its position const auto& p = ptd.getSuperParticle(i); @@ -108,7 +108,7 @@ public: } private: - MCCProcess::Executor m_mcc_process; + ScatteringProcess::Executor m_mcc_process; double m_mass; amrex::ParticleReal m_total_collision_prob = 0; amrex::ParticleReal m_nu_max; @@ -227,4 +227,4 @@ private: amrex::ParserExecutor<4> m_T_a_func; amrex::Real m_t; }; -#endif // WARPX_PARTICLES_COLLISION_MCC_SCATTERING_H_ +#endif // WARPX_PARTICLES_COLLISION_IMPACT_IONIZATION_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/Make.package b/Source/Particles/Collision/BackgroundMCC/Make.package index 0b007e64e6d..242e85a8764 100644 --- a/Source/Particles/Collision/BackgroundMCC/Make.package +++ b/Source/Particles/Collision/BackgroundMCC/Make.package @@ -1,4 +1,3 @@ CEXE_sources += BackgroundMCCCollision.cpp -CEXE_sources += MCCProcess.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 83f74840478..8122d7225b4 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -104,7 +104,7 @@ BackgroundStopping::doCollisions (amrex::Real cur_time, amrex::Real dt, MultiPar auto const flvl = species.finestLevel(); for (int lev = 0; lev <= flvl; ++lev) { - auto cost = WarpX::getCosts(lev); + auto *cost = WarpX::getCosts(lev); // loop over particles box by box #ifdef _OPENMP diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 58fe2c9911a..78381f22258 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -72,7 +72,8 @@ class BinaryCollision final // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; @@ -87,8 +88,9 @@ public: BinaryCollision (std::string collision_name, MultiParticleContainer const * const mypc) : CollisionBase(collision_name) { - if(m_species_names.size() != 2) + if(m_species_names.size() != 2) { WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); + } m_isSameSpecies = (m_species_names[0] == m_species_names[1]); @@ -108,8 +110,10 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = default; - BinaryCollision& operator= ( BinaryCollision&& ) = default; + + + BinaryCollision ( BinaryCollision&& ) = delete; + BinaryCollision& operator= ( BinaryCollision&& ) = delete; /** Perform the collisions * @@ -156,8 +160,8 @@ public: auto copy_species1_data = device_copy_species1.data(); auto copy_species2_data = device_copy_species2.data(); #else - auto copy_species1_data = copy_species1.data(); - auto copy_species2_data = copy_species2.data(); + auto *copy_species1_data = copy_species1.data(); + auto *copy_species2_data = copy_species2.data(); #endif if (m_have_product_species){ species1.defineAllParticleTiles(); @@ -166,14 +170,14 @@ public: // Enable tiling amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) info.EnableTiling(species1.tile_size); + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(species1.tile_size); } // Loop over refinement levels for (int lev = 0; lev <= species1.finestLevel(); ++lev){ amrex::LayoutData* cost = WarpX::getCosts(lev); - // Loop over all grids/tiles at this level + // Loop over all grids/tiles at this level #ifdef AMREX_USE_OMP info.SetDynamic(true); #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) @@ -239,7 +243,7 @@ public: products_np.push_back(ptile_product.numParticles()); products_mass.push_back(product_species_vector[i]->getMass()); } - auto tile_products_data = tile_products.data(); + auto *tile_products_data = tile_products.data(); if ( m_isSameSpecies ) // species_1 == species_2 { @@ -260,9 +264,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -341,7 +342,7 @@ public: p_pair_offsets[i_cell] : 0; // Do not collide if there is only one particle in the cell - if ( cell_stop_1 - cell_start_1 <= 1 ) return; + if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( @@ -367,8 +368,10 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, tile_products_data, - particle_ptr_1, particle_ptr_1, m1, m1, + soa_1, soa_1, + product_species_vector, + tile_products_data, + m1, m1, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, @@ -400,9 +403,6 @@ public: const amrex::ParticleReal q1 = species_1.getCharge(); const amrex::ParticleReal m1 = species_1.getMass(); auto get_position_1 = GetParticlePosition(ptile_1, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_1 = ptile_1.GetArrayOfStructs()().data(); // - Species 2 const auto soa_2 = ptile_2.getParticleTileData(); index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); @@ -410,9 +410,6 @@ public: const amrex::ParticleReal q2 = species_2.getCharge(); const amrex::ParticleReal m2 = species_2.getMass(); auto get_position_2 = GetParticlePosition(ptile_2, getpos_offset); - // Needed to access the particle id - ParticleType * AMREX_RESTRICT - particle_ptr_2 = ptile_2.GetArrayOfStructs()().data(); amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); #if defined WARPX_DIM_1D_Z @@ -447,11 +444,12 @@ public: const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) { p_n_pairs_in_each_cell[i_cell] = 0; - else + } else { p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } } ); @@ -502,7 +500,7 @@ public: // Do not collide if one species is missing in the cell if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) return; + cell_stop_2 - cell_start_2 < 1 ) { return; } // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); @@ -528,8 +526,10 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, tile_products_data, - particle_ptr_1, particle_ptr_2, m1, m2, + soa_1, soa_2, + product_species_vector, + tile_products_data, + m1, m2, products_mass, p_mask, products_np, copy_species1, copy_species2, p_pair_indices_1, p_pair_indices_2, diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 0a7aae9835e..b8a390ddb93 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -12,6 +12,8 @@ #include "Particles/MultiParticleContainer.H" +#include + enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToProtonTritiumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, @@ -36,6 +38,92 @@ namespace BinaryCollisionUtils{ MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); + + /** + * \brief Return (relativistic) collision energy, collision speed and + * Lorentz factor for transforming between the lab and center-of-momentum + * frames. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void get_collision_parameters ( + const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal& E_kin_COM, amrex::ParticleReal& v_rel_COM, + amrex::ParticleReal& lab_to_COM_lorentz_factor ) + { + // General notations in this function: + // x_sq denotes the square of x + // x_star denotes the value of x in the center of mass frame + + using namespace amrex::literals; + using namespace amrex::Math; + + constexpr auto one_pr = amrex::ParticleReal(1.); + constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); + constexpr double c_sq = PhysConst::c * PhysConst::c; + constexpr double inv_csq = 1.0 / c_sq; + + const amrex::ParticleReal m1_sq = m1*m1; + const amrex::ParticleReal m2_sq = m2*m2; + + // Compute Lorentz factor gamma in the lab frame + const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); + const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + + // Compute momenta + const amrex::ParticleReal p1x = u1x * m1; + const amrex::ParticleReal p1y = u1y * m1; + const amrex::ParticleReal p1z = u1z * m1; + const amrex::ParticleReal p2x = u2x * m2; + const amrex::ParticleReal p2y = u2y * m2; + const amrex::ParticleReal p2z = u2z * m2; + // Square norm of the total (sum between the two particles) momenta in the lab frame + const auto p_total_sq = static_cast( + powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) + ); + + // Total energy in the lab frame + // Note the use of `double` for energy since this calculation is + // prone to error with single precision. + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + // Total energy squared in the center of mass frame, calculated using the Lorentz invariance + // of the four-momentum norm + const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; + + // Kinetic energy in the center of mass frame + const double E_star = std::sqrt(E_star_sq); + E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); + + // Square of the norm of the momentum of one of the particles in the center of mass frame + // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle + // The expression below is specifically written in a form that avoids returning + // small negative numbers due to machine precision errors, for low-energy particles + const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); + const auto p_star_sq = static_cast( + m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) + + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) + ); + + // Lorentz factors in the center of mass frame + const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); + const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + + // relative velocity in the center of mass frame + v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + + // Cross sections and relative velocity are computed in the center of mass frame. + // On the other hand, the particle densities (weight over volume) in the lab frame are used. + // To take this disrepancy into account, it is needed to multiply the + // collision probability by the ratio between the Lorentz factors in the + // COM frame and the Lorentz factors in the lab frame (see + // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor + // is calculated here. + lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + } } #endif // BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 8eddad7c496..430bac5e548 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -57,13 +57,11 @@ namespace BinaryCollisionUtils{ ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) ||(product_species1.AmIA() && product_species2.AmIA())){ return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or tritium and proton"); + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } } else if ((species1.AmIA() && species2.AmIA()) @@ -77,15 +75,15 @@ namespace BinaryCollisionUtils{ auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) + (product_species1.AmIA() && product_species2.AmIA()) || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type proton and helium4"); + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - else if ((species1.AmIA() && species2.AmIA()) + else if ((species1.AmIA() && species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) + (species1.AmIA() && species2.AmIA()) ) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -93,8 +91,8 @@ namespace BinaryCollisionUtils{ "ERROR: Proton-boron must contain exactly one product species"); auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type alpha"); + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); return NuclearFusionType::ProtonBoronToAlphas; } WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + @@ -120,16 +118,21 @@ namespace BinaryCollisionUtils{ CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { return CollisionType::ProtonBoronToAlphasFusion; + } WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); return CollisionType::Undefined; } diff --git a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt index 55bbf21e310..60933c83a21 100644 --- a/Source/Particles/Collision/BinaryCollision/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/CMakeLists.txt @@ -6,3 +6,5 @@ foreach(D IN LISTS WarpX_DIMS) ParticleCreationFunc.cpp ) endforeach() + +add_subdirectory(DSMC) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index b54404d46ba..d7403eeacf9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -104,7 +104,7 @@ void ElasticCollisionPerez ( n12 = n12 / dV; } // Intra-species: Apply correction in collision rate - if (isSameSpecies) n12 *= T_PR(2.0); + if (isSameSpecies) { n12 *= T_PR(2.0); } // compute Debye length lmdD T_PR lmdD; diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index feb7acf81d3..cfdc36d3c50 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -23,10 +23,13 @@ * \brief This functor performs pairwise Coulomb collision on a single cell by calling the function * ElasticCollisionPerez. It also reads and contains the Coulomb logarithm. */ -class PairWiseCoulombCollisionFunc{ +class PairWiseCoulombCollisionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt new file mode 100644 index 00000000000..3575d53ad29 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -0,0 +1,7 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + DSMC.cpp + ) +endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H new file mode 100644 index 00000000000..c714cfdb133 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -0,0 +1,222 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies), Neil Zaim + * + * License: BSD-3-Clause-LBNL + */ +#ifndef COLLISION_FILTER_FUNC_H_ +#define COLLISION_FILTER_FUNC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/ScatteringProcess.H" + +#include + +/** + * \brief This function determines whether a collision occurs for a given + * pair of particles. + * + * @param[in] u1x,u1y,u1z momenta of the first colliding particle + * @param[in] u2x,u2y,u2z momenta of the second colliding particle + * @param[in] m1,m2 masses + * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] pair_index is the index of the colliding pair + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_reaction_weight stores the weight of the product particles + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, + const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, + const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, + const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, + amrex::ParticleReal w1, amrex::ParticleReal w2, + const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, + index_type* AMREX_RESTRICT p_mask, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + const int& multiplier_ratio, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + const amrex::RandomEngine& engine) +{ + using namespace amrex::literals; + + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); + + // convert E_coll to eV + E_coll /= PhysConst::q_e; + + amrex::ParticleReal sigma_tot = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma_tot += scattering_process.getCrossSection(E_coll); + } + + // calculate total collision probability + amrex::ParticleReal exponent = ( + lab_to_COM_factor * multiplier_ratio * w_max + * sigma_tot * v_coll * dt / dV + ); + + // Compute actual collision probability that is always between zero and one + // In principle this is obtained by computing 1 - exp(-probability_estimate) + // However, the computation of this quantity can fail numerically when probability_estimate is + // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-exponent); + + // Now we determine if a collision should occur + if (amrex::Random(engine) < probability) + { + const amrex::ParticleReal random_number = amrex::Random(engine); + amrex::ParticleReal sigma = 0._prt; + for (int ii = 0; ii < process_count; ii++) { + auto const& scattering_process = *(scattering_processes + ii); + sigma += scattering_process.getCrossSection(E_coll); + if (random_number <= sigma / sigma_tot) + { + p_mask[pair_index] = int(scattering_process.m_type); + p_pair_reaction_weight[pair_index] = w_min; + break; + } + } + } + else + { + p_mask[pair_index] = false; + } +} + +/** + * \brief Function that determines if a collision occurs and if so, what + * type. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] ptd_1,ptd_2 contain the particle data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to a non-zero integer if a + * collision occurs. The integer encodes the scattering process. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] process_count number of scattering processes to consider + * @param[in] scattering processes an array of scattering processes included for consideration + * @param[in] engine the random engine. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void CollisionFilter ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + PData ptd_1, PData ptd_2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + int const process_count, + ScatteringProcess::Executor* scattering_processes, + amrex::RandomEngine const& engine) +{ + + amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const int NI1 = I1e - I1s; + const int NI2 = I2e - I2s; + const int max_N = amrex::max(NI1,NI2); + + int i1 = I1s; + int i2 = I2s; + int pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + const int c1 = amrex::max(NI2/NI1,1); + const int c2 = amrex::max(NI1/NI2,1); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; +#endif + + for (int k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, pair_index, p_mask, p_pair_reaction_weight, + max_N, process_count, scattering_processes, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } + ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } + ++pair_index; + } +} + +#endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H new file mode 100644 index 00000000000..ab01eba2c81 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -0,0 +1,86 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef DSMC_H_ +#define DSMC_H_ + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMC final + : public CollisionBase +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using index_type = ParticleBins::index_type; + +public: + + /** + * \brief Constructor of the DSMC class + * + * @param[in] collision_name the name of the collision + */ + DSMC (std::string collision_name); + + /** Perform the collisions + * + * @param cur_time Current time + * @param dt Time step size + * @param mypc Container of species involved + * + */ + void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; + + /** Perform all binary collisions within a tile + * + * \param[in] lev the mesh-refinement level + * \param[in] mfi iterator for multifab + * \param species_1 first species container + * \param species_2 second species container + * \param copy_species1 SmartCopy for species_1 + * \param copy_species2 SmartCopy for species_2 + * + */ + void doCollisionsWithinTile ( + amrex::Real dt, int lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2 ); + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; +}; + +#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp new file mode 100644 index 00000000000..70a14712a0c --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -0,0 +1,300 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "CollisionFilterFunc.H" +#include "DSMC.H" +#include "SplitAndScatterFunc.H" + + +DSMC::DSMC (const std::string collision_name) + : CollisionBase(collision_name) +{ + using namespace amrex::literals; + amrex::ParmParse pp_collision_name(collision_name); + +#if defined WARPX_DIM_RZ + amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); +#endif + + if(m_species_names.size() != 2) + { + amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); + } + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif +} + +void +DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) +{ + WARPX_PROFILE("DSMC::doCollisions()"); + + auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); + + // SmartCopy objects are created that will facilitate the particle splitting + // operation involved in DSMC collisions between particles with arbitrary + // weights. + SmartCopyFactory copy_factory_species1(species1, species1); + SmartCopyFactory copy_factory_species2(species2, species2); + auto copy_species1 = copy_factory_species1.getSmartCopy(); + auto copy_species2 = copy_factory_species2.getSmartCopy(); + + species1.defineAllParticleTiles(); + species2.defineAllParticleTiles(); + + // Enable tiling + amrex::MFItInfo info; + if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } + + // Loop over refinement levels + for (int lev = 0; lev <= species1.finestLevel(); ++lev){ + + amrex::LayoutData* cost = WarpX::getCosts(lev); + + // Loop over all grids/tiles at this level +#ifdef AMREX_USE_OMP + info.SetDynamic(true); +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + } + auto wt = static_cast(amrex::second()); + + doCollisionsWithinTile(dt, lev, mfi, species1, species2, + copy_species1, copy_species2); + + if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) + { + amrex::Gpu::synchronize(); + wt = static_cast(amrex::second()) - wt; + amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); + } + } + + // Call redistribute to remove particles with negative ids + species1.Redistribute(lev, lev, 0, true, true); + species2.Redistribute(lev, lev, 0, true, true); + } +} + +void +DSMC::doCollisionsWithinTile( + amrex::Real dt, int const lev, amrex::MFIter const& mfi, + WarpXParticleContainer& species_1, + WarpXParticleContainer& species_2, + SmartCopy& copy_species1, + SmartCopy& copy_species2) +{ + using namespace ParticleUtils; + using namespace amrex::literals; + + // get collision processes + auto *scattering_processes = m_scattering_processes_exe.data(); + int const process_count = static_cast(m_scattering_processes_exe.size()); + + // Extract particles in the tile that `mfi` points to + ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); + ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); + + // Find the particles that are in each cell of this tile + ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); + ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); + + // Extract low-level data + int const n_cells = static_cast(bins_1.numBins()); + + // - Species 1 + index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); + amrex::ParticleReal m1 = species_1.getMass(); + const auto ptd_1 = ptile_1.getParticleTileData(); + + // - Species 2 + index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); + index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); + amrex::ParticleReal m2 = species_2.getMass(); + const auto ptd_2 = ptile_2.getParticleTileData(); + + amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); +#if defined WARPX_DIM_1D_Z + auto dV = geom.CellSize(0); +#elif defined WARPX_DIM_XZ + auto dV = geom.CellSize(0) * geom.CellSize(1); +#elif defined WARPX_DIM_RZ + amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box + const auto lo = lbound(cbx); + const auto hi = ubound(cbx); + int nz = hi.y-lo.y+1; + auto dr = geom.CellSize(0); + auto dz = geom.CellSize(1); +#elif defined(WARPX_DIM_3D) + auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); +#endif + + // In the following we set up the "mask" used for creating new particles + // (from splitting events). There is a mask index for every collision + // pair. Below we find the size of the mask based on the greater of the + // number of each species' particle in each cell. + amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); + index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); + + // Compute how many pairs in each cell and store in n_pairs_in_each_cell array + // For different species, the number of pairs in a cell is the number of particles of + // the species that has the most particles in that cell + amrex::ParallelFor( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + // Particular case: no pair if a species has no particle in that cell + if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) + { + p_n_pairs_in_each_cell[i_cell] = 0; + } + else + { + p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); + } + } + ); + + // Start indices of the pairs in a cell. Will be used for particle creation + amrex::Gpu::DeviceVector pair_offsets(n_cells); + const index_type n_total_pairs = (n_cells == 0) ? 0: + amrex::Scan::ExclusiveSum(n_cells, + p_n_pairs_in_each_cell, pair_offsets.data()); + index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + + // Now we create the mask. In the DSMC scheme the mask will serve two + // purposes, 1) record whether a given pair should collide and 2) record + // the scattering process that should occur. + amrex::Gpu::DeviceVector mask(n_total_pairs); + index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); + + // Will be filled with the index of the first particle of a given pair + amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); + // Will be filled with the index of the second particle of a given pair + amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); + index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); + + // How much weight should be given to the produced particle - based on the + // weight of the collision partner which is not split + amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = + pair_reaction_weight.dataPtr(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + // Same but for the pairs + index_type const cell_start_pair = p_pair_offsets[i_cell]; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + + // Do not collide if one species is missing in the cell + if ( cell_stop_1 - cell_start_1 < 1 || + cell_stop_2 - cell_start_2 < 1 ) { return; } + + // shuffle + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); + ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); +#if defined WARPX_DIM_RZ + int ri = (i_cell - i_cell%nz) / nz; + auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; +#endif + // Call the function in order to perform collisions + // If there are product species, p_mask, p_pair_indices_1/2, and + // p_pair_reaction_weight are filled here + CollisionFilter( + cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, + indices_1, indices_2, + ptd_1, ptd_2, + m1, m2, dt, dV, + cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight, + process_count, scattering_processes, engine ); + } + ); + + const auto num_p_tile1 = ptile_1.numParticles(); + const auto num_p_tile2 = ptile_2.numParticles(); + + // Create the new product particles and define their initial values + // num_added: how many particles of each product species have been created + const int num_added = splitScatteringParticles( + n_total_pairs, + ptile_1, ptile_2, + p_mask, + copy_species1, copy_species2, + m1, m2, + p_pair_indices_1, p_pair_indices_2, + p_pair_reaction_weight); + + if (num_added > 0) { + setNewParticleIDs(ptile_1, num_p_tile1, num_added); + setNewParticleIDs(ptile_2, num_p_tile2, num_added); + } +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package new file mode 100644 index 00000000000..b4cfab89c64 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -0,0 +1,3 @@ +CEXE_sources += DSMC.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H new file mode 100644 index 00000000000..f684b60da78 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -0,0 +1,181 @@ +/* Copyright 2023 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SPLIT_AND_SCATTER_FUNC_H_ +#define SPLIT_AND_SCATTER_FUNC_H_ + +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/NamedComponentParticleContainer.H" + +#include + +/** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ +template ::value, int> foo = 0> +int splitScatteringParticles ( + const index_type& n_total_pairs, + Tile& ptile1, Tile& ptile2, + const Index* AMREX_RESTRICT mask, + CopyFunc&& copy1, CopyFunc&& copy2, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +{ + using namespace amrex; + + if (n_total_pairs == 0) { + return 0; + } + + Gpu::DeviceVector offsets(n_total_pairs); + Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + const auto ptile1_index = ptile1.numParticles(); + const auto ptile2_index = ptile2.numParticles(); + ptile1.resize(ptile1_index + num_added); + ptile2.resize(ptile2_index + num_added); + + const auto ptile1_data = ptile1.getParticleTileData(); + const auto ptile2_data = ptile2.getParticleTileData(); + + ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept + { + if (mask[i]) + { + // First, make copies of the colliding particles + copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); + copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); + + // Now we adjust the properties of the original and child particles, + // starting with the parent particles + auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; + auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; + uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; + + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + }; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights. + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to -1. + Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); + if (w1 <= 0._prt) { + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + } + + Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); + if (w2 <= 0._prt) { + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + } + + // Set the child particle properties appropriately + ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; + ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; + + auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; + auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; + auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; + auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; + auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; + auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (m1 == m2) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + Abort("Uneven mass charge-exchange not implemented yet."); + } + } + else { + Abort("Unknown scattering process."); + } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; + } + }); + + Gpu::synchronize(); + return static_cast(num_added); +} +#endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Make.package b/Source/Particles/Collision/BinaryCollision/Make.package index 92e2f0b5dd1..393e6270345 100644 --- a/Source/Particles/Collision/BinaryCollision/Make.package +++ b/Source/Particles/Collision/BinaryCollision/Make.package @@ -1,4 +1,6 @@ CEXE_sources += BinaryCollisionUtils.cpp CEXE_sources += ParticleCreationFunc.cpp +include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC/Make.package + VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 397536b67bf..b2a2112ca68 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -33,10 +33,13 @@ * creation functor. * This functor also reads and contains the fusion multiplier. */ -class NuclearFusionFunc{ +class NuclearFusionFunc +{ // Define shortcuts for frequently-used type names using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; @@ -154,12 +157,13 @@ public: // other species and we need to decrease their weight accordingly. // c1 corresponds to the minimum number of times a particle of species 1 will be paired // with a particle of species 2. Same for c2. - const index_type c1 = amrex::max(NI2/NI1,1u); - const index_type c2 = amrex::max(NI1/NI2,1u); + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); // multiplier ratio to take into account unsampled pairs const auto multiplier_ratio = static_cast( - (m_isSameSpecies)?(2u*max_N - 1):(max_N)); + m_isSameSpecies ? 2*max_N - 1 : max_N); #if (defined WARPX_DIM_RZ) amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index 4a5d518b750..ec9a1989189 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -42,8 +42,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute Gamow factor, in MeV constexpr auto one_pr = 1._prt; constexpr auto Z_boron = 5._prt; - constexpr amrex::ParticleReal m_boron = 10.7319_prt * PhysConst::m_p; - constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/PhysConst::m_p); + constexpr amrex::ParticleReal m_boron = 11.00930536_prt * PhysConst::m_u; + constexpr amrex::ParticleReal m_hydrogen = 1.00782503223 * PhysConst::m_u; + constexpr amrex::ParticleReal m_reduced = m_boron / (one_pr + m_boron/m_hydrogen); constexpr amrex::ParticleReal gamow_factor = m_reduced / 2._prt * (PhysConst::q_e*PhysConst::q_e * Z_boron / (2._prt*PhysConst::ep0*PhysConst::hbar)) * diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0e92bda5beb..0b51d6b4b61 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -22,10 +22,12 @@ namespace { // Define shortcuts for frequently-used type names - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; /** * \brief This function initializes the momentum of the alpha particles produced from @@ -81,7 +83,7 @@ namespace { // which can cause compilation to fail or generate a warning, so we're explicitly setting // them as double. Note that nuclear fusion module does not currently work with single // precision anyways. - constexpr double m_alpha = PhysConst::m_u * 4.002602_prt; + constexpr double m_alpha = PhysConst::m_u * 4.00260325413_prt; constexpr double m_beryllium = PhysConst::m_p * 7.94748_prt; constexpr double mBe_sq = m_beryllium*m_beryllium; constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 8b0e71763bc..07e0174438b 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -15,7 +15,6 @@ #include "Utils/WarpXConst.H" #include -#include #include #include @@ -67,89 +66,33 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part const NuclearFusionType& fusion_type, const amrex::RandomEngine& engine) { - // General notations in this function: - // x_sq denotes the square of x - // x_star denotes the value of x in the center of mass frame + amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; + + BinaryCollisionUtils::get_collision_parameters( + u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, + E_coll, v_coll, lab_to_COM_factor); using namespace amrex::literals; - using namespace amrex::Math; const amrex::ParticleReal w_min = amrex::min(w1, w2); const amrex::ParticleReal w_max = amrex::max(w1, w2); - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); - constexpr amrex::ParticleReal c_sq = PhysConst::c * PhysConst::c; - constexpr amrex::ParticleReal inv_csq = one_pr / ( c_sq ); - - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; - - // Compute Lorentz factor gamma in the lab frame - const amrex::ParticleReal g1 = std::sqrt( one_pr + (u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const amrex::ParticleReal g2 = std::sqrt( one_pr + (u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); - - // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; - // Square norm of the total (sum between the two particles) momenta in the lab frame - const amrex::ParticleReal p_total_sq = powi<2>(p1x + p2x) + - powi<2>(p1y+p2y) + - powi<2>(p1z+p2z); - - // Total energy in the lab frame - const amrex::ParticleReal E_lab = (m1 * g1 + m2 * g2) * c_sq; - // Total energy squared in the center of mass frame, calculated using the Lorentz invariance - // of the four-momentum norm - const amrex::ParticleReal E_star_sq = E_lab*E_lab - c_sq*p_total_sq; - - // Kinetic energy in the center of mass frame - const amrex::ParticleReal E_star = std::sqrt(E_star_sq); - const amrex::ParticleReal E_kin_star = E_star - (m1 + m2)*c_sq; - // Compute fusion cross section as a function of kinetic energy in the center of mass frame auto fusion_cross_section = amrex::ParticleReal(0.); if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - fusion_cross_section = ProtonBoronFusionCrossSection(E_kin_star); + fusion_cross_section = ProtonBoronFusionCrossSection(E_coll); } else if ((fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) || (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium)) { - fusion_cross_section = BoschHaleFusionCrossSection(E_kin_star, fusion_type, m1, m2); + fusion_cross_section = BoschHaleFusionCrossSection(E_coll, fusion_type, m1, m2); } - // Square of the norm of the momentum of one of the particles in the center of mass frame - // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle - // The expression below is specifically written in a form that avoids returning - // small negative numbers due to machine precision errors, for low-energy particles - const amrex::ParticleReal E_ratio = E_star/((m1 + m2)*c_sq); - const amrex::ParticleReal p_star_sq = m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio ); - - // Lorentz factors in the center of mass frame - const amrex::ParticleReal g1_star = std::sqrt(one_pr + p_star_sq / (m1_sq*c_sq)); - const amrex::ParticleReal g2_star = std::sqrt(one_pr + p_star_sq / (m2_sq*c_sq)); - - // relative velocity in the center of mass frame - const amrex::ParticleReal v_rel = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + - one_pr/(m2*g2_star)); - - // Fusion cross section and relative velocity are computed in the center of mass frame. - // On the other hand, the particle densities (weight over volume) in the lab frame are used. To - // take into account this discrepancy, we need to multiply the fusion probability by the ratio - // between the Lorentz factors in the COM frame and the Lorentz factors in the lab frame - // (see Perez et al., Phys.Plasmas.19.083104 (2012)) - const amrex::ParticleReal lab_to_COM_factor = g1_star*g2_star/(g1*g2); - // First estimate of probability to have fusion reaction amrex::ParticleReal probability_estimate = multiplier_ratio * fusion_multiplier * - lab_to_COM_factor * w_max * fusion_cross_section * v_rel * dt / dV; + lab_to_COM_factor * w_max * fusion_cross_section * v_coll * dt / dV; // Effective fusion multiplier amrex::ParticleReal fusion_multiplier_eff = fusion_multiplier; @@ -162,7 +105,7 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // We aim for a fusion probability of probability_target_value but take into account // the constraint that the fusion_multiplier cannot be smaller than one fusion_multiplier_eff = amrex::max(fusion_multiplier * - probability_target_value / probability_estimate , one_pr); + probability_target_value / probability_estimate , 1._prt); probability_estimate *= fusion_multiplier_eff/fusion_multiplier; } @@ -170,18 +113,8 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part // In principle this is obtained by computing 1 - exp(-probability_estimate) // However, the computation of this quantity can fail numerically when probability_estimate is // too small (e.g. exp(-probability_estimate) returns 1 and the computation returns 0). - // In this case, we simply use "probability_estimate" instead of 1 - exp(-probability_estimate) - // The threshold exp_threshold at which we switch between the two formulas is determined by the - // fact that computing the exponential is only useful if it can resolve the x^2/2 term of its - // Taylor expansion, i.e. the square of probability_estimate should be greater than the - // machine epsilon. -#ifdef AMREX_SINGLE_PRECISION_PARTICLES - constexpr auto exp_threshold = amrex::ParticleReal(1.e-3); -#else - constexpr auto exp_threshold = amrex::ParticleReal(5.e-8); -#endif - const amrex::ParticleReal probability = (probability_estimate < exp_threshold) ? - probability_estimate: one_pr - std::exp(-probability_estimate); + // std::expm1 is used since it maintains correctness for small exponent. + const amrex::ParticleReal probability = -std::expm1(-probability_estimate); // Get a random number const amrex::ParticleReal random_number = amrex::Random(engine); @@ -198,6 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } - - #endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index be3f5b2d957..52e9db8aa94 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -24,7 +24,9 @@ namespace { // Define shortcuts for frequently-used type names using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleBins = amrex::DenseBins; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; using index_type = ParticleBins::index_type; /** diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index ca994ed1f56..7a2853e3db5 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -30,13 +30,15 @@ * \brief This functor creates particles produced from a binary collision and sets their initial * properties (position, momentum, weight). */ -class ParticleCreationFunc{ +class ParticleCreationFunc +{ // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -69,12 +71,6 @@ public: * @param[in, out] soa_1 struct of array data of the first colliding particle species * @param[in, out] soa_2 struct of array data of the second colliding particle species * @param[out] tile_products array containing tile data of the product particles. - * @param[out] particle_ptr_1 pointer to data of the first colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. - * @param[out] particle_ptr_2 pointer to data of the second colliding particle species. Is - * needed to set the id of a particle to -1 in order to delete it when its weight - * reaches 0. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species * @param[in] products_mass array storing the mass of product particles @@ -100,8 +96,8 @@ public: amrex::Vector operator() ( const index_type& n_total_pairs, const SoaData_type& soa_1, const SoaData_type& soa_2, + const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, - ParticleType* particle_ptr_1, ParticleType* particle_ptr_2, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, const amrex::Vector& products_mass, const index_type* AMREX_RESTRICT p_mask, @@ -115,7 +111,7 @@ public: { using namespace amrex::literals; - if (n_total_pairs == 0) return amrex::Vector(m_num_product_species, 0); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } // Compute offset array and allocate memory for the produced species amrex::Gpu::DeviceVector offsets(n_total_pairs); @@ -136,6 +132,8 @@ public: amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; // Create necessary GPU vectors, that will be used in the kernel below amrex::Vector soa_products; @@ -204,16 +202,31 @@ public: amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], -p_pair_reaction_weight[i]); + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + }; + // If the colliding particle weight decreases to zero, remove particle by // setting its id to -1 - constexpr amrex::Long minus_one_long = -1; if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_1[p_pair_indices_1[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + } if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) { - particle_ptr_2[p_pair_indices_2[i]].atomicSetID(minus_one_long); + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); } // Initialize the product particles' momentum, using a function depending on the @@ -251,6 +264,27 @@ public: } }); + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + int start_index = int(products_np[i]); + int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); + } + amrex::Gpu::synchronize(); return num_added_vec; @@ -272,12 +306,14 @@ private: * \brief This class does nothing and is used as second template parameter for binary collisions * that do not create particles. */ -class NoParticleCreationFunc{ - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = amrex::DenseBins; - using index_type = ParticleBins::index_type; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; +class NoParticleCreationFunc +{ + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: NoParticleCreationFunc () = default; @@ -289,8 +325,8 @@ public: amrex::Vector operator() ( const index_type& /*n_total_pairs*/, const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, - ParticleType* /*particle_ptr_1*/, ParticleType* /*particle_ptr_2*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, const amrex::Vector& /*products_mass*/, const index_type* /*p_mask*/, const amrex::Vector& /*products_np*/, diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 1ef5ecad229..33629cd530f 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -19,45 +19,45 @@ ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); +{ + const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) - { - // Proton-Boron fusion only produces alpha particles - m_num_product_species = 1; - // Proton-Boron fusion produces 3 alpha particles per fusion reaction - m_num_products_host.push_back(3); + if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) + { + // Proton-Boron fusion only produces alpha particles + m_num_product_species = 1; + // Proton-Boron fusion produces 3 alpha particles per fusion reaction + m_num_products_host.push_back(3); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly - m_num_products_device.push_back(3); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(3); #endif - } - else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) - || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) - { - m_num_product_species = 2; - m_num_products_host.push_back(1); - m_num_products_host.push_back(1); + } + else if ((m_collision_type == CollisionType::DeuteriumTritiumToNeutronHeliumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToProtonTritiumFusion) + || (m_collision_type == CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion)) + { + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); #ifndef AMREX_USE_GPU - // On CPU, the device vector can be filled immediatly - m_num_products_device.push_back(1); - m_num_products_device.push_back(1); + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); #endif - } - else - { - WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); - } + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in ParticleCreationFunc"); + } #ifdef AMREX_USE_GPU - m_num_products_device.resize(m_num_product_species); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), - m_num_products_host.end(), - m_num_products_device.begin()); - amrex::Gpu::streamSynchronize(); + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); #endif - } +} diff --git a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H index 42259512b0d..3b8f72f4b84 100644 --- a/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H +++ b/Source/Particles/Collision/BinaryCollision/ShuffleFisherYates.H @@ -12,7 +12,7 @@ /* \brief Shuffle array according to Fisher-Yates algorithm. * Only shuffle the part between is <= i < ie, n = ie-is. * T_index shall be - * amrex::DenseBins::index_type + * amrex::DenseBins::index_type */ template diff --git a/Source/Particles/Collision/CMakeLists.txt b/Source/Particles/Collision/CMakeLists.txt index f3183032179..62eb3b3ad9c 100644 --- a/Source/Particles/Collision/CMakeLists.txt +++ b/Source/Particles/Collision/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE CollisionHandler.cpp CollisionBase.cpp + ScatteringProcess.cpp ) endforeach() diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index 272cc056b1a..db79eaf1a01 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -29,7 +29,7 @@ public: virtual ~CollisionBase() = default; - int get_ndt() {return m_ndt;} + [[nodiscard]] int get_ndt() const {return m_ndt;} protected: diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 213e5edd577..5abee34c86a 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -10,6 +10,7 @@ #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -52,6 +53,9 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) else if (type == "background_stopping") { allcollisions[i] = std::make_unique(collision_names[i]); } + else if (type == "dsmc") { + allcollisions[i] = std::make_unique(collision_names[i]); + } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( diff --git a/Source/Particles/Collision/Make.package b/Source/Particles/Collision/Make.package index e9aad3691a3..9ec153879f4 100644 --- a/Source/Particles/Collision/Make.package +++ b/Source/Particles/Collision/Make.package @@ -1,5 +1,6 @@ CEXE_sources += CollisionHandler.cpp CEXE_sources += CollisionBase.cpp +CEXE_sources += ScatteringProcess.cpp include $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/Make.package include $(WARPX_HOME)/Source/Particles/Collision/BackgroundMCC/Make.package diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H b/Source/Particles/Collision/ScatteringProcess.H similarity index 81% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.H rename to Source/Particles/Collision/ScatteringProcess.H index 9b844959e52..d05ea32e2fe 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -1,18 +1,20 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#ifndef WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ -#define WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#ifndef WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ +#define WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ #include #include #include #include -enum class MCCProcessType { +enum class ScatteringProcessType { INVALID, ELASTIC, BACK, @@ -21,29 +23,29 @@ enum class MCCProcessType { IONIZATION, }; -class MCCProcess +class ScatteringProcess { public: - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, amrex::ParticleReal energy ); template - MCCProcess ( + ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, amrex::ParticleReal energy ); - ~MCCProcess() = default; + ~ScatteringProcess() = default; - MCCProcess (MCCProcess const&) = delete; - MCCProcess& operator= (MCCProcess const&) = delete; - MCCProcess (MCCProcess &&) = default; - MCCProcess& operator= (MCCProcess &&) = default; + ScatteringProcess (ScatteringProcess const&) = delete; + ScatteringProcess& operator= (ScatteringProcess const&) = delete; + ScatteringProcess (ScatteringProcess &&) = default; + ScatteringProcess& operator= (ScatteringProcess &&) = default; /** Read the given cross-section data file to memory. * @@ -99,7 +101,7 @@ public: amrex::ParticleReal* m_sigmas_data = nullptr; amrex::ParticleReal m_energy_lo, m_energy_hi, m_sigma_lo, m_sigma_hi, m_dE; amrex::ParticleReal m_energy_penalty; - MCCProcessType m_type; + ScatteringProcessType m_type; }; [[nodiscard]] @@ -121,12 +123,12 @@ public: [[nodiscard]] amrex::ParticleReal getMaxEnergyInput () const { return m_exe_h.m_energy_hi; } [[nodiscard]] amrex::ParticleReal getEnergyInputStep () const { return m_exe_h.m_dE; } - [[nodiscard]] MCCProcessType type () const { return m_exe_h.m_type; } + [[nodiscard]] ScatteringProcessType type () const { return m_exe_h.m_type; } private: static - MCCProcessType parseProcessType(const std::string& process); + ScatteringProcessType parseProcessType(const std::string& process); void init (const std::string& scattering_process, amrex::ParticleReal energy); @@ -142,4 +144,4 @@ private: int m_grid_size; }; -#endif // WARPX_PARTICLES_COLLISION_MCCPROCESS_H_ +#endif // WARPX_PARTICLES_COLLISION_SCATTERING_PROCESS_H_ diff --git a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp similarity index 79% rename from Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp rename to Source/Particles/Collision/ScatteringProcess.cpp index bf2138bc78b..c15d11eea07 100644 --- a/Source/Particles/Collision/BackgroundMCC/MCCProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -1,15 +1,17 @@ -/* Copyright 2021 Modern Electron +/* Copyright 2021-2023 The WarpX Community * * This file is part of WarpX. * + * Authors: Modern Electron, Roelof Groenewald (TAE Technologies) + * * License: BSD-3-Clause-LBNL */ -#include "MCCProcess.H" +#include "ScatteringProcess.H" #include "Utils/TextMsg.H" #include "WarpX.H" -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const std::string& cross_section_file, const amrex::ParticleReal energy ) @@ -21,7 +23,7 @@ MCCProcess::MCCProcess ( } template -MCCProcess::MCCProcess ( +ScatteringProcess::ScatteringProcess ( const std::string& scattering_process, const InputVector&& energies, const InputVector&& sigmas, @@ -34,7 +36,7 @@ MCCProcess::MCCProcess ( } void -MCCProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) +ScatteringProcess::init (const std::string& scattering_process, const amrex::ParticleReal energy) { using namespace amrex::literals; m_exe_h.m_sigmas_data = m_sigmas_h.data(); @@ -72,44 +74,44 @@ MCCProcess::init (const std::string& scattering_process, const amrex::ParticleRe #endif } -MCCProcessType -MCCProcess::parseProcessType(const std::string& scattering_process) +ScatteringProcessType +ScatteringProcess::parseProcessType(const std::string& scattering_process) { if (scattering_process == "elastic") { - return MCCProcessType::ELASTIC; + return ScatteringProcessType::ELASTIC; } else if (scattering_process == "back") { - return MCCProcessType::BACK; + return ScatteringProcessType::BACK; } else if (scattering_process == "charge_exchange") { - return MCCProcessType::CHARGE_EXCHANGE; + return ScatteringProcessType::CHARGE_EXCHANGE; } else if (scattering_process == "ionization") { - return MCCProcessType::IONIZATION; + return ScatteringProcessType::IONIZATION; } else if (scattering_process.find("excitation") != std::string::npos) { - return MCCProcessType::EXCITATION; + return ScatteringProcessType::EXCITATION; } else { - return MCCProcessType::INVALID; + return ScatteringProcessType::INVALID; } } void -MCCProcess::readCrossSectionFile ( +ScatteringProcess::readCrossSectionFile ( const std::string cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { std::ifstream infile(cross_section_file); - if(!infile.is_open()) WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); + if(!infile.is_open()) { WARPX_ABORT_WITH_MESSAGE("Failed to open cross-section data file"); } amrex::ParticleReal energy, sigma; while (infile >> energy >> sigma) { energies.push_back(energy); sigmas.push_back(sigma); } - if (infile.bad()) WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); + if (infile.bad()) { WARPX_ABORT_WITH_MESSAGE("Failed to read cross-section data from file."); } infile.close(); } void -MCCProcess::sanityCheckEnergyGrid ( +ScatteringProcess::sanityCheckEnergyGrid ( const amrex::Vector& energies, amrex::ParticleReal dE ) diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index df7097f7600..d0822789015 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -27,7 +27,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dx 3D cell size * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. @@ -41,7 +41,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, const int* ion_lev, amrex::FArrayBox& rho_fab, - long np_to_depose, + long np_to_deposit, const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, @@ -95,8 +95,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, - [=] AMREX_GPU_DEVICE (long ip) { + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), @@ -252,7 +252,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const int n_rz_azimuthal_modes, amrex::Real* cost, const long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size, @@ -261,7 +261,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { using namespace amrex; - auto permutation = a_bins.permutationPtr(); + const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 2c4e4bd448d..2252a63fd07 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -26,6 +26,246 @@ #include #include +/** + * \brief Kernel for the direct current deposition for thread thread_num + * \tparam depos_order deposition order + * \param xp, yp, zp The particle positions. + * \param wq The charge of the macroparticle + * \param vx,vy,vz The particle velocities + * \param jx_arr,jy_arr,jz_arr Array4 of current density, either full array or tile. + * \param jx_type,jy_type,jz_type The grid types along each direction, either NODE or CELL + * \param relative_time Time at which to deposit J, relative to the time of the + * current positions of the particles. When different than 0, + * the particle position will be temporarily modified to match + * the time of the deposition. + * \param dzi, dxi, dyi The inverse cell sizes + * \param zmin, xmin, ymin The lower bounds of the domain + * \param invvol The inverse volume of a grid cell + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + */ +template +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void doDepositionShapeNKernel(const amrex::ParticleReal xp, + const amrex::ParticleReal yp, + const amrex::ParticleReal zp, + const amrex::ParticleReal wq, + const amrex::ParticleReal vx, + const amrex::ParticleReal vy, + const amrex::ParticleReal vz, + amrex::Array4 const& jx_arr, + amrex::Array4 const& jy_arr, + amrex::Array4 const& jz_arr, + amrex::IntVect const& jx_type, + amrex::IntVect const& jy_type, + amrex::IntVect const& jz_type, + const amrex::Real relative_time, + AMREX_D_DECL(const amrex::Real dzi, + const amrex::Real dxi, + const amrex::Real dyi), + AMREX_D_DECL(const amrex::Real zmin, + const amrex::Real xmin, + const amrex::Real ymin), + const amrex::Real invvol, + const amrex::Dim3 lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); +#endif +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(xp, yp); +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + amrex::ignore_unused(yp); +#endif + + constexpr int zdir = WARPX_ZINDEX; + constexpr int NODE = amrex::IndexType::NODE; + constexpr int CELL = amrex::IndexType::CELL; + + // wqx, wqy wqz are particle current in each direction +#if defined(WARPX_DIM_RZ) + // In RZ, wqx is actually wqr, and wqy is wqtheta + // Convert to cylindrical at the mid point + const amrex::Real xpmid = xp + relative_time*vx; + const amrex::Real ypmid = yp + relative_time*vy; + const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); + amrex::Real costheta; + amrex::Real sintheta; + if (rpmid > 0._rt) { + costheta = xpmid/rpmid; + sintheta = ypmid/rpmid; + } else { + costheta = 1._rt; + sintheta = 0._rt; + } + const Complex xy0 = Complex{costheta, sintheta}; + const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); + const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); +#else + const amrex::Real wqx = wq*invvol*vx; + const amrex::Real wqy = wq*invvol*vy; +#endif + const amrex::Real wqz = wq*invvol*vz; + + // --- Compute shape factors + Compute_shape_factor< depos_order > const compute_shape_factor; +#if (AMREX_SPACEDIM >= 2) + // x direction + // Get particle position after 1/2 push back in position +#if defined(WARPX_DIM_RZ) + // Keep these double to avoid bug in single precision + const double xmid = (rpmid - xmin)*dxi; +#else + const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +#endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current + // sx_j[xyz] shape factor along x for the centering of each current + // There are only two possible centerings, node or cell centered, so at most only two shape factor + // arrays will be needed. + // Keep these double to avoid bug in single precision + double sx_node[depos_order + 1] = {0.}; + double sx_cell[depos_order + 1] = {0.}; + int j_node = 0; + int j_cell = 0; + if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { + j_node = compute_shape_factor(sx_node, xmid); + } + if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { + j_cell = compute_shape_factor(sx_cell, xmid - 0.5); + } + + amrex::Real sx_jx[depos_order + 1] = {0._rt}; + amrex::Real sx_jy[depos_order + 1] = {0._rt}; + amrex::Real sx_jz[depos_order + 1] = {0._rt}; + for (int ix=0; ix<=depos_order; ix++) + { + sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); + } + + int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); + int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); + int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); +#endif //AMREX_SPACEDIM >= 2 + +#if defined(WARPX_DIM_3D) + // y direction + // Keep these double to avoid bug in single precision + const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + double sy_node[depos_order + 1] = {0.}; + double sy_cell[depos_order + 1] = {0.}; + int k_node = 0; + int k_cell = 0; + if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { + k_node = compute_shape_factor(sy_node, ymid); + } + if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { + k_cell = compute_shape_factor(sy_cell, ymid - 0.5); + } + amrex::Real sy_jx[depos_order + 1] = {0._rt}; + amrex::Real sy_jy[depos_order + 1] = {0._rt}; + amrex::Real sy_jz[depos_order + 1] = {0._rt}; + for (int iy=0; iy<=depos_order; iy++) + { + sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); + } + int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); + int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); + int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); +#endif + + // z direction + // Keep these double to avoid bug in single precision + const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + double sz_node[depos_order + 1] = {0.}; + double sz_cell[depos_order + 1] = {0.}; + int l_node = 0; + int l_cell = 0; + if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { + l_node = compute_shape_factor(sz_node, zmid); + } + if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { + l_cell = compute_shape_factor(sz_cell, zmid - 0.5); + } + amrex::Real sz_jx[depos_order + 1] = {0._rt}; + amrex::Real sz_jy[depos_order + 1] = {0._rt}; + amrex::Real sz_jz[depos_order + 1] = {0._rt}; + for (int iz=0; iz<=depos_order; iz++) + { + sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); + } + int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); + int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); + int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + + // Deposit current into jx_arr, jy_arr and jz_arr +#if defined(WARPX_DIM_1D_Z) + for (int iz=0; iz<=depos_order; iz++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+l_jx+iz, 0, 0, 0), + sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+l_jy+iz, 0, 0, 0), + sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+l_jz+iz, 0, 0, 0), + sz_jz[iz]*wqz); + } +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + for (int iz=0; iz<=depos_order; iz++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), + sx_jx[ix]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), + sx_jy[ix]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), + sx_jz[ix]*sz_jz[iz]*wqz); +#if defined(WARPX_DIM_RZ) + Complex xy = xy0; // Note that xy is equal to e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 on the weighting comes from the normalization of the modes + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); + amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); + xy = xy*xy0; + } +#endif + } + } +#elif defined(WARPX_DIM_3D) + for (int iz=0; iz<=depos_order; iz++){ + for (int iy=0; iy<=depos_order; iy++){ + for (int ix=0; ix<=depos_order; ix++){ + amrex::Gpu::Atomic::AddNoRet( + &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), + sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); + amrex::Gpu::Atomic::AddNoRet( + &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), + sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); + amrex::Gpu::Atomic::AddNoRet( + &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), + sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); + } + } + } +#endif +} + /** * \brief Current Deposition for thread thread_num * \tparam depos_order deposition order @@ -37,7 +277,7 @@ since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match @@ -60,7 +300,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -113,10 +353,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jy_type = jy_fab.box().type(); amrex::IntVect const jz_type = jz_fab.box().type(); - constexpr int zdir = WARPX_ZINDEX; - constexpr int NODE = amrex::IndexType::NODE; - constexpr int CELL = amrex::IndexType::CELL; - // Loop over particles and deposit into jx_fab, jy_fab and jz_fab #if defined(WARPX_USE_GPUCLOCK) amrex::Real* cost_real = nullptr; @@ -126,7 +362,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, + np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -134,200 +370,173 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, cost_real); #endif + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + // --- Get particle quantities const amrex::Real gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + amrex::Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; } - amrex::ParticleReal xp, yp, zp; - GetPosition(ip, xp, yp, zp); + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); - const amrex::Real vx = uxp[ip]*gaminv; - const amrex::Real vy = uyp[ip]*gaminv; - const amrex::Real vz = uzp[ip]*gaminv; - // wqx, wqy wqz are particle current in each direction -#if defined(WARPX_DIM_RZ) - // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point - const amrex::Real xpmid = xp + relative_time*vx; - const amrex::Real ypmid = yp + relative_time*vy; - const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } - const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); - const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); -#else - const amrex::Real wqx = wq*invvol*vx; - const amrex::Real wqy = wq*invvol*vy; + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } #endif - const amrex::Real wqz = wq*invvol*vz; +} - // --- Compute shape factors - Compute_shape_factor< depos_order > const compute_shape_factor; -#if (AMREX_SPACEDIM >= 2) - // x direction - // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) - // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; -#else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; +/** + * \brief Direct current deposition for thread thread_num for the implicit scheme + * The only difference from doDepositionShapeN is in how the particle gamma + * is calculated. + * \tparam depos_order deposition order + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time n. + * \param uxp,uyp,uzp Pointer to arrays of particle momentum at time n+1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + const amrex::ParticleReal * const uxp_n, + const amrex::ParticleReal * const uyp_n, + const amrex::ParticleReal * const uzp_n, + const amrex::ParticleReal * const uxp, + const amrex::ParticleReal * const uyp, + const amrex::ParticleReal * const uzp, + const int * const ion_lev, + amrex::FArrayBox& jx_fab, + amrex::FArrayBox& jy_fab, + amrex::FArrayBox& jz_fab, + const long np_to_deposit, + const std::array& dx, + const std::array& xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real* cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex::literals; +#if !defined(WARPX_DIM_RZ) + amrex::ignore_unused(n_rz_azimuthal_modes); #endif - // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current - // sx_j[xyz] shape factor along x for the centering of each current - // There are only two possible centerings, node or cell centered, so at most only two shape factor - // arrays will be needed. - // Keep these double to avoid bug in single precision - double sx_node[depos_order + 1] = {0.}; - double sx_cell[depos_order + 1] = {0.}; - int j_node = 0; - int j_cell = 0; - if (jx_type[0] == NODE || jy_type[0] == NODE || jz_type[0] == NODE) { - j_node = compute_shape_factor(sx_node, xmid); - } - if (jx_type[0] == CELL || jy_type[0] == CELL || jz_type[0] == CELL) { - j_cell = compute_shape_factor(sx_cell, xmid - 0.5); - } - amrex::Real sx_jx[depos_order + 1] = {0._rt}; - amrex::Real sx_jy[depos_order + 1] = {0._rt}; - amrex::Real sx_jz[depos_order + 1] = {0._rt}; - for (int ix=0; ix<=depos_order; ix++) - { - sx_jx[ix] = ((jx_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jy[ix] = ((jy_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - sx_jz[ix] = ((jz_type[0] == NODE) ? amrex::Real(sx_node[ix]) : amrex::Real(sx_cell[ix])); - } +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif - int const j_jx = ((jx_type[0] == NODE) ? j_node : j_cell); - int const j_jy = ((jy_type[0] == NODE) ? j_node : j_cell); - int const j_jz = ((jz_type[0] == NODE) ? j_node : j_cell); -#endif //AMREX_SPACEDIM >= 2 + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + const bool do_ionization = ion_lev; + const amrex::Real dzi = 1.0_rt/dx[2]; +#if defined(WARPX_DIM_1D_Z) + const amrex::Real invvol = dzi; +#endif +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real invvol = dxi*dzi; +#elif defined(WARPX_DIM_3D) + const amrex::Real dxi = 1.0_rt/dx[0]; + const amrex::Real dyi = 1.0_rt/dx[1]; + const amrex::Real invvol = dxi*dyi*dzi; +#endif +#if (AMREX_SPACEDIM >= 2) + const amrex::Real xmin = xyzmin[0]; +#endif #if defined(WARPX_DIM_3D) - // y direction - // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; - double sy_node[depos_order + 1] = {0.}; - double sy_cell[depos_order + 1] = {0.}; - int k_node = 0; - int k_cell = 0; - if (jx_type[1] == NODE || jy_type[1] == NODE || jz_type[1] == NODE) { - k_node = compute_shape_factor(sy_node, ymid); - } - if (jx_type[1] == CELL || jy_type[1] == CELL || jz_type[1] == CELL) { - k_cell = compute_shape_factor(sy_cell, ymid - 0.5); - } - amrex::Real sy_jx[depos_order + 1] = {0._rt}; - amrex::Real sy_jy[depos_order + 1] = {0._rt}; - amrex::Real sy_jz[depos_order + 1] = {0._rt}; - for (int iy=0; iy<=depos_order; iy++) - { - sy_jx[iy] = ((jx_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jy[iy] = ((jy_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - sy_jz[iy] = ((jz_type[1] == NODE) ? amrex::Real(sy_node[iy]) : amrex::Real(sy_cell[iy])); - } - int const k_jx = ((jx_type[1] == NODE) ? k_node : k_cell); - int const k_jy = ((jy_type[1] == NODE) ? k_node : k_cell); - int const k_jz = ((jz_type[1] == NODE) ? k_node : k_cell); + const amrex::Real ymin = xyzmin[1]; #endif + const amrex::Real zmin = xyzmin[2]; - // z direction - // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; - double sz_node[depos_order + 1] = {0.}; - double sz_cell[depos_order + 1] = {0.}; - int l_node = 0; - int l_cell = 0; - if (jx_type[zdir] == NODE || jy_type[zdir] == NODE || jz_type[zdir] == NODE) { - l_node = compute_shape_factor(sz_node, zmid); - } - if (jx_type[zdir] == CELL || jy_type[zdir] == CELL || jz_type[zdir] == CELL) { - l_cell = compute_shape_factor(sz_cell, zmid - 0.5); - } - amrex::Real sz_jx[depos_order + 1] = {0._rt}; - amrex::Real sz_jy[depos_order + 1] = {0._rt}; - amrex::Real sz_jz[depos_order + 1] = {0._rt}; - for (int iz=0; iz<=depos_order; iz++) - { - sz_jx[iz] = ((jx_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jy[iz] = ((jy_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - sz_jz[iz] = ((jz_type[zdir] == NODE) ? amrex::Real(sz_node[iz]) : amrex::Real(sz_cell[iz])); - } - int const l_jx = ((jx_type[zdir] == NODE) ? l_node : l_cell); - int const l_jy = ((jy_type[zdir] == NODE) ? l_node : l_cell); - int const l_jz = ((jz_type[zdir] == NODE) ? l_node : l_cell); + amrex::Array4 const& jx_arr = jx_fab.array(); + amrex::Array4 const& jy_arr = jy_fab.array(); + amrex::Array4 const& jz_arr = jz_fab.array(); + amrex::IntVect const jx_type = jx_fab.box().type(); + amrex::IntVect const jy_type = jy_fab.box().type(); + amrex::IntVect const jz_type = jz_fab.box().type(); - // Deposit current into jx_arr, jy_arr and jz_arr -#if defined(WARPX_DIM_1D_Z) - for (int iz=0; iz<=depos_order; iz++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+l_jx+iz, 0, 0, 0), - sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+l_jy+iz, 0, 0, 0), - sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+l_jz+iz, 0, 0, 0), - sz_jz[iz]*wqz); - } + // Loop over particles and deposit into jx_fab, jy_fab and jz_fab +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } #endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - for (int iz=0; iz<=depos_order; iz++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 0), - sx_jx[ix]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 0), - sx_jy[ix]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 0), - sx_jz[ix]*sz_jz[iz]*wqz); -#if defined(WARPX_DIM_RZ) - Complex xy = xy0; // Note that xy is equal to e^{i m theta} - for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { - // The factor 2 on the weighting comes from the normalization of the modes - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode-1), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jx_arr(lo.x+j_jx+ix, lo.y+l_jx+iz, 0, 2*imode ), 2._rt*sx_jx[ix]*sz_jx[iz]*wqx*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode-1), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jy_arr(lo.x+j_jy+ix, lo.y+l_jy+iz, 0, 2*imode ), 2._rt*sx_jy[ix]*sz_jy[iz]*wqy*xy.imag()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode-1), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.real()); - amrex::Gpu::Atomic::AddNoRet( &jz_arr(lo.x+j_jz+ix, lo.y+l_jz+iz, 0, 2*imode ), 2._rt*sx_jz[ix]*sz_jz[iz]*wqz*xy.imag()); - xy = xy*xy0; - } + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); #endif - } - } -#elif defined(WARPX_DIM_3D) - for (int iz=0; iz<=depos_order; iz++){ - for (int iy=0; iy<=depos_order; iy++){ - for (int ix=0; ix<=depos_order; ix++){ - amrex::Gpu::Atomic::AddNoRet( - &jx_arr(lo.x+j_jx+ix, lo.y+k_jx+iy, lo.z+l_jx+iz), - sx_jx[ix]*sy_jx[iy]*sz_jx[iz]*wqx); - amrex::Gpu::Atomic::AddNoRet( - &jy_arr(lo.x+j_jy+ix, lo.y+k_jy+iy, lo.z+l_jy+iz), - sx_jy[ix]*sy_jy[iy]*sz_jy[iz]*wqy); - amrex::Gpu::Atomic::AddNoRet( - &jz_arr(lo.x+j_jz+ix, lo.y+k_jz+iy, lo.z+l_jz+iz), - sx_jz[ix]*sy_jz[iy]*sz_jz[iz]*wqz); - } - } + + amrex::ParticleReal xp, yp, zp; + GetPosition(ip, xp, yp, zp); + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); + + const amrex::Real vx = uxp[ip]*gaminv; + const amrex::Real vy = uyp[ip]*gaminv; + const amrex::Real vz = uzp[ip]*gaminv; + + amrex::Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; } -#endif + + const amrex::Real relative_time = 0._rt; + doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, + jx_type, jy_type, jz_type, + relative_time, + AMREX_D_DECL(dzi, dxi, dyi), + AMREX_D_DECL(zmin, xmin, ymin), + invvol, lo, n_rz_azimuthal_modes); + } ); #if defined(WARPX_USE_GPUCLOCK) @@ -350,7 +559,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -374,7 +583,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jx_fab, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, - long np_to_depose, + long np_to_deposit, const amrex::Real relative_time, const std::array& dx, const std::array& xyzmin, @@ -383,7 +592,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, int n_rz_azimuthal_modes, amrex::Real* cost, long load_balance_costs_update_algo, - const amrex::DenseBins& a_bins, + const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, const amrex::IntVect& a_tbox_max_size) @@ -437,7 +646,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, WARPX_ALWAYS_ASSERT_WITH_MESSAGE(shared_mem_bytes <= max_shared_mem_bytes, "Tile size too big for GPU shared memory current deposition"); - amrex::ignore_unused(np_to_depose); + amrex::ignore_unused(np_to_deposit); // Launch one thread-block per bin amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), @@ -545,7 +754,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused( GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -562,7 +771,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, since q is a scalar. For non-ionizable species, ion_lev is a null pointer. * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. - * \param np_to_depose Number of particles for which current is deposited. + * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level * \param[in] relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, @@ -586,7 +795,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, const amrex::Array4& Jx_arr, const amrex::Array4& Jy_arr, const amrex::Array4& Jz_arr, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -656,7 +865,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } #endif amrex::ParallelFor( - np_to_depose, + np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -790,17 +999,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, // computes min/max positions of current contributions #if !defined(WARPX_DIM_1D_Z) int dil = 1, diu = 1; - if (i_old < i_new) dil = 0; - if (i_old > i_new) diu = 0; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } #endif #if defined(WARPX_DIM_3D) int djl = 1, dju = 1; - if (j_old < j_new) djl = 0; - if (j_old > j_new) dju = 0; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } #endif int dkl = 1, dku = 1; - if (k_old < k_new) dkl = 0; - if (k_old > k_new) dku = 0; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } #if defined(WARPX_DIM_3D) @@ -927,6 +1136,1069 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } +/** + * \brief Esirkepov Current Deposition for thread thread_num for implicit scheme + * The difference from doEsirkepovDepositionShapeN is in how the old and new + * particles positions are determined and in how the particle gamma is calculated. + * + * \tparam depos_order deposition order + * \param xp_n,yp_n,zp_n Pointer to arrays of particle position at time level n. + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time level n. + * \param uxp_nph,uyp_nph,uzp_nph Pointer to arrays of particle momentum at time level n + 1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); + Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invdtdx = 1.0_rt / (dt*dx[2]); + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invdtdz = 1.0_rt / (dt*dx[0]); + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + +#if !defined(WARPX_DIM_1D_Z) + amrex::Real const wqx = wq*invdtdx; +#endif +#if defined(WARPX_DIM_3D) + amrex::Real const wqy = wq*invdtdy; +#endif + amrex::Real const wqz = wq*invdtdz; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_new, sintheta_new; + if (rp_new > 0._rt) { + costheta_new = xp_new/rp_new; + sintheta_new = yp_new/rp_new; + } else { + costheta_new = 1._rt; + sintheta_new = 0._rt; + } + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + amrex::Real costheta_old, sintheta_old; + if (rp_old > 0._rt) { + costheta_old = xp_old/rp_old; + sintheta_old = yp_old/rp_old; + } else { + costheta_old = 1._rt; + sintheta_old = 0._rt; + } + const Complex xy_new0 = Complex{costheta_new, sintheta_new}; + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + const Complex xy_old0 = Complex{costheta_old, sintheta_old}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + +#if defined(WARPX_DIM_RZ) + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#endif + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_new[depos_order + 3] = {0.}; + double sx_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_new = compute_shape_factor(sx_new+1, x_new); + const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_new = compute_shape_factor(sy_new+1, y_new); + const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); +#endif + const int k_new = compute_shape_factor(sz_new+1, z_new); + const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil = 1, diu = 1; + if (i_old < i_new) { dil = 0; } + if (i_old > i_new) { diu = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl = 1, dju = 1; + if (j_old < j_new) { djl = 0; } + if (j_old > j_new) { dju = 0; } +#endif + int dkl = 1, dku = 1; + if (k_old < k_new) { dkl = 0; } + if (k_old > k_new) { dku = 0; } + +#if defined(WARPX_DIM_3D) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int j=djl; j<=depos_order+2-dju; j++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*( + one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) + +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); + } + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdyj = 0._rt; + for (int j=djl; j<=depos_order+1-dju; j++) { + sdyj += wqy*(sy_old[j] - sy_new[j])*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); + } + } + } + for (int j=djl; j<=depos_order+2-dju; j++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*( + one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) + +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real sdxi = 0._rt; + for (int i=dil; i<=depos_order+1-diu; i++) { + sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djr_cmplx = 2._rt *sdxi*xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djr_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djr_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + for (int i=dil; i<=depos_order+2-diu; i++) { + Real const sdyj = wq*vy*invvol*( + one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) + +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); +#if defined(WARPX_DIM_RZ) + Complex xy_new = xy_new0; + Complex xy_mid = xy_mid0; + Complex xy_old = xy_old0; + // Throughout the following loop, xy_ takes the value e^{i m theta_} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + // The minus sign comes from the different convention with respect to Davidson et al. + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djt_cmplx.imag()); + xy_new = xy_new*xy_new0; + xy_mid = xy_mid*xy_mid0; + xy_old = xy_old*xy_old0; + } +#endif + } + } + for (int i=dil; i<=depos_order+2-diu; i++) { + Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + // The factor 2 comes from the normalization of the modes + const Complex djz_cmplx = 2._rt * sdzk * xy_mid; + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djz_cmplx.real()); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode), djz_cmplx.imag()); + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdxi = wq*vx*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+k_new-1+k, 0, 0, 0), sdxi); + } + for (int k=dkl; k<=depos_order+2-dku; k++) { + amrex::Real const sdyj = wq*vy*invvol*0.5_rt*(sz_old[k] + sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+k_new-1+k, 0, 0, 0), sdyj); + } + amrex::Real sdzk = 0._rt; + for (int k=dkl; k<=depos_order+1-dku; k++) { + sdzk += wqz*(sz_old[k] - sz_new[k]); + amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); + } +#endif + } + ); +#if defined(WARPX_USE_GPUCLOCK) + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + amrex::Gpu::streamSynchronize(); + *cost += *cost_real; + amrex::The_Managed_Arena()->free(cost_real); + } +#endif +} + +/** + * \brief Villasenor and Buneman Current Deposition for thread thread_num for implicit scheme. + * The specifics for the implicit scheme are in how gamma is determined. This is a charge- + * conserving deposition. The difference from Esirkepov is that the deposit is done segment + * by segment, where the segments are determined by cell crossings. In general, this results + * in a tighter stencil. The implementation is valid for an arbitrary number of cell crossings. + * + * \param depos_order deposition order + * \param xp_n,yp_n,zp_n Pointer to arrays of particle position at time level n. + * \param GetPosition A functor for returning the particle position. + * \param wp Pointer to array of particle weights. + * \param uxp_n,uyp_n,uzp_n Pointer to arrays of particle momentum at time level n. + * \param uxp_nph,uyp_nph,uzp_nph Pointer to arrays of particle momentum at time level n + 1/2. + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. + * \param np_to_deposit Number of particles for which current is deposited. + * \param dt Time step for particle level + * \param dx 3D cell size + * \param xyzmin Physical lower bounds of domain. + * \param lo Index lower bounds of domain. + * \param q species charge. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. + * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. + * \param load_balance_costs_update_algo Selected method for updating load balance costs. + */ +template +void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, + const amrex::ParticleReal * const yp_n, + const amrex::ParticleReal * const zp_n, + const GetParticlePosition& GetPosition, + const amrex::ParticleReal * const wp, + [[maybe_unused]]const amrex::ParticleReal * const uxp_n, + [[maybe_unused]]const amrex::ParticleReal * const uyp_n, + [[maybe_unused]]const amrex::ParticleReal * const uzp_n, + [[maybe_unused]]const amrex::ParticleReal * const uxp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uyp_nph, + [[maybe_unused]]const amrex::ParticleReal * const uzp_nph, + const int * const ion_lev, + const amrex::Array4& Jx_arr, + const amrex::Array4& Jy_arr, + const amrex::Array4& Jz_arr, + const long np_to_deposit, + const amrex::Real dt, + const std::array& dx, + const std::array xyzmin, + const amrex::Dim3 lo, + const amrex::Real q, + const int n_rz_azimuthal_modes, + amrex::Real * const cost, + const long load_balance_costs_update_algo) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(AMREX_USE_GPU) + amrex::ignore_unused(cost, load_balance_costs_update_algo); +#endif + + // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer + // (do_ionization=1) + bool const do_ionization = ion_lev; +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if defined(WARPX_DIM_3D) + Real const invvol = 1.0_rt / (dx[0]*dx[1]*dx[2]); +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Real const invvol = 1.0_rt / (dx[0]*dx[2]); +#elif defined(WARPX_DIM_1D_Z) + Real const invvol = 1.0_rt / (dx[2]); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr +#if defined(WARPX_USE_GPUCLOCK) + amrex::Real* cost_real = nullptr; + if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { + cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); + *cost_real = 0._rt; + } +#endif + amrex::ParallelFor( + np_to_deposit, + [=] AMREX_GPU_DEVICE (long const ip) { +#if defined(WARPX_USE_GPUCLOCK) + const auto KernelTimer = ablastr::parallelization::KernelTimer( + cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), + cost_real); +#endif + +#if !defined(WARPX_DIM_3D) + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The uxp,uyp,uzp are the velocities at time level n+1/2 + const amrex::ParticleReal uxp_np1 = 2._prt*uxp_nph[ip] - uxp_n[ip]; + const amrex::ParticleReal uyp_np1 = 2._prt*uyp_nph[ip] - uyp_n[ip]; + const amrex::ParticleReal uzp_np1 = 2._prt*uzp_nph[ip] - uzp_n[ip]; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (uxp_n[ip]*uxp_n[ip] + uyp_n[ip]*uyp_n[ip] + uzp_n[ip]*uzp_n[ip])*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (uxp_np1*uxp_np1 + uyp_np1*uyp_np1 + uzp_np1*uzp_np1)*inv_c2); + const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); +#endif + + // wqx, wqy wqz are particle current in each direction + Real wq = q*wp[ip]; + if (do_ionization){ + wq *= ion_lev[ip]; + } + + ParticleReal xp_nph, yp_nph, zp_nph; + GetPosition(ip, xp_nph, yp_nph, zp_nph); + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; +#else + ignore_unused(xp_n); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; +#else + ignore_unused(yp_n); +#endif + ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n[ip]; + amrex::Real const yp_old = yp_n[ip]; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; + amrex::Real const vx = (rp_new - rp_old)/dt; + amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; +#elif defined(WARPX_DIM_XZ) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; + amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_1D_Z) + amrex::Real const vx = uxp_nph[ip]*gaminv; + amrex::Real const vy = uyp_nph[ip]*gaminv; +#elif defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n[ip] - xmin)*dxi; + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n[ip] - ymin)*dyi; + amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; + amrex::Real const vy = (yp_np1 - yp_n[ip])/dt; +#endif + + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n[ip] - zmin)*dzi; + amrex::Real const vz = (zp_np1 - zp_n[ip])/dt; + + // Define velocity kernals to deposit + amrex::Real const wqx = wq*vx*invvol; + amrex::Real const wqy = wq*vy*invvol; + amrex::Real const wqz = wq*vz*invvol; + + // 1) Determine the number of segments. + // 2) Loop over segments and deposit current. + + // cell crossings are defined at cell edges if depos_order is odd + // cell crossings are defined at cell centers if depos_order is even + + int num_segments = 1; + double shift = 0.0; + if ( (depos_order % 2) == 0 ) { shift = 0.5; } + +#if defined(WARPX_DIM_3D) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Y-direction + const auto j_old = static_cast(y_old-shift); + const auto j_new = static_cast(y_new-shift); + int cell_crossings_y = std::abs(j_new-j_old); + num_segments += cell_crossings_y; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 7) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dyp = y_new - y_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirY_sign = static_cast(dyp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Ycell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Ycell = static_cast(j_old) + shift + 0.5*(1.-dirY_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dyp_seg, dzp_seg; + double x0_new, y0_new, z0_new; + double x0_old = x_old; + double y0_old = y_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_y = static_cast(dyp == 0. ? 1. : dyp_seg/dyp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sy_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const y0_bar = (y0_new + y0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int j0_cell = compute_shape_factor_cell( sy_cell, y0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sy_old_cell[depos_order] = {0.}; + double sy_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int j0_cell_2 = compute_shape_factors_cell( sy_old_cell, sy_new_cell, y0_old-0.5, y0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, j0_cell_2, k0_cell_2); + for (int m=0; m(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 5) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dzp_seg; + double x0_new, z0_new; + double x0_old = x_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, k0_cell_2); + for (int m=0; m(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 3) ... + + // compute dzp and the initial cell location used to find the cell crossings. + double const dzp = z_new - z_old; + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dzp_seg; + double z0_new; + double z0_old = z_old; + + for (int ns=0; ns(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sz_cell[depos_order] = {0.}; + double const z0_bar = (z0_new + z0_old)/2.0; + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(k0_cell_2); + for (int m=0; mfree(cost_real); + } +#endif +} + /** * \brief Vay current deposition * ( Vay et al, 2013) @@ -941,7 +2213,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, required to have the charge of each macroparticle since \c q is a scalar. For non-ionizable species, \c ion_lev is \c null * \param[in,out] Dx_fab,Dy_fab,Dz_fab FArrayBox of Vay current density, either full array or tile - * \param[in] np_to_depose Number of particles for which current is deposited + * \param[in] np_to_deposit Number of particles for which current is deposited * \param[in] dt Time step for particle level * \param[in] relative_time Time at which to deposit D, relative to the time of the * current positions of the particles. When different than 0, @@ -966,7 +2238,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& Dx_fab, amrex::FArrayBox& Dy_fab, amrex::FArrayBox& Dz_fab, - long np_to_depose, + long np_to_deposit, amrex::Real dt, amrex::Real relative_time, const std::array& dx, @@ -982,14 +2254,14 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_depose, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif @@ -1054,7 +2326,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, *cost_real = 0._rt; } #endif - amrex::ParallelFor(np_to_depose, [=] AMREX_GPU_DEVICE (long ip) + amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { #if defined(WARPX_USE_GPUCLOCK) const auto KernelTimer = ablastr::parallelization::KernelTimer( @@ -1068,7 +2340,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, + uzp[ip] * uzp[ip] * invcsq); // Product of particle charges and weights amrex::Real wq = q * wp[ip]; - if (do_ionization) wq *= ion_lev[ip]; + if (do_ionization) { wq *= ion_lev[ip]; } // Current particle positions (in physical units) amrex::ParticleReal xp, yp, zp; diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 646a6add811..e28835a57df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -132,7 +132,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // pcurrent is the particle current in the deposited direction #if defined(WARPX_DIM_RZ) // In RZ, wqx is actually wqr, and wqy is wqtheta - // Convert to cylinderical at the mid point + // Convert to cylindrical at the mid point const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 573491645ea..61aeb19aea1 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -33,9 +33,11 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT m_adk_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_exp_prefactor; const amrex::Real* AMREX_RESTRICT m_adk_power; + const amrex::Real* AMREX_RESTRICT m_adk_correction_factors; int comp; int m_atomic_number; + int m_do_adk_correction = 0; GetParticlePosition m_get_position; GetExternalEBField m_get_externalEB; @@ -82,8 +84,10 @@ struct IonizationFilterFunc const amrex::Real* AMREX_RESTRICT a_adk_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* AMREX_RESTRICT a_adk_power, + const amrex::Real* AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset = 0) noexcept; template @@ -133,9 +137,16 @@ struct IonizationFilterFunc ); // Compute probability of ionization p - const amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * + amrex::Real w_dtau = (E <= 0._rt) ? 0._rt : 1._rt/ ga * m_adk_prefactor[ion_lev] * std::pow(E, m_adk_power[ion_lev]) * std::exp( m_adk_exp_prefactor[ion_lev]/E ); + // if requested, do Zhang's correction of ADK + if (m_do_adk_correction) { + const amrex::Real r = E / m_adk_correction_factors[3]; + w_dtau *= std::exp(m_adk_correction_factors[0]*r*r+m_adk_correction_factors[1]*r+ + m_adk_correction_factors[2]); + } + const amrex::Real p = 1._rt - std::exp( - w_dtau ); const amrex::Real random_draw = amrex::Random(engine); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index c3681a30cad..b7b91e4d4e3 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -30,15 +30,19 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, const amrex::Real* const AMREX_RESTRICT a_adk_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_exp_prefactor, const amrex::Real* const AMREX_RESTRICT a_adk_power, + const amrex::Real* const AMREX_RESTRICT a_adk_correction_factors, int a_comp, int a_atomic_number, + int a_do_adk_correction, int a_offset) noexcept: m_ionization_energies{a_ionization_energies}, m_adk_prefactor{a_adk_prefactor}, m_adk_exp_prefactor{a_adk_exp_prefactor}, m_adk_power{a_adk_power}, + m_adk_correction_factors{a_adk_correction_factors}, comp{a_comp}, m_atomic_number{a_atomic_number}, + m_do_adk_correction{a_do_adk_correction}, m_Ex_external_particle{E_external_particle[0]}, m_Ey_external_particle{E_external_particle[1]}, m_Ez_external_particle{E_external_particle[2]}, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H index f80705fd043..3f9ef133477 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H @@ -170,8 +170,9 @@ public: amrex::ParticleReal, pxr_p::unit_system::SI>( px, py, pz); if (gamma_photon < pxr_m::two || - chi_phot < m_bw_minimum_chi_phot) + chi_phot < m_bw_minimum_chi_phot) { return 0; + } const auto is_out = pxr_bw::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp index a43453da925..da50d41dc8f 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the breit_wheeler engine +//This file provides a wrapper around the breit_wheeler engine //provided by the PICSAR library // Factory class ============================= @@ -69,7 +69,7 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -80,8 +80,9 @@ BreitWheelerEngine::init_lookup_tables_from_raw_data ( m_dndt_table = BW_dndt_table{raw_dndt_table}; m_pair_prod_table = BW_pair_prod_table{raw_pair_prod_table}; - if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) + if (!m_dndt_table.is_init() || !m_pair_prod_table.is_init()) { return false; + } m_bw_minimum_chi_phot = bw_minimum_chi_phot; @@ -104,8 +105,9 @@ void BreitWheelerEngine::init_builtin_tables( vector BreitWheelerEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_pair_prod = m_pair_prod_table.serialize(); @@ -114,10 +116,12 @@ vector BreitWheelerEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_pair_prod) + } + for (const auto& tmp : data_pair_prod) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H index 3b2093c40b7..4f26c256c76 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QedWrapperCommons.H @@ -36,7 +36,7 @@ /** * PICSAR uses internally some specifiers analogous to * AMREX_RESTRICT and AMREX_FORCE_INLINE. These definitions - * set the aformentioned specifiers to AMREX_RESTRICT and + * set the aforementioned specifiers to AMREX_RESTRICT and * AMREX_FORCE_INLINE. */ #define PXRMP_RESTRICT AMREX_RESTRICT diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H index ceacb2016e8..3ae27e411b6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H @@ -153,8 +153,9 @@ public: const auto chi_part = QedUtils::chi_ele_pos( m_e*ux, m_e*uy, m_e*uz, ex, ey, ez, bx, by, bz); - if (chi_part < m_qs_minimum_chi_part) + if (chi_part < m_qs_minimum_chi_part) { return 0; + } const auto is_out = pxr_qs::evolve_optical_depth< amrex::ParticleReal, diff --git a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp index 2a52829272d..59a881814c6 100644 --- a/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp +++ b/Source/Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.cpp @@ -30,7 +30,7 @@ using namespace std; using namespace amrex; namespace pxr_sr = picsar::multi_physics::utils::serialization; -//This file provides a wrapper aroud the quantum_sync engine +//This file provides a wrapper around the quantum_sync engine //provided by the PICSAR library // Factory class ============================= @@ -68,7 +68,7 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( { auto raw_iter = raw_data.begin(); const auto size_first = pxr_sr::get_out(raw_iter); - if(size_first <= 0 || size_first >= raw_data.size() ) return false; + if(size_first <= 0 || size_first >= raw_data.size() ) { return false; } const auto raw_dndt_table = vector{ raw_iter, raw_iter+static_cast(size_first)}; @@ -79,8 +79,9 @@ QuantumSynchrotronEngine::init_lookup_tables_from_raw_data ( m_dndt_table = QS_dndt_table{raw_dndt_table}; m_phot_em_table = QS_phot_em_table{raw_phot_em_table}; - if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) + if (!m_dndt_table.is_init() || !m_phot_em_table.is_init()) { return false; + } m_qs_minimum_chi_part = qs_minimum_chi_part; @@ -103,8 +104,9 @@ void QuantumSynchrotronEngine::init_builtin_tables( vector QuantumSynchrotronEngine::export_lookup_tables_data () const { - if(!m_lookup_tables_initialized) + if(!m_lookup_tables_initialized) { return vector{}; + } const auto data_dndt = m_dndt_table.serialize(); const auto data_phot_em = m_phot_em_table.serialize(); @@ -113,10 +115,12 @@ vector QuantumSynchrotronEngine::export_lookup_tables_data () const vector res{}; pxr_sr::put_in(size_first, res); - for (const auto& tmp : data_dndt) + for (const auto& tmp : data_dndt) { pxr_sr::put_in(tmp, res); - for (const auto& tmp : data_phot_em) + } + for (const auto& tmp : data_phot_em) { pxr_sr::put_in(tmp, res); + } return res; } diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index 5abc9282d4f..fb723f0b79a 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -167,7 +167,7 @@ public: p_ux, p_uy, p_uz, engine); - src.m_aos[i_src].id() = -1; //destroy photon after pair generation + src.m_idcpu[i_src] = amrex::ParticleIdCpus::Invalid; // destroy photon after pair generation } private: diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 8ba5c63ad57..567b260d0e4 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -237,12 +238,11 @@ void cleanLowEnergyPhotons( const int old_size, const int num_added, const amrex::ParticleReal energy_threshold) { - auto pp = ptile.GetArrayOfStructs()().data() + old_size; - - const auto& soa = ptile.GetStructOfArrays(); + auto& soa = ptile.GetStructOfArrays(); + auto p_idcpu = soa.GetIdCPUData().data() + old_size; const auto p_ux = soa.GetRealData(PIdx::ux).data() + old_size; - const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; - const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; + const auto p_uy = soa.GetRealData(PIdx::uy).data() + old_size; + const auto p_uz = soa.GetRealData(PIdx::uz).data() + old_size; //The square of the energy threshold const auto energy_threshold2 = std::max( @@ -251,8 +251,6 @@ void cleanLowEnergyPhotons( amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - const auto ux = p_ux[ip]; const auto uy = p_uy[ip]; const auto uz = p_uz[ip]; @@ -262,8 +260,8 @@ void cleanLowEnergyPhotons( constexpr amrex::ParticleReal me_c = PhysConst::m_e*PhysConst::c; const auto phot_energy2 = (ux*ux + uy*uy + uz*uz)*me_c*me_c; - if (phot_energy2 < energy_threshold2){ - p.id() = - 1; + if (phot_energy2 < energy_threshold2) { + p_idcpu[ip] = amrex::ParticleIdCpus::Invalid; } }); } diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index b7334953861..1924c1685dd 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -197,14 +197,14 @@ struct GeometryFilter GeometryFilter(bool a_is_active, amrex::RealBox a_domain) : m_is_active(a_is_active), m_domain(a_domain) {} /** - * \brief return 1 if the partcile is within the region described by the RealBox + * \brief return 1 if the particle is within the region described by the RealBox * \param p one particle * \return whether or not the particle is inside the region defined by m_domain */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator () (const SuperParticleType& p, const amrex::RandomEngine&) const noexcept { - if ( !m_is_active ) return true; + if ( !m_is_active ) { return true; } return ! (AMREX_D_TERM( (p.pos(0) < m_domain.lo(0)) || (p.pos(0) > m_domain.hi(0) ), || (p.pos(1) < m_domain.lo(1)) || (p.pos(1) > m_domain.hi(1) ), || (p.pos(2) < m_domain.lo(2)) || (p.pos(2) > m_domain.hi(2) ))); diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index 1b5cede3c6f..d43c6c0741d 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -451,6 +451,1110 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif } +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Esirkepov current deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNEsirkepovStencilImplicit ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; +#else +#if !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; +#endif +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + + // Shape factor arrays + // Note that there are extra values above and below + // to possibly hold the factor for the old particle + // which can be at a different grid location. + // Keep these double to avoid bug in single precision +#if !defined(WARPX_DIM_1D_Z) + double sx_E_new[depos_order + 3] = {0.}; + double sx_E_old[depos_order + 3] = {0.}; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double sy_E_new[depos_order + 3] = {0.}; + double sy_E_old[depos_order + 3] = {0.}; +#endif + // Keep these double to avoid bug in single precision + double sz_E_new[depos_order + 3] = {0.}; + double sz_E_old[depos_order + 3] = {0.}; + +#if defined(WARPX_DIM_3D) + double sx_B_new[depos_order + 3] = {0.}; + double sx_B_old[depos_order + 3] = {0.}; + double sy_B_new[depos_order + 3] = {0.}; + double sy_B_old[depos_order + 3] = {0.}; + double sz_B_new[depos_order + 3] = {0.}; + double sz_B_old[depos_order + 3] = {0.}; +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + // Special shape functions are needed for By which is cell + // centered in both x and z. One lower order shape function is used. + double sx_By_new[depos_order + 2] = {0.}; + double sx_By_old[depos_order + 2] = {0.}; + double sz_By_new[depos_order + 2] = {0.}; + double sz_By_old[depos_order + 2] = {0.}; +#endif + + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + Compute_shape_factor< depos_order > compute_shape_factor; + Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + +#if !defined(WARPX_DIM_1D_Z) + const int i_E_new = compute_shape_factor(sx_E_new+1, x_new); + const int i_E_old = compute_shifted_shape_factor(sx_E_old, x_old, i_E_new); +#endif +#if defined(WARPX_DIM_3D) + const int j_E_new = compute_shape_factor(sy_E_new+1, y_new); + const int j_E_old = compute_shifted_shape_factor(sy_E_old, y_old, j_E_new); +#endif + const int k_E_new = compute_shape_factor(sz_E_new+1, z_new); + const int k_E_old = compute_shifted_shape_factor(sz_E_old, z_old, k_E_new); + +#if defined(WARPX_DIM_3D) + const int i_B_new = compute_shape_factor(sx_B_new+1, x_new - 0.5_rt); + const int i_B_old = compute_shifted_shape_factor(sx_B_old, x_old - 0.5_rt, i_B_new); + const int j_B_new = compute_shape_factor(sy_B_new+1, y_new - 0.5_rt); + const int j_B_old = compute_shifted_shape_factor(sy_B_old, y_old - 0.5_rt, j_B_new); + const int k_B_new = compute_shape_factor(sz_B_new+1, z_new - 0.5_rt); + const int k_B_old = compute_shifted_shape_factor(sz_B_old, z_old - 0.5_rt, k_B_new); +#endif + + // computes min/max positions of current contributions +#if !defined(WARPX_DIM_1D_Z) + int dil_E = 1, diu_E = 1; + if (i_E_old < i_E_new) { dil_E = 0; } + if (i_E_old > i_E_new) { diu_E = 0; } +#endif +#if defined(WARPX_DIM_3D) + int djl_E = 1, dju_E = 1; + if (j_E_old < j_E_new) { djl_E = 0; } + if (j_E_old > j_E_new) { dju_E = 0; } +#endif + int dkl_E = 1, dku_E = 1; + if (k_E_old < k_E_new) { dkl_E = 0; } + if (k_E_old > k_E_new) { dku_E = 0; } + +#if defined(WARPX_DIM_3D) + int dil_B = 1, diu_B = 1; + if (i_B_old < i_B_new) { dil_B = 0; } + if (i_B_old > i_B_new) { diu_B = 0; } + int djl_B = 1, dju_B = 1; + if (j_B_old < j_B_new) { djl_B = 0; } + if (j_B_old > j_B_new) { dju_B = 0; } + int dkl_B = 1, dku_B = 1; + if (k_B_old < k_B_new) { dkl_B = 0; } + if (k_B_old > k_B_new) { dku_B = 0; } +#endif + +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + Compute_shape_factor< depos_order-1 > compute_shape_factor_By; + Compute_shifted_shape_factor< depos_order-1 > compute_shifted_shape_factor_By; + const int i_By_new = compute_shape_factor_By(sx_By_new+1, x_new - 0.5_rt); + const int i_By_old = compute_shifted_shape_factor_By(sx_By_old, x_old - 0.5_rt, i_By_new); + const int k_By_new = compute_shape_factor_By(sz_By_new+1, z_new - 0.5_rt); + const int k_By_old = compute_shifted_shape_factor_By(sz_By_old, z_old - 0.5_rt, k_By_new); + int dil_By = 1, diu_By = 1; + if (i_By_old < i_By_new) { dil_By = 0; } + if (i_By_old > i_By_new) { diu_By = 0; } + int dkl_By = 1, dku_By = 1; + if (k_By_old < k_By_new) { dkl_By = 0; } + if (k_By_old > k_By_new) { dku_By = 0; } +#endif + +#if defined(WARPX_DIM_3D) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + amrex::Real sdzjk = one_third*(sy_E_new[j]*sz_E_new[k] + sy_E_old[j]*sz_E_old[k]) + +one_sixth*(sy_E_new[j]*sz_E_old[k] + sy_E_old[j]*sz_E_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdyik = one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_E; j<=depos_order+1-dju_E; j++) { + sdyj += (sy_E_old[j] - sy_E_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_E; j<=depos_order+2-dju_E; j++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdzij = one_third*(sx_E_new[i]*sy_E_new[j] + sx_E_old[i]*sy_E_old[j]) + +one_sixth*(sx_E_new[i]*sy_E_old[j] + sx_E_old[i]*sy_E_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+j_E_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + amrex::Real sdzjk = one_third*(sy_B_new[j]*sz_B_new[k] + sy_B_old[j]*sz_B_old[k]) + +one_sixth*(sy_B_new[j]*sz_B_old[k] + sy_B_old[j]*sz_B_new[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_B; i<=depos_order+1-diu_B; i++) { + sdxi += (sx_B_old[i] - sx_B_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Bxp += Bx_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdxiov*sdzjk; + } + } + } + for (int k=dkl_B; k<=depos_order+2-dku_B; k++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdyik = one_third*(sx_B_new[i]*sz_B_new[k] + sx_B_old[i]*sz_B_old[k]) + +one_sixth*(sx_B_new[i]*sz_B_old[k] + sx_B_old[i]*sz_B_new[k]); + amrex::Real sdyj = 0._rt; + for (int j=djl_B; j<=depos_order+1-dju_B; j++) { + sdyj += (sy_B_old[j] - sy_B_new[j]); + auto sdyjov = static_cast((y_new - y_old) == 0. ? 1. : sdyj/(y_new - y_old)); + Byp += By_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_B_new-1+k)*sdyjov*sdyik; + } + } + } + for (int j=djl_B; j<=depos_order+2-dju_B; j++) { + for (int i=dil_B; i<=depos_order+2-diu_B; i++) { + amrex::Real sdzij = one_third*(sx_B_new[i]*sy_B_new[j] + sx_B_old[i]*sy_B_old[j]) + +one_sixth*(sx_B_new[i]*sy_B_old[j] + sx_B_old[i]*sy_B_new[j]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_B; k<=depos_order+1-dku_B; k++) { + sdzk += (sz_B_old[k] - sz_B_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bzp += Bz_arr(lo.x+i_B_new-1+i, lo.y+j_B_new-1+j, lo.z+k_E_new-1+k)*sdzkov*sdzij; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + Exp += Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + Bzp += Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdxiov*sdzk; + } + } + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + Eyp += Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdyj; + } + } + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Ezp += Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + Bxp += Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 0)*sdzkov*sdxi; + } + } + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + Byp += By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 0)*sdyj; + } + } + +#ifdef WARPX_DIM_RZ + Complex xy_mid = xy_mid0; + + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + + // Gather field on particle Exp from field on grid ex_arr + // Gather field on particle Bzp from field on grid bz_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real sdzk = 0.5_rt*(sz_E_new[k] + sz_E_old[k]); + amrex::Real sdxi = 0._rt; + for (int i=dil_E; i<=depos_order+1-diu_E; i++) { + sdxi += (sx_E_old[i] - sx_E_new[i]); + auto sdxiov = static_cast((x_new - x_old) == 0. ? 1. : sdxi/(x_new - x_old)); + const amrex::Real dEx = (+ Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ex_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBz = (+ Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Exp += dEx*sdxiov*sdzk; + Bzp += dBz*sdxiov*sdzk; + } + } + // Gather field on particle Eyp from field on grid ey_arr + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + Real const sdyj = ( + one_third*(sx_E_new[i]*sz_E_new[k] + sx_E_old[i]*sz_E_old[k]) + +one_sixth*(sx_E_new[i]*sz_E_old[k] + sx_E_old[i]*sz_E_new[k])); + const amrex::Real dEy = (+ Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ey_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Eyp += dEy*sdyj; + } + } + // Gather field on particle Ezp from field on grid ez_arr + // Gather field on particle Bxp from field on grid bx_arr + for (int i=dil_E; i<=depos_order+2-diu_E; i++) { + amrex::Real sdxi = 0.5_rt*(sx_E_new[i] + sx_E_old[i]); + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + const amrex::Real dEz = (+ Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Ez_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + const amrex::Real dBx = (+ Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_E_new-1+i, lo.y+k_E_new-1+k, 0, 2*imode)*xy_mid.imag()); + Ezp += dEz*sdzkov*sdxi; + Bxp += dBx*sdzkov*sdxi; + } + } + // Gather field on particle Byp from field on grid by_arr + for (int k=dkl_By; k<=depos_order+1-dku_By; k++) { + for (int i=dil_By; i<=depos_order+1-diu_By; i++) { + Real const sdyj = ( + one_third*(sx_By_new[i]*sz_By_new[k] + sx_By_old[i]*sz_By_old[k]) + +one_sixth*(sx_By_new[i]*sz_By_old[k] + sx_By_old[i]*sz_By_new[k])); + const amrex::Real dBy = (+ By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_By_new-1+i, lo.y+k_By_new-1+k, 0, 2*imode)*xy_mid.imag()); + Byp += dBy*sdyj; + } + } + xy_mid = xy_mid*xy_mid0; + } + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + for (int k=dkl_E; k<=depos_order+2-dku_E; k++) { + amrex::Real const sdzk = 0.5_rt*(sz_E_old[k] + sz_E_new[k]); + Exp += Ex_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Eyp += Ey_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + Bzp += Bz_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzk; + } + amrex::Real sdzk = 0._rt; + for (int k=dkl_E; k<=depos_order+1-dku_E; k++) { + sdzk += (sz_E_old[k] - sz_E_new[k]); + auto sdzkov = static_cast((z_new - z_old) == 0. ? 1. : sdzk/(z_new - z_old)); + Bxp += Bx_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Byp += By_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + Ezp += Ez_arr(lo.x+k_E_new-1+k, 0, 0, 0)*sdzkov; + } +#endif +} + +/** + * \brief Energy conserving field gather for thread thread_num for the implicit scheme + * This uses the same stencil for the gather that is used for Villasenor current deposition. + * The magnetic field is deposited using direct deposition. + * + * \tparam depos_order Particle shape order + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half step + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param Ex_arr,Ey_arr,Ez_arr Array4 of the electric field, either full array or tile. + * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. + * \param Ex_type,Ey_type,Ez_type IndexType of the electric field + * \param Bx_type,By_type,Bz_type IndexType of the magnetic field + * \param dx 3D cell spacing + * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + */ +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherPicnicShapeN ( + [[maybe_unused]] const amrex::ParticleReal xp_n, + [[maybe_unused]] const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + [[maybe_unused]] const amrex::ParticleReal xp_nph, + [[maybe_unused]] const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& Ex_arr, + amrex::Array4 const& Ey_arr, + amrex::Array4 const& Ez_arr, + amrex::Array4 const& Bx_arr, + amrex::Array4 const& By_arr, + amrex::Array4 const& Bz_arr, + [[maybe_unused]] const amrex::IndexType Ex_type, + [[maybe_unused]] const amrex::IndexType Ey_type, + [[maybe_unused]] const amrex::IndexType Ez_type, + [[maybe_unused]] const amrex::IndexType Bx_type, + [[maybe_unused]] const amrex::IndexType By_type, + [[maybe_unused]] const amrex::IndexType Bz_type, + const amrex::GpuArray& dx, + const amrex::GpuArray& xyzmin, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes) +{ + using namespace amrex; +#if !defined(WARPX_DIM_RZ) + ignore_unused(n_rz_azimuthal_modes); +#endif + +#if !defined(WARPX_DIM_1D_Z) + Real const dxi = 1.0_rt / dx[0]; +#endif +#if !defined(WARPX_DIM_1D_Z) + Real const xmin = xyzmin[0]; +#endif +#if defined(WARPX_DIM_3D) + Real const dyi = 1.0_rt / dx[1]; + Real const ymin = xyzmin[1]; +#endif + Real const dzi = 1.0_rt / dx[2]; + Real const zmin = xyzmin[2]; + +#if !defined(WARPX_DIM_1D_Z) + ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal yp_np1 = 2._prt*yp_nph - yp_n; +#endif + ParticleReal zp_np1 = 2._prt*zp_nph - zp_n; + +#if !defined(WARPX_DIM_1D_Z) + Real constexpr one_third = 1.0_rt / 3.0_rt; + Real constexpr one_sixth = 1.0_rt / 6.0_rt; +#endif + + // computes current and old position in grid units +#if defined(WARPX_DIM_RZ) + amrex::Real const xp_new = xp_np1; + amrex::Real const yp_new = yp_np1; + amrex::Real const xp_mid = xp_nph; + amrex::Real const yp_mid = yp_nph; + amrex::Real const xp_old = xp_n; + amrex::Real const yp_old = yp_n; + amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); + amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); + amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; + amrex::Real costheta_mid, sintheta_mid; + if (rp_mid > 0._rt) { + costheta_mid = xp_mid/rp_mid; + sintheta_mid = yp_mid/rp_mid; + } else { + costheta_mid = 1._rt; + sintheta_mid = 0._rt; + } + const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; + // Keep these double to avoid bug in single precision + double const x_new = (rp_new - xmin)*dxi; + double const x_old = (rp_old - xmin)*dxi; + double const x_bar = (rp_mid - xmin)*dxi; +#elif !defined(WARPX_DIM_1D_Z) + // Keep these double to avoid bug in single precision + double const x_new = (xp_np1 - xmin)*dxi; + double const x_old = (xp_n - xmin)*dxi; + double const x_bar = (xp_nph - xmin)*dxi; +#endif +#if defined(WARPX_DIM_3D) + // Keep these double to avoid bug in single precision + double const y_new = (yp_np1 - ymin)*dyi; + double const y_old = (yp_n - ymin)*dyi; + double const y_bar = (yp_nph - ymin)*dyi; +#endif + // Keep these double to avoid bug in single precision + double const z_new = (zp_np1 - zmin)*dzi; + double const z_old = (zp_n - zmin)*dzi; + double const z_bar = (zp_nph - zmin)*dzi; + + // 1) Determine the number of segments. + // 2) Loop over segments and gather electric field. + // 3) Gather magnetic field. + + // cell crossings are defined at cell edges if depos_order is odd + // cell crossings are defined at cell centers if depos_order is even + + int num_segments = 1; + double shift = 0.0; + if ( (depos_order % 2) == 0 ) { shift = 0.5; } + +#if defined(WARPX_DIM_3D) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Y-direction + const auto j_old = static_cast(y_old-shift); + const auto j_new = static_cast(y_new-shift); + int cell_crossings_y = std::abs(j_new-j_old); + num_segments += cell_crossings_y; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 7) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dyp = y_new - y_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirY_sign = static_cast(dyp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Ycell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Ycell = static_cast(j_old) + shift + 0.5*(1.-dirY_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dyp_seg, dzp_seg; + double x0_new, y0_new, z0_new; + double x0_old = x_old; + double y0_old = y_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_y = static_cast(dyp == 0. ? 1. : dyp_seg/dyp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sy_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const y0_bar = (y0_new + y0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell( sx_cell, x0_bar-0.5 ); + const int j0_cell = compute_shape_factor_cell( sy_cell, y0_bar-0.5 ); + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sy_old_cell[depos_order] = {0.}; + double sy_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int j0_cell_2 = compute_shape_factors_cell( sy_old_cell, sy_new_cell, y0_old-0.5, y0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, j0_cell_2, k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5); + double sy_bar_node[depos_order_B+1] = {0.}; + double sy_bar_cell[depos_order_B+1] = {0.}; + const int j_bar_node = compute_shape_factor_B(sy_bar_node, y_bar); + const int j_bar_cell = compute_shape_factor_B(sy_bar_cell, y_bar-0.5); + double sx_bar_node[depos_order_B+1] = {0.}; + double sx_bar_cell[depos_order_B+1] = {0.}; + const int i_bar_node = compute_shape_factor_B(sx_bar_node, x_bar); + const int i_bar_cell = compute_shape_factor_B(sx_bar_cell, x_bar-0.5); + + amrex::Real weight; + for (int i=0; i<=depos_order_B; i++) { + for (int j=0; j<=depos_order_B; j++) { + for (int k=0; k<=depos_order_B; k++) { + weight = static_cast(sx_bar_node[i]*sy_bar_cell[j]*sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+i_bar_node+i, lo.y+j_bar_cell+j, lo.z+k_bar_cell+k)*weight; + // + weight = static_cast(sx_bar_cell[i]*sy_bar_node[j]*sz_bar_cell[k]); + Byp += By_arr(lo.x+i_bar_cell+i, lo.y+j_bar_node+j, lo.z+k_bar_cell+k)*weight; + // + weight = static_cast(sx_bar_cell[i]*sy_bar_cell[j]*sz_bar_node[k]); + Bzp += Bz_arr(lo.x+i_bar_cell+i, lo.y+j_bar_cell+j, lo.z+k_bar_node+k)*weight; + } + } + } + +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + + // compute cell crossings in X-direction + const auto i_old = static_cast(x_old-shift); + const auto i_new = static_cast(x_new-shift); + int cell_crossings_x = std::abs(i_new-i_old); + num_segments += cell_crossings_x; + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 5) ... + + // compute total change in particle position and the initial cell + // locations in each direction used to find the position at cell crossings. + const double dxp = x_new - x_old; + const double dzp = z_new - z_old; + const auto dirX_sign = static_cast(dxp < 0. ? -1. : 1.); + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Xcell = 0., Zcell = 0.; + if (num_segments > 1) { + Xcell = static_cast(i_old) + shift + 0.5*(1.-dirX_sign); + Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + } + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dxp_seg, dzp_seg; + double x0_new, z0_new; + double x0_old = x_old; + double z0_old = z_old; + + for (int ns=0; ns(dxp == 0. ? 1. : dxp_seg/dxp); + const auto seg_factor_z = static_cast(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sx_cell[depos_order] = {0.}; + double sz_cell[depos_order] = {0.}; + double const x0_bar = (x0_new + x0_old)/2.0; + double const z0_bar = (z0_new + z0_old)/2.0; + const int i0_cell = compute_shape_factor_cell(sx_cell, x0_bar-0.5); + const int k0_cell = compute_shape_factor_cell(sz_cell, z0_bar-0.5); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sx_old_cell[depos_order] = {0.}; + double sx_new_cell[depos_order] = {0.}; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int i0_cell_2 = compute_shape_factors_cell( sx_old_cell, sx_new_cell, x0_old-0.5, x0_new-0.5 ); + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(i0_cell_2, k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5); + double sx_bar_node[depos_order_B+1] = {0.}; + double sx_bar_cell[depos_order_B+1] = {0.}; + const int i_bar_node = compute_shape_factor_B(sx_bar_node, x_bar); + const int i_bar_cell = compute_shape_factor_B(sx_bar_cell, x_bar-0.5); + + for (int i=0; i<=depos_order_B; i++) { + for (int k=0; k<=depos_order_B; k++) { + const auto weight_Bz = static_cast(sx_bar_cell[i]*sz_bar_node[k]); + Bzp += Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 0)*weight_Bz; + // + const auto weight_Bx = static_cast(sx_bar_node[i]*sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 0)*weight_Bx; + // + const auto weight_By = static_cast(sx_bar_cell[i]*sz_bar_cell[k]); + Byp += By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 0)*weight_By; +#if defined(WARPX_DIM_RZ) + Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} + for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { + const auto dBx = (+ Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 2*imode-1)*xy_mid.real() + - Bx_arr(lo.x+i_bar_node+i, lo.y+k_bar_cell+k, 0, 2*imode)*xy_mid.imag()); + const auto dBy = (+ By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 2*imode-1)*xy_mid.real() + - By_arr(lo.x+i_bar_cell+i, lo.y+k_bar_cell+k, 0, 2*imode)*xy_mid.imag()); + const auto dBz = (+ Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 2*imode-1)*xy_mid.real() + - Bz_arr(lo.x+i_bar_cell+i, lo.y+k_bar_node+k, 0, 2*imode)*xy_mid.imag()); + Bxp += weight_Bx*dBx; + Byp += weight_By*dBy; + Bzp += weight_Bz*dBz; + xy_mid = xy_mid*xy_mid0; + } +#endif + } + } + +#ifdef WARPX_DIM_RZ + + // Convert Exp and Eyp (which are actually Er and Etheta) to Ex and Ey + const amrex::Real Exp_save = Exp; + Exp = costheta_mid*Exp - sintheta_mid*Eyp; + Eyp = costheta_mid*Eyp + sintheta_mid*Exp_save; + const amrex::Real Bxp_save = Bxp; + Bxp = costheta_mid*Bxp - sintheta_mid*Byp; + Byp = costheta_mid*Byp + sintheta_mid*Bxp_save; + +#endif + +#elif defined(WARPX_DIM_1D_Z) + + // compute cell crossings in Z-direction + const auto k_old = static_cast(z_old-shift); + const auto k_new = static_cast(z_new-shift); + int cell_crossings_z = std::abs(k_new-k_old); + num_segments += cell_crossings_z; + + // need to assert that the number of cell crossings in each direction + // is within the range permitted by the number of guard cells + // e.g., if (num_segments > 3) ... + + // compute dzp and the initial cell location used to find the cell crossings. + double const dzp = z_new - z_old; + const auto dirZ_sign = static_cast(dzp < 0. ? -1. : 1.); + double Zcell = static_cast(k_old) + shift + 0.5*(1.-dirZ_sign); + + // loop over the number of segments and deposit + Compute_shape_factor< depos_order-1 > compute_shape_factor_cell; + Compute_shape_factor_pair< depos_order > compute_shape_factors_node; + double dzp_seg; + double z0_new; + double z0_old = z_old; + + for (int ns=0; ns(dzp == 0. ? 1. : dzp_seg/dzp); + + // compute cell-based weights using the average segment position + double sz_cell[depos_order] = {0.}; + double const z0_bar = (z0_new + z0_old)/2.0; + const int k0_cell = compute_shape_factor_cell( sz_cell, z0_bar-0.5 ); + + if constexpr (depos_order >= 3) { // higher-order correction to the cell-based weights + Compute_shape_factor_pair compute_shape_factors_cell; + double sz_old_cell[depos_order] = {0.}; + double sz_new_cell[depos_order] = {0.}; + const int k0_cell_2 = compute_shape_factors_cell( sz_old_cell, sz_new_cell, z0_old-0.5, z0_new-0.5 ); + ignore_unused(k0_cell_2); + for (int m=0; m compute_shape_factor_B; + double sz_bar_node[depos_order_B+1] = {0.}; + double sz_bar_cell[depos_order_B+1] = {0.}; + const int k_bar_node = compute_shape_factor_B(sz_bar_node, z_bar); + const int k_bar_cell = compute_shape_factor_B(sz_bar_cell, z_bar-0.5_rt); + + amrex::Real weight; + for (int k=0; k<=depos_order_B; k++) { + weight = static_cast(sz_bar_node[k]); + Bzp += Bz_arr(lo.x+k_bar_node+k, 0, 0)*weight; + // + weight = static_cast(sz_bar_cell[k]); + Bxp += Bx_arr(lo.x+k_bar_cell+k, 0, 0)*weight; + Byp += By_arr(lo.x+k_bar_cell+k, 0, 0)*weight; + } + +#endif +} + /** * \brief Field gather for particles * @@ -585,6 +1689,11 @@ void doGatherShapeN (const amrex::ParticleReal xp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } else { if (nox == 1) { @@ -602,6 +1711,143 @@ void doGatherShapeN (const amrex::ParticleReal xp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } +} + + +/** + * \brief Field gather for a single particle + * + * \param xp_n,yp_n,zp_n Particle position coordinates at start of step + * \param xp_nph,yp_nph,zp_nph Particle position coordinates at half time level (n + half) + * \param Exp,Eyp,Ezp Electric field on particles. + * \param Bxp,Byp,Bzp Magnetic field on particles. + * \param ex_arr,ey_arr,ez_arr Array4 of the electric field, either full array or tile. + * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. + * \param ex_type,ey_type,ez_type IndexType of the electric field + * \param bx_type,by_type,bz_type IndexType of the magnetic field + * \param dx_arr 3D cell spacing + * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param lo Index lower bounds of domain. + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param nox order of the particle shape function + * \param gather_type integer identifier for which algorithm to use + * \param galerkin_interpolation whether to use lower order in v + */ +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void doGatherShapeNImplicit ( + const amrex::ParticleReal xp_n, + const amrex::ParticleReal yp_n, + const amrex::ParticleReal zp_n, + const amrex::ParticleReal xp_nph, + const amrex::ParticleReal yp_nph, + const amrex::ParticleReal zp_nph, + amrex::ParticleReal& Exp, + amrex::ParticleReal& Eyp, + amrex::ParticleReal& Ezp, + amrex::ParticleReal& Bxp, + amrex::ParticleReal& Byp, + amrex::ParticleReal& Bzp, + amrex::Array4 const& ex_arr, + amrex::Array4 const& ey_arr, + amrex::Array4 const& ez_arr, + amrex::Array4 const& bx_arr, + amrex::Array4 const& by_arr, + amrex::Array4 const& bz_arr, + const amrex::IndexType ex_type, + const amrex::IndexType ey_type, + const amrex::IndexType ez_type, + const amrex::IndexType bx_type, + const amrex::IndexType by_type, + const amrex::IndexType bz_type, + const amrex::GpuArray& dx_arr, + const amrex::GpuArray& xyzmin_arr, + const amrex::Dim3& lo, + const int n_rz_azimuthal_modes, + const int nox, + const int depos_type ) +{ + if (depos_type==0) { // CurrentDepositionAlgo::Esirkepov + if (nox == 1) { + doGatherShapeNEsirkepovStencilImplicit<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeNEsirkepovStencilImplicit<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } + else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor + if (nox == 1) { + doGatherPicnicShapeN<1>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherPicnicShapeN<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherPicnicShapeN<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherPicnicShapeN<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } + } + else if (depos_type==1) { // CurrentDepositionAlgo::Direct + if (nox == 1) { + doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 2) { + doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 3) { + doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + } else if (nox == 4) { + doGatherShapeN<4,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/Gather/GetExternalFields.H b/Source/Particles/Gather/GetExternalFields.H index 5e74a409824..9fd534e33d9 100644 --- a/Source/Particles/Gather/GetExternalFields.H +++ b/Source/Particles/Gather/GetExternalFields.H @@ -58,7 +58,7 @@ struct GetExternalEBField std::optional d_lattice_element_finder; - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.has_value()); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -78,7 +78,7 @@ struct GetExternalEBField field_Bx, field_By, field_Bz); } - if (m_Etype == None && m_Btype == None) return; + if (m_Etype == None && m_Btype == None) { return; } amrex::ParticleReal Ex = 0._prt; amrex::ParticleReal Ey = 0._prt; diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 4aeb6ce05b9..8e3c679aa3a 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -31,12 +31,12 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_Etype = Unknown; m_Btype = Unknown; - if (mypc.m_E_ext_particle_s == "none") m_Etype = None; - if (mypc.m_B_ext_particle_s == "none") m_Btype = None; + if (mypc.m_E_ext_particle_s == "none") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "none") { m_Btype = None; } // These lines will be removed once the user interface is redefined and the CI tests updated - if (mypc.m_E_ext_particle_s == "constant") m_Etype = None; - if (mypc.m_B_ext_particle_s == "constant") m_Btype = None; + if (mypc.m_E_ext_particle_s == "constant") { m_Etype = None; } + if (mypc.m_B_ext_particle_s == "constant") { m_Btype = None; } if (mypc.m_E_ext_particle_s == "parse_e_ext_particle_function" || mypc.m_B_ext_particle_s == "parse_b_ext_particle_function" || @@ -68,10 +68,10 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset if (mypc.m_E_ext_particle_s == "repeated_plasma_lens" || mypc.m_B_ext_particle_s == "repeated_plasma_lens") { - if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") m_Etype = RepeatedPlasmaLens; - if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") m_Btype = RepeatedPlasmaLens; + if (mypc.m_E_ext_particle_s == "repeated_plasma_lens") { m_Etype = RepeatedPlasmaLens; } + if (mypc.m_B_ext_particle_s == "repeated_plasma_lens") { m_Btype = RepeatedPlasmaLens; } m_dt = warpx.getdt(a_pti.GetLevel()); - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); m_ux = attribs[PIdx::ux].dataPtr() + a_offset; m_uy = attribs[PIdx::uy].dataPtr() + a_offset; m_uz = attribs[PIdx::uz].dataPtr() + a_offset; diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 8605868a87e..4cddfa5a342 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -41,14 +41,14 @@ struct ScaleFields { using namespace amrex::literals; - if (!m_do_scale) return; + if (!m_do_scale) { return; } // Scale the fields of particles about to cross the injection plane. // This only approximates what should be happening. The particles // should by advanced a fraction of a time step instead. // Scaling the fields is much easier and may be good enough. - const amrex::Real dtscale = m_dt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost); - if (0._rt < dtscale && dtscale < m_dt) + const amrex::Real dtscale = 1._rt - (m_z_plane_previous - zp)/(m_vz_ave_boosted + m_v_boost)/m_dt; + if (0._rt < dtscale && dtscale < 1._rt) { Exp *= dtscale; Eyp *= dtscale; diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index b722daa40fc..fac94ff20a3 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -10,6 +10,7 @@ #define WARPX_LaserParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "WarpXParticleContainer.H" @@ -55,11 +56,9 @@ public: * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer. */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/, - const int /*n_external_attr_real*/, - const int /*n_external_attr_int*/, - const amrex::RandomEngine& /*engine*/) final {} + typename ContainerLike::ParticleTileType& /*pinned_tile*/, + int /*n_external_attr_real*/, + int /*n_external_attr_int*/) final {} void ReadHeader (std::istream& is) final; @@ -74,7 +73,7 @@ public: const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, const amrex::MultiFab*, amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) final; + bool skip_deposition=false, PushType push_type=PushType::Explicit) final; void PushP (int lev, amrex::Real dt, const amrex::MultiFab& , diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 7d918d5dfb5..fd4e9397253 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -9,6 +9,7 @@ #include "LaserParticleContainer.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Laser/LaserProfiles.H" #include "Particles/LaserParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -279,7 +280,7 @@ LaserParticleContainer::LaserParticleContainer (AmrCore* amr_core, int ispecies, void LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) { - if (!m_enabled) return; + if (!m_enabled) { return; } // Input parameter injection_box contains small box where injection // should occur. @@ -321,7 +322,7 @@ LaserParticleContainer::ContinuousInjection (const RealBox& injection_box) void LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) { - if (!m_enabled) return; + if (!m_enabled) { return; } const int dir = WarpX::moving_window_dir; if (do_continuous_injection and (WarpX::gamma_boost > 1)){ @@ -350,7 +351,7 @@ LaserParticleContainer::UpdateAntennaPosition (const amrex::Real dt) void LaserParticleContainer::InitData () { - if (!m_enabled) return; + if (!m_enabled) { return; } // Call InitData on max level to inject one laser particle per // finest cell. @@ -367,7 +368,7 @@ LaserParticleContainer::InitData () void LaserParticleContainer::InitData (int lev) { - if (!m_enabled) return; + if (!m_enabled) { return; } // spacing of laser particles in the laser plane. // has to be done after geometry is set up. @@ -541,7 +542,7 @@ LaserParticleContainer::InitData (int lev) amrex::Vector particle_uy(np, 0.0); amrex::Vector particle_uz(np, 0.0); - if (Verbose()) amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); + if (Verbose()) { amrex::Print() << Utils::TextMsg::Info("Adding laser particles"); } amrex::Vector> attr; attr.push_back(particle_w); amrex::Vector> attr_int; @@ -561,12 +562,12 @@ LaserParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, const MultiFab*, - Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition) + Real t, Real dt, DtType /*a_dt_type*/, bool skip_deposition, PushType push_type) { WARPX_PROFILE("LaserParticleContainer::Evolve()"); WARPX_PROFILE_VAR_NS("LaserParticleContainer::Evolve::ParticlePush", blp_pp); - if (!m_enabled) return; + if (!m_enabled) { return; } Real t_lab = t; if (WarpX::gamma_boost > 1) { @@ -664,14 +665,14 @@ LaserParticleContainer::Evolve (int lev, // Deposit inside domains DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, &jx, &jy, &jz, 0, np_current, thread_num, - lev, lev, dt, relative_time); + lev, lev, dt, relative_time, push_type); if (has_buffer) { // Deposit in buffers DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, cjx, cjy, cjz, np_current, np-np_current, thread_num, - lev, lev-1, dt, relative_time); + lev, lev-1, dt, relative_time, push_type); } } @@ -701,7 +702,7 @@ LaserParticleContainer::Evolve (int lev, void LaserParticleContainer::PostRestart () { - if (!m_enabled) return; + if (!m_enabled) { return; } Real Sx, Sy; const int lev = finestLevel(); @@ -869,7 +870,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, np, [=] AMREX_GPU_DEVICE (int i) { // Calculate the velocity according to the amplitude of E - const Real sign_charge = (pwp[i]>0) ? 1 : -1; + const Real sign_charge = (pwp[i]>0) ? -1 : 1; const Real v_over_c = sign_charge * tmp_mobility * amplitude[i]; AMREX_ALWAYS_ASSERT_WITH_MESSAGE(amrex::Math::abs(v_over_c) < amrex::Real(1.), "Error: calculated laser particle velocity greater than c." diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index a45d9fd0dc0..9946a680fcd 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -14,6 +14,7 @@ #include "MultiParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Collision/CollisionHandler.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper_fwd.H" @@ -94,11 +95,11 @@ public: void InitMultiPhysicsModules (); - /// - /// This evolves all the particles by one PIC time step, including current deposition, the - /// field solve, and pushing the particles, for all the species in the MultiParticleContainer. - /// This is the electromagnetic version. - /// + /** + * \brief This evolves all the particles by one PIC time step, including current deposition, the + * field solve, and pushing the particles, for all the species in the MultiParticleContainer. + * This is the electromagnetic version. + */ void Evolve (int lev, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz, @@ -107,21 +108,21 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false); + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit); - /// - /// This pushes the particle positions by one half time step for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton - /// or when restarting from a checkpoint. - /// + /** + * \brief This pushes the particle positions by one time step for all the species in the + * MultiParticleContainer. + */ void PushX (amrex::Real dt); - /// - /// This pushes the particle momenta by dt for all the species in the - /// MultiParticleContainer. It is used to desynchronize the particles after initializaton - /// or when restarting from a checkpoint. It is also used to synchronize particles at the - /// the end of the run. This is the electromagnetic version. - /// + /** + * This pushes the particle momenta by dt for all the species in the + * MultiParticleContainer. It is used to desynchronize the particles after initialization + * or when restarting from a checkpoint. It is also used to synchronize particles at the + * the end of the run. This is the electromagnetic version. + */ void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, const amrex::MultiFab& Ez, const amrex::MultiFab& Bx, const amrex::MultiFab& By, const amrex::MultiFab& Bz); @@ -193,7 +194,7 @@ public: /** This function computes the box outside which Schwinger process is disabled. The box is * defined by m_qed_schwinger_xmin/xmax/ymin/ymax/zmin/zmax and the warpx level 0 geometry - * object (to make the link between Real and int quatities). + * object (to make the link between Real and int quantities). */ [[nodiscard]] amrex::Box ComputeSchwingerGlobalBox () const; #endif @@ -331,6 +332,9 @@ public: [[nodiscard]] int getSpeciesID (std::string product_str) const; + amrex::Vector>::iterator begin() {return allcontainers.begin();} + amrex::Vector>::iterator end() {return allcontainers.end();} + protected: #ifdef WARPX_QED diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 4eaf89c2aa4..a31f426a0e4 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -455,21 +455,22 @@ MultiParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { if (! skip_deposition) { jx.setVal(0.0); jy.setVal(0.0); jz.setVal(0.0); - if (cjx) cjx->setVal(0.0); - if (cjy) cjy->setVal(0.0); - if (cjz) cjz->setVal(0.0); - if (rho) rho->setVal(0.0); - if (crho) crho->setVal(0.0); + if (cjx) { cjx->setVal(0.0); } + if (cjy) { cjy->setVal(0.0); } + if (cjz) { cjz->setVal(0.0); } + if (rho) { rho->setVal(0.0); } + if (crho) { crho->setVal(0.0); } } for (auto& pc : allcontainers) { pc->Evolve(lev, Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, cjx, cjy, cjz, - rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition); + rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, t, dt, a_dt_type, skip_deposition, push_type); } } @@ -507,8 +508,9 @@ MultiParticleContainer::GetZeroChargeDensity (const int lev) const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } auto zero_rho = std::make_unique(nba, dmap, WarpX::ncomps, ng_rho); zero_rho->setVal(amrex::Real(0.0)); @@ -555,7 +557,7 @@ MultiParticleContainer::DepositCharge ( } // Push the particles in time, if needed - if (relative_time != 0.) PushX(relative_time); + if (relative_time != 0.) { PushX(relative_time); } bool const local = true; bool const reset = false; @@ -564,13 +566,13 @@ MultiParticleContainer::DepositCharge ( // Call the deposition kernel for each species for (auto& pc : allcontainers) { - if (pc->do_not_deposit) continue; + if (pc->do_not_deposit) { continue; } pc->DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } // Push the particles back in time - if (relative_time != 0.) PushX(-relative_time); + if (relative_time != 0.) { PushX(-relative_time); } #ifdef WARPX_DIM_RZ for (int lev = 0; lev < rho.size(); ++lev) @@ -586,7 +588,7 @@ MultiParticleContainer::GetChargeDensity (int lev, bool local) std::unique_ptr rho = GetZeroChargeDensity(lev); for (auto& container : allcontainers) { - if (container->do_not_deposit) continue; + if (container->do_not_deposit) { continue; } const std::unique_ptr rhoi = container->GetChargeDensity(lev, true); MultiFab::Add(*rho, *rhoi, 0, 0, rho->nComp(), rho->nGrowVect()); } @@ -710,7 +712,7 @@ MultiParticleContainer::SetParticleDistributionMap (int lev, DistributionMapping void MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->ContinuousInjection(injection_box); } @@ -720,7 +722,7 @@ MultiParticleContainer::ContinuousInjection (const RealBox& injection_box) const void MultiParticleContainer::UpdateAntennaPosition (const amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ pc->UpdateAntennaPosition(dt); } @@ -731,7 +733,7 @@ int MultiParticleContainer::doContinuousInjection () const { int warpx_do_continuous_injection = 0; - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ if (pc->do_continuous_injection){ warpx_do_continuous_injection = 1; } @@ -746,7 +748,7 @@ MultiParticleContainer::doContinuousInjection () const void MultiParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) const { - for (auto& pc : allcontainers){ + for (const auto& pc : allcontainers){ pc->ContinuousFluxInjection(t, dt); } } @@ -790,10 +792,10 @@ MultiParticleContainer::mapSpeciesProduct () #ifdef WARPX_QED if (m_do_qed_schwinger) { - m_qed_schwinger_ele_product = - getSpeciesID(m_qed_schwinger_ele_product_name); - m_qed_schwinger_pos_product = - getSpeciesID(m_qed_schwinger_pos_product_name); + m_qed_schwinger_ele_product = + getSpeciesID(m_qed_schwinger_ele_product_name); + m_qed_schwinger_pos_product = + getSpeciesID(m_qed_schwinger_pos_product_name); } #endif } @@ -870,7 +872,7 @@ MultiParticleContainer::doFieldIonization (int lev, auto& pc_product = allcontainers[pc_source->ionization_product]; const SmartCopyFactory copy_factory(*pc_source, *pc_product); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); auto Copy = copy_factory.getSmartCopy(); auto Transform = IonizationTransformFunc(); @@ -899,7 +901,7 @@ MultiParticleContainer::doFieldIonization (int lev, Bx[pti], By[pti], Bz[pti]); const auto np_dst = dst_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + const auto num_added = filterCopyTransformParticles<1>(*pc_product, dst_tile, src_tile, np_dst, Filter, Copy, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); @@ -976,11 +978,13 @@ void MultiParticleContainer::InitQED () } } - if(m_nspecies_quantum_sync != 0) + if(m_nspecies_quantum_sync != 0) { InitQuantumSync(); + } - if(m_nspecies_breit_wheeler !=0) + if(m_nspecies_breit_wheeler !=0) { InitBreitWheeler(); + } } @@ -1065,8 +1069,9 @@ void MultiParticleContainer::InitBreitWheeler () // considered for pair production. If a photon has chi < chi_min, // the optical depth is not evolved and photon generation is ignored amrex::Real bw_minimum_chi_part; - if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) + if(!utils::parser::queryWithParser(pp_qed_bw, "chi_min", bw_minimum_chi_part)) { WARPX_ABORT_WITH_MESSAGE("qed_bw.chi_min should be provided!"); + } pp_qed_bw.query("lookup_table_mode", lookup_table_mode); if(lookup_table_mode.empty()){ @@ -1383,7 +1388,7 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; - const auto num_added = filterCreateTransformFromFAB<1>( dst_ele_tile, + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, Transform); @@ -1488,7 +1493,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, // in pc_product_ele and positrons in pc_product_pos for (auto& pc_source : allcontainers){ - if(!pc_source->has_breit_wheeler()) continue; + if(!pc_source->has_breit_wheeler()) { continue; } // Get product species auto& pc_product_ele = @@ -1498,7 +1503,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const SmartCopyFactory copy_factory_ele(*pc_source, *pc_product_ele); const SmartCopyFactory copy_factory_pos(*pc_source, *pc_product_pos); - auto phys_pc_ptr = static_cast(pc_source.get()); + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPairGenerationFilterFunc(); const auto CopyEle = copy_factory_ele.getSmartCopy(); @@ -1536,7 +1541,7 @@ void MultiParticleContainer::doQedBreitWheeler (int lev, const auto np_dst_ele = dst_ele_tile.numParticles(); const auto np_dst_pos = dst_pos_tile.numParticles(); - const auto num_added = filterCopyTransformParticles<1>( + const auto num_added = filterCopyTransformParticles<1>(*pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, src_tile, np_dst_ele, np_dst_pos, Filter, CopyEle, CopyPos, Transform); @@ -1578,7 +1583,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, allcontainers[pc_source->m_qed_quantum_sync_phot_product]; const SmartCopyFactory copy_factory_phot(*pc_source, *pc_product_phot); - auto phys_pc_ptr = + auto *phys_pc_ptr = static_cast(pc_source.get()); const auto Filter = phys_pc_ptr->getPhotonEmissionFilterFunc(); @@ -1616,7 +1621,7 @@ void MultiParticleContainer::doQedQuantumSync (int lev, const auto np_dst = dst_tile.numParticles(); const auto num_added = - filterCopyTransformParticles<1>(dst_tile, src_tile, np_dst, + filterCopyTransformParticles<1>(*pc_product_phot, dst_tile, src_tile, np_dst, Filter, CopyPhot, Transform); setNewParticleIDs(dst_tile, np_dst, num_added); diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index c9f66e6c2b0..e7a7a20fad5 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -18,24 +18,39 @@ #include -/** Particle Attributes stored in amrex::ParticleContainer's struct of array +/** Real Particle Attributes stored in amrex::ParticleContainer's struct of array */ struct PIdx { enum { - w = 0, ///< weight +#if !defined (WARPX_DIM_1D_Z) + x, +#endif +#if defined (WARPX_DIM_3D) + y, +#endif + z, + w, ///< weight ux, uy, uz, #ifdef WARPX_DIM_RZ theta, ///< RZ needs all three position components #endif - nattribs ///< number of attributes + nattribs ///< number of compile-time attributes + }; +}; + +/** Integer Particle Attributes stored in amrex::ParticleContainer's struct of array + */ +struct PIdxInt +{ + enum { + nattribs ///< number of compile-time attributes }; }; /** Particle Container class that allows to add/access particle components * with a name (string) instead of doing so with an integer index. - * (The "components" are all the particle quantities - except those - * that are stored in an AoS by amrex, i.e. the particle positions and ID) + * (The "components" are all the particle amrex::Real quantities.) * * This is done by storing maps that give the index of the component * that corresponds to a given string. @@ -45,11 +60,11 @@ struct PIdx */ template class T_Allocator=amrex::DefaultAllocator> class NamedComponentParticleContainer : -public amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> +public amrex::ParticleContainerPureSoA { public: /** Construct an empty NamedComponentParticleContainer **/ - NamedComponentParticleContainer () : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>() {} + NamedComponentParticleContainer () : amrex::ParticleContainerPureSoA() {} /** Construct a NamedComponentParticleContainer from an AmrParGDB object * @@ -61,8 +76,15 @@ public: * AMR hierarchy. Usually, this is generated by an AmrCore or AmrLevel object. */ NamedComponentParticleContainer (amrex::AmrParGDB* amr_pgdb) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(amr_pgdb) { + : amrex::ParticleContainerPureSoA(amr_pgdb) { // build up the map of string names to particle component numbers +#if !defined (WARPX_DIM_1D_Z) + particle_comps["x"] = PIdx::x; +#endif +#if defined (WARPX_DIM_3D) + particle_comps["y"] = PIdx::y; +#endif + particle_comps["z"] = PIdx::z; particle_comps["w"] = PIdx::w; particle_comps["ux"] = PIdx::ux; particle_comps["uy"] = PIdx::uy; @@ -85,12 +107,12 @@ public: * @param p_ricomps name-to-index map for run-time integer components */ NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator> && pc, + amrex::ParticleContainerPureSoA && pc, std::map p_comps, std::map p_icomps, std::map p_rcomps, std::map p_ricomps) - : amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>(std::move(pc)), + : amrex::ParticleContainerPureSoA(std::move(pc)), particle_comps(std::move(p_comps)), particle_icomps(std::move(p_icomps)), particle_runtime_comps(std::move(p_rcomps)), @@ -102,9 +124,9 @@ public: NamedComponentParticleContainer& operator= ( const NamedComponentParticleContainer & ) = delete; /** Move constructor for NamedComponentParticleContainer */ - NamedComponentParticleContainer ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer ( NamedComponentParticleContainer && ) noexcept = default; /** Move operator for NamedComponentParticleContainer */ - NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) = default; + NamedComponentParticleContainer& operator= ( NamedComponentParticleContainer && ) noexcept = default; /** Create an empty particle container * @@ -118,7 +140,7 @@ public: NamedComponentParticleContainer make_alike () const { auto tmp = NamedComponentParticleContainer( - amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::template make_alike(), + amrex::ParticleContainerPureSoA::template make_alike(), particle_comps, particle_icomps, particle_runtime_comps, @@ -127,10 +149,10 @@ public: return tmp; } - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumRealComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::NumIntComps; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddRealComp; - using amrex::ParticleContainer<0,0,PIdx::nattribs,0,T_Allocator>::AddIntComp; + using amrex::ParticleContainerPureSoA::NumRealComps; + using amrex::ParticleContainerPureSoA::NumIntComps; + using amrex::ParticleContainerPureSoA::AddRealComp; + using amrex::ParticleContainerPureSoA::AddIntComp; /** Allocate a new run-time real component * diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 1f4fb0372eb..78b090056b1 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -27,7 +27,7 @@ struct ParticleBoundaries void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); - bool CheckAll (ParticleBoundaryType bc); + [[nodiscard]] bool CheckAll (ParticleBoundaryType bc) const; void BuildReflectionModelParsers (); diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index a6e80717e81..5b51fa3fd25 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -54,7 +54,7 @@ ParticleBoundaries::SetBoundsZ (ParticleBoundaryType bc_lo, ParticleBoundaryType } bool -ParticleBoundaries::CheckAll (ParticleBoundaryType bc) +ParticleBoundaries::CheckAll (ParticleBoundaryType bc) const { return (data.xmin_bc == bc && data.xmax_bc == bc #ifdef WARPX_DIM_3D diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 662e1240f8b..bbec1f54e01 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -140,10 +140,10 @@ namespace ApplyParticleBoundaries { uy = ur*std::sin(y) + ut*std::cos(y); } #else - if (change_sign_ux) ux = -ux; - if (change_sign_uy) uy = -uy; + if (change_sign_ux) { ux = -ux; } + if (change_sign_uy) { uy = -uy; } #endif - if (change_sign_uz) uz = -uz; + if (change_sign_uz) { uz = -uz; } } } diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index b6eba51347c..f991f211d28 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -11,6 +11,8 @@ #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" +#include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/Pusher/UpdatePosition.H" #include @@ -19,6 +21,8 @@ #include #include #include +#include +using namespace amrex::literals; struct IsOutsideDomainBoundary { amrex::GpuArray m_plo; @@ -41,26 +45,149 @@ struct IsOutsideDomainBoundary { } }; +#ifdef AMREX_USE_EB +struct FindEmbeddedBoundaryIntersection { + const int m_step_index; + const int m_delta_index; + const int m_normal_index; + const int m_step; + const amrex::Real m_dt; + amrex::Array4 m_phiarr; + amrex::GpuArray m_dxi; + amrex::GpuArray m_plo; + + template + AMREX_GPU_HOST_DEVICE + void operator() (const DstData& dst, const SrcData& src, + int src_i, int dst_i) const noexcept + { + // Copy all particle attributes, from the source to the destination + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; + for (int j = 0; j < SrcData::NAR; ++j) { + dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { + dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { + dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; + } + + // Modify the position of the destination particle: + // Move it to the point of intersection with the embedded boundary + // (which is found by using a bisection algorithm) + + const auto& p = dst.getSuperParticle(dst_i); + amrex::ParticleReal xp, yp, zp; + get_particle_position( p, xp, yp, zp ); + amrex::ParticleReal const ux = dst.m_rdata[PIdx::ux][dst_i]; + amrex::ParticleReal const uy = dst.m_rdata[PIdx::uy][dst_i]; + amrex::ParticleReal const uz = dst.m_rdata[PIdx::uz][dst_i]; + + // Temporary variables to avoid implicit capture + amrex::Real dt = m_dt; + amrex::Array4 phiarr = m_phiarr; + amrex::GpuArray dxi = m_dxi; + amrex::GpuArray plo = m_plo; + + // Bisection algorithm to find the point where phi(x,y,z)=0 (i.e. on the embedded boundary) + amrex::Real dt_fraction = amrex::bisect( 0.0, 1.0, + [=] (amrex::Real dt_frac) { + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; + UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); + return phi_value; + } ); + + // Also record the real time on the destination + dst.m_runtime_idata[m_step_index][dst_i] = m_step; + dst.m_runtime_rdata[m_delta_index][dst_i] = (1._rt- dt_fraction)*m_dt; + + // Now that dt_fraction has be obtained (with bisect) + // Save the corresponding position of the particle at the boundary + amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; + UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_fraction*m_dt); + + // record the components of the normal on the destination + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + int ic, jc, kc; // Cell-centered indices + int nodal; + amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weight + ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc, nodal=0); // nodal=0 to calculate the weights with respect to the cell-centered nodes + amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phiarr, dxi); + DistanceToEB::normalize(normal); + +#if (defined WARPX_DIM_3D) + dst.m_rdata[PIdx::x][dst_i] = x_temp; + dst.m_rdata[PIdx::y][dst_i] = y_temp; + dst.m_rdata[PIdx::z][dst_i] = z_temp; + //save normal components + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[1]; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[2]; +#elif (defined WARPX_DIM_XZ) + dst.m_rdata[PIdx::x][dst_i] = x_temp; + dst.m_rdata[PIdx::z][dst_i] = z_temp; + amrex::ignore_unused(y_temp); + //save normal components + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; +#elif (defined WARPX_DIM_RZ) + dst.m_rdata[PIdx::x][dst_i] = std::sqrt(x_temp*x_temp + y_temp*y_temp); + dst.m_rdata[PIdx::z][dst_i] = z_temp; + dst.m_rdata[PIdx::theta][dst_i] = std::atan2(y_temp, x_temp); + //save normal components + amrex::Real theta=std::atan2(y_temp, x_temp); + dst.m_runtime_rdata[m_normal_index][dst_i] = normal[0]*std::cos(theta); + dst.m_runtime_rdata[m_normal_index+1][dst_i] = normal[0]*std::sin(theta); + dst.m_runtime_rdata[m_normal_index+2][dst_i] = normal[1]; +#elif (defined WARPX_DIM_1D_Z) + dst.m_rdata[PIdx::z][dst_i] = z_temp; + amrex::ignore_unused(x_temp, y_temp); + //normal not defined + dst.m_runtime_rdata[m_normal_index][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+1][dst_i] = 0.0; + dst.m_runtime_rdata[m_normal_index+2][dst_i] = 0.0; +#else + amrex::ignore_unused(x_temp, y_temp, z_temp,normal); +#endif + } +}; +#endif + struct CopyAndTimestamp { - int m_index; + int m_step_index; + int m_delta_index; int m_step; + const amrex::Real m_dt; template AMREX_GPU_HOST_DEVICE void operator() (const DstData& dst, const SrcData& src, int src_i, int dst_i) const noexcept { - dst.m_aos[dst_i] = src.m_aos[src_i]; - for (int j = 0; j < SrcData::NAR; ++j) + dst.m_idcpu[dst_i] = src.m_idcpu[src_i]; + for (int j = 0; j < SrcData::NAR; ++j) { dst.m_rdata[j][dst_i] = src.m_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_real; ++j) + } + for (int j = 0; j < src.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][dst_i] = src.m_runtime_rdata[j][src_i]; - for (int j = 0; j < src.m_num_runtime_int; ++j) + } + for (int j = 0; j < src.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][dst_i] = src.m_runtime_idata[j][src_i]; - dst.m_runtime_idata[m_index][dst_i] = m_step; + } + dst.m_runtime_idata[m_step_index][dst_i] = m_step; + dst.m_runtime_rdata[m_delta_index][dst_i] = 0._rt; //delta_fraction is initialized to zero } }; + ParticleBoundaryBuffer::ParticleBoundaryBuffer () { m_particle_containers.resize(numBoundaries()); @@ -115,7 +242,7 @@ ParticleBoundaryBuffer::ParticleBoundaryBuffer () #endif // Set the flag whether the boundary is active or any species for (int i = 0; i < numBoundaries(); ++i) { - if (m_do_boundary_buffer[i][ispecies]) m_do_any_boundary[i] = 1; + if (m_do_boundary_buffer[i][ispecies]) { m_do_any_boundary[i] = 1; } } } @@ -146,7 +273,7 @@ void ParticleBoundaryBuffer::printNumParticles () const { { for (int iside = 0; iside < 2; ++iside) { - auto& buffer = m_particle_containers[2*idim+iside]; + const auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { const auto np = buffer[i].isDefined() ? buffer[i].TotalNumberOfParticles(false) : 0; @@ -210,7 +337,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { for (int ispecies = 0; ispecies < numSpecies(); ++ispecies) { auto& species_buffer = buffer[ispecies]; - if (species_buffer.isDefined()) species_buffer.clearParticles(); + if (species_buffer.isDefined()) { species_buffer.clearParticles(); } } } @@ -219,7 +346,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); - using PIter = amrex::ParConstIter<0,0,PIdx::nattribs>; + using PIter = amrex::ParConstIterSoA; const auto& warpx_instance = WarpX::GetInstance(); const amrex::Geometry& geom = warpx_instance.Geom(0); auto plo = geom.ProbLoArray(); @@ -227,18 +354,19 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (geom.isPeriodic(idim)) continue; + if (geom.isPeriodic(idim)) { continue; } for (int iside = 0; iside < 2; ++iside) { auto& buffer = m_particle_containers[2*idim+iside]; for (int i = 0; i < numSpecies(); ++i) { - if (!m_do_boundary_buffer[2*idim+iside][i]) continue; + if (!m_do_boundary_buffer[2*idim+iside][i]) { continue; } const WarpXParticleContainer& pc = mypc.GetParticleContainer(i); if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("timestamp", false); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -250,13 +378,13 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, for(PIter pti(pc, lev); pti.isValid(); ++pti) { auto index = std::make_pair(pti.index(), pti.LocalTileIndex()); - if(plevel.find(index) == plevel.end()) continue; + if(plevel.find(index) == plevel.end()) { continue; } auto& ptile_buffer = species_buffer.DefineAndReturnParticleTile( lev, pti.index(), pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } auto predicate = IsOutsideDomainBoundary{plo, phi, idim, iside}; @@ -278,12 +406,17 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterAndTransform"); - const int timestamp_index = ptile_buffer.NumRuntimeIntComps()-1; - const int timestep = warpx_instance.getistep(0); + auto& warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); + const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); + auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); + const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); + const int step = warpx_instance.getistep(0); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - CopyAndTimestamp{timestamp_index, timestep}, + CopyAndTimestamp{step_scraped_index, delta_index, step, dt}, 0, dst_index); } } @@ -303,7 +436,12 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, if (!buffer[i].isDefined()) { buffer[i] = pc.make_alike(); - buffer[i].AddIntComp("timestamp", false); + buffer[i].AddIntComp("stepScraped", false); + buffer[i].AddRealComp("deltaTimeScraped", false); + buffer[i].AddRealComp("nx", false); + buffer[i].AddRealComp("ny", false); + buffer[i].AddRealComp("nz", false); + } auto& species_buffer = buffer[i]; for (int lev = 0; lev < pc.numLevels(); ++lev) @@ -324,7 +462,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, pti.LocalTileIndex()); const auto& ptile = plevel.at(index); auto np = ptile.numParticles(); - if (np == 0) continue; + if (np == 0) { continue; } using SrcData = WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType; auto predicate = [=] AMREX_GPU_HOST_DEVICE (const SrcData& /*src*/, const int ip) @@ -354,13 +492,20 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::resize_eb"); ptile_buffer.resize(dst_index + amrex::get<0>(reduce_data.value())); } + auto& warpx = WarpX::GetInstance(); + const auto dt = warpx.getdt(pti.GetLevel()); + auto string_to_index_intcomp = buffer[i].getParticleRuntimeiComps(); + const int step_scraped_index = string_to_index_intcomp.at("stepScraped"); + auto string_to_index_realcomp = buffer[i].getParticleRuntimeComps(); + const int delta_index = string_to_index_realcomp.at("deltaTimeScraped"); + const int normal_index = string_to_index_realcomp.at("nx"); + const int step = warpx_instance.getistep(0); - const int timestamp_index = ptile_buffer.NumRuntimeIntComps()-1; - const int timestep = warpx_instance.getistep(0); { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::filterTransformEB"); amrex::filterAndTransformParticles(ptile_buffer, ptile, predicate, - CopyAndTimestamp{timestamp_index, timestep}, 0, dst_index); + FindEmbeddedBoundaryIntersection{step_scraped_index,delta_index, normal_index, step, dt, phiarr, dxi, plo}, 0, dst_index); + } } } diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 09ff5cce6f0..870fc82bd0f 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -8,6 +8,12 @@ #ifndef DEFAULTINITIALIZATION_H_ #define DEFAULTINITIALIZATION_H_ +#include +#ifdef WARPX_QED +# include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" +# include "Particles/ElementaryProcess/QEDInternals/QuantumSyncEngineWrapper.H" +#endif + #include #include @@ -81,4 +87,197 @@ int initializeIntValue (const InitializationPolicy policy) noexcept } } +namespace ParticleCreation { + + /** + * \brief Default initialize runtime attributes in a tile. This routine does not initialize the + * first n_external_attr_real real attributes and the first n_external_attr_int integer + * attributes, which have been in principle externally set elsewhere. + * + * @tparam[in] The type of the particle tile to operate on (e.g. could use different allocators) + * @param[inout] ptile the tile in which attributes are initialized + * @param[in] n_external_attr_real The number of real attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] n_external_attr_int The number of integer attributes that have been externally set. + * These are NOT initialized by this function. + * @param[in] user_real_attribs The names of the real components for this particle tile + * @param[in] user_int_attribs The names of the int components for this particle tile + * @param[in] particle_comps map between particle component index and component name for real comps + * @param[in] particle_icomps map between particle component index and component name for int comps + * @param[in] user_real_attrib_parser the parser functions used to initialize the user real components + * @param[in] user_int_attrib_parser the parser functions used to initialize the user int components + * @param[in] do_qed_comps whether to initialize the qed components (these are usually handled by + * SmartCopy, but NOT when adding particles in AddNParticles) + * @param[in] p_bw_engine the engine to use for setting the breit-wheeler component for QED + * @param[in] p_qs_engine the engine to use for setting the quantum synchrotron component for QED + * @param[in] ionization_initial_level the ionization level particles created should start at + * @param[in] start the index to start initializing particles + * @param[in] stop the index to stop initializing particles + */ +template +void DefaultInitializeRuntimeAttributes (PTile& ptile, + const int n_external_attr_real, + const int n_external_attr_int, + const std::vector& user_real_attribs, + const std::vector& user_int_attribs, + const std::map& particle_comps, + const std::map& particle_icomps, + const std::vector& user_real_attrib_parser, + const std::vector& user_int_attrib_parser, +#ifdef WARPX_QED + const bool do_qed_comps, + BreitWheelerEngine* p_bw_engine, + QuantumSynchrotronEngine* p_qs_engine, +#endif + const int ionization_initial_level, + int start, int stop) +{ + using namespace amrex::literals; + + // Preparing data needed for user defined attributes + const auto n_user_real_attribs = static_cast(user_real_attribs.size()); + const auto n_user_int_attribs = static_cast(user_int_attribs.size()); + const auto get_position = GetParticlePosition(ptile); + const auto soa = ptile.getParticleTileData(); + const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; + const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; + const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; + constexpr int lev = 0; + const amrex::Real t = WarpX::GetInstance().gett_new(lev); + + // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes + for (int j = PIdx::nattribs + n_external_attr_real; j < ptile.NumRealComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetRealData(j).data(); +#ifdef WARPX_QED + // Current runtime comp is quantum synchrotron optical depth + if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && + particle_comps.at("opticalDepthQSR") == j) + { + if (!do_qed_comps) { continue; } + const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = + p_qs_engine->build_optical_depth_functor(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = quantum_sync_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = quantum_sync_get_opt(amrex::RandomEngine{}); + } + } + } + + // Current runtime comp is Breit-Wheeler optical depth + if (particle_comps.find("opticalDepthBW") != particle_comps.end() && + particle_comps.at("opticalDepthBW") == j) + { + if (!do_qed_comps) { continue; } + const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = + p_bw_engine->build_optical_depth_functor();; + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelForRNG(stop - start, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept { + int ip = i + start; + attr_ptr[ip] = breit_wheeler_get_opt(engine); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = breit_wheeler_get_opt(amrex::RandomEngine{}); + } + } + } +#endif + + for (int ia = 0; ia < n_user_real_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_comps.find(user_real_attribs[ia]) != particle_comps.end() && + particle_comps.at(user_real_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_real_attrib_parserexec = + user_real_attrib_parser[ia]->compile<7>(); + // If the particle tile was allocated in a memory pool that can run on GPU, launch GPU kernel + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + }); + // Otherwise (e.g. particle tile allocated in pinned memory), run on CPU + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = user_real_attrib_parserexec(xp, yp, zp, + ux[ip], uy[ip], uz[ip], t); + } + } + } + } + } + + // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes + for (int j = n_external_attr_int; j < ptile.NumIntComps() ; ++j) + { + auto attr_ptr = ptile.GetStructOfArrays().GetIntData(j).data(); + + // Current runtime comp is ionization level + if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && + particle_icomps.at("ionizationLevel") == j) + { + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + attr_ptr[ip] = ionization_initial_level; + }); + } else { + for (int ip = start; ip < stop; ++ip) { + attr_ptr[ip] = ionization_initial_level; + } + } + } + + for (int ia = 0; ia < n_user_int_attribs; ++ia) + { + // Current runtime comp is ia-th user defined attribute + if (particle_icomps.find(user_int_attribs[ia]) != particle_icomps.end() && + particle_icomps.at(user_int_attribs[ia]) == j) + { + const amrex::ParserExecutor<7> user_int_attrib_parserexec = + user_int_attrib_parser[ia]->compile<7>(); + if constexpr (amrex::RunOnGpu>::value) { + amrex::ParallelFor(stop - start, + [=] AMREX_GPU_DEVICE (int i) noexcept { + int ip = i + start; + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + }); + } else { + for (int ip = start; ip < stop; ++ip) { + amrex::ParticleReal xp, yp, zp; + get_position(ip, xp, yp, zp); + attr_ptr[ip] = static_cast( + user_int_attrib_parserexec(xp, yp, zp, ux[ip], uy[ip], uz[ip], t)); + } + } + } + } + } +} + +} + #endif diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 7a85f59318f..095ebbf6a85 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -8,6 +8,8 @@ #ifndef FILTER_COPY_TRANSFORM_H_ #define FILTER_COPY_TRANSFORM_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" + #include #include @@ -47,23 +49,26 @@ * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, + Index* mask, Index dst_index, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst.resize(std::max(dst_index + num_added, dst.numParticles())); + auto old_np = dst.size(); + auto new_np = std::max(dst_index + num_added, dst.numParticles()); + dst.resize(new_np); - const auto p_offsets = offsets.dataPtr(); + auto *const p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst_data = dst.getParticleTileData(); @@ -80,6 +85,21 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst, + 0, 0, + pc.getUserRealAttribs(), pc.getUserIntAttribs(), + pc.getParticleComps(), pc.getParticleiComps(), + pc.getUserRealAttribParser(), + pc.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc.get_breit_wheeler_engine_ptr(), + pc.get_quantum_sync_engine_ptr(), +#endif + pc.getIonizationInitialLevel(), + old_np, new_np); + Gpu::synchronize(); return num_added; } @@ -121,19 +141,19 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index* mask, Ind * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, +Index filterCopyTransformParticles (DstPC& pc, DstTile& dst, SrcTile& src, Index dst_index, PredFunc&& filter, CopyFunc&& copy, TransFunc&& transform) noexcept { using namespace amrex; const auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -142,9 +162,9 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst, src, mask.dataPtr(), dst_index, - std::forward(copy), - std::forward(transform)); + return filterCopyTransformParticles(pc, dst, src, mask.dataPtr(), dst_index, + std::forward(copy), + std::forward(transform)); } /** @@ -188,10 +208,10 @@ Index filterCopyTransformParticles (DstTile& dst, SrcTile& src, Index dst_index, * * \return num_added the number of particles that were written to dst. */ -template ::value, int> foo = 0> -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index* mask, Index dst1_index, Index dst2_index, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -199,15 +219,20 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector offsets(np); auto total = amrex::Scan::ExclusiveSum(np, mask, offsets.data()); const Index num_added = N * total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto src_data = src.getParticleTileData(); const auto dst1_data = dst1.getParticleTileData(); @@ -230,6 +255,35 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CopyFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -276,9 +330,9 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, * * \return num_added the number of particles that were written to dst. */ -template -Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, +Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, SrcTile& src, Index dst1_index, Index dst2_index, PredFunc&& filter, CopyFunc1&& copy1, CopyFunc2&& copy2, TransFunc&& transform) noexcept @@ -286,11 +340,11 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, using namespace amrex; auto np = src.numParticles(); - if (np == 0) return 0; + if (np == 0) { return 0; } Gpu::DeviceVector mask(np); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); const auto src_data = src.getParticleTileData(); amrex::ParallelForRNG(np, @@ -299,7 +353,7 @@ Index filterCopyTransformParticles (DstTile& dst1, DstTile& dst2, SrcTile& src, p_mask[i] = filter(src_data, i, engine); }); - return filterCopyTransformParticles(dst1, dst2, src, mask.dataPtr(), + return filterCopyTransformParticles(pc1, pc2, dst1, dst2, src, mask.dataPtr(), dst1_index, dst2_index, std::forward(copy1), std::forward(copy2), diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 38dc1d6daf2..2a4c9fccad0 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -8,6 +8,7 @@ #ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" #include @@ -42,19 +43,20 @@ * * \return num_added the number of particles that were written to dst1 and dst2. */ -template ::value, int> foo = 0> -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, - const FAB *src_FAB, const Index* mask, - const Index dst1_index, const Index dst2_index, - CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, + DstTile& dst1, DstTile& dst2, const amrex::Box box, + const FAB *src_FAB, const Index* mask, + const Index dst1_index, const Index dst2_index, + CreateFunc1&& create1, CreateFunc2&& create2, + TransFunc&& transform) noexcept { using namespace amrex; const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } auto & warpx = WarpX::GetInstance(); const int level_0 = 0; @@ -83,10 +85,15 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B Gpu::DeviceVector offsets(ncells); auto total = amrex::Scan::ExclusiveSum(ncells, mask, offsets.data()); const Index num_added = N*total; - dst1.resize(std::max(dst1_index + num_added, dst1.numParticles())); - dst2.resize(std::max(dst2_index + num_added, dst2.numParticles())); + auto old_np1 = dst1.size(); + auto new_np1 = std::max(dst1_index + num_added, dst1.numParticles()); + dst1.resize(new_np1); + + auto old_np2 = dst2.size(); + auto new_np2 = std::max(dst2_index + num_added, dst2.numParticles()); + dst2.resize(new_np2); - auto p_offsets = offsets.dataPtr(); + auto *p_offsets = offsets.dataPtr(); const auto dst1_data = dst1.getParticleTileData(); const auto dst2_data = dst2.getParticleTileData(); @@ -130,6 +137,35 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B } }); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst1, + 0, 0, + pc1.getUserRealAttribs(), pc1.getUserIntAttribs(), + pc1.getParticleComps(), pc1.getParticleiComps(), + pc1.getUserRealAttribParser(), + pc1.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc1.get_breit_wheeler_engine_ptr(), + pc1.get_quantum_sync_engine_ptr(), +#endif + pc1.getIonizationInitialLevel(), + old_np1, new_np1); + ParticleCreation::DefaultInitializeRuntimeAttributes(dst2, + 0, 0, + pc2.getUserRealAttribs(), pc2.getUserIntAttribs(), + pc2.getParticleComps(), pc2.getParticleiComps(), + pc2.getUserRealAttribParser(), + pc2.getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the CreateFunc functor + pc2.get_breit_wheeler_engine_ptr(), + pc2.get_quantum_sync_engine_ptr(), +#endif + pc2.getIonizationInitialLevel(), + old_np2, new_np2); + Gpu::synchronize(); return num_added; } @@ -166,10 +202,10 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B * * \return num_added the number of particles that were written to dst1 and dst2. */ -template -Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::Box box, +Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTile& dst2, const amrex::Box box, const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, @@ -184,11 +220,11 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B auto arrNumPartCreation = NumPartCreation.array(); const auto ncells = box.volume(); - if (ncells == 0) return 0; + if (ncells == 0) { return 0; } Gpu::DeviceVector mask(ncells); - auto p_mask = mask.dataPtr(); + auto *p_mask = mask.dataPtr(); // for loop over all cells in the box. We apply the filter function to each cell // and store the result in arrNumPartCreation. If the result is strictly greater than @@ -201,7 +237,7 @@ Index filterCreateTransformFromFAB (DstTile& dst1, DstTile& dst2, const amrex::B p_mask[mask_position] = (arrNumPartCreation(i,j,k) > 0); }); - return filterCreateTransformFromFAB(dst1, dst2, box, &NumPartCreation, + return filterCreateTransformFromFAB(pc1, pc2, dst1, dst2, box, &NumPartCreation, mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index da5637a7919..6a6ceb3d290 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -26,7 +26,7 @@ * type. Second, if a given component name is found in both the src * and the dst, then the src value is copied. * - * Particle structs - positions and id numbers - are always copied. + * Particle positions and id numbers are always copied. * * You don't create this directly - use the SmartCopyFactory object below. */ @@ -48,20 +48,21 @@ struct SmartCopy void operator() (DstData& dst, const SrcData& src, int i_src, int i_dst, amrex::RandomEngine const& engine) const noexcept { - // the particle struct is always copied over - dst.m_aos[i_dst] = src.m_aos[i_src]; - // initialize the real components - for (int j = 0; j < DstData::NAR; ++j) + for (int j = 0; j < DstData::NAR; ++j) { dst.m_rdata[j][i_dst] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < dst.m_num_runtime_real; ++j) + } + for (int j = 0; j < dst.m_num_runtime_real; ++j) { dst.m_runtime_rdata[j][i_dst] = initializeRealValue(m_policy_real[j+DstData::NAR], engine); + } // initialize the int components - for (int j = 0; j < DstData::NAI; ++j) + for (int j = 0; j < DstData::NAI; ++j) { dst.m_idata[j][i_dst] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < dst.m_num_runtime_int; ++j) + } + for (int j = 0; j < dst.m_num_runtime_int; ++j) { dst.m_runtime_idata[j][i_dst] = initializeIntValue(m_policy_int[j+DstData::NAI]); + } // copy the shared real components for (int j = 0; j < m_num_copy_real; ++j) diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index 1be3bd79cf3..b4f25d5daad 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -14,6 +14,8 @@ #include #include #include +#include +#include /** * \brief This is a functor for performing a "smart create" that works @@ -47,32 +49,35 @@ struct SmartCreate const int id = 0) const noexcept { #if defined(WARPX_DIM_3D) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = y; - prt.m_aos[i_prt].pos(2) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::y][i_prt] = y; + prt.m_rdata[PIdx::z][i_prt] = z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - prt.m_aos[i_prt].pos(0) = x; - prt.m_aos[i_prt].pos(1) = z; + prt.m_rdata[PIdx::x][i_prt] = x; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(y); #else - prt.m_aos[i_prt].pos(0) = z; + prt.m_rdata[PIdx::z][i_prt] = z; amrex::ignore_unused(x,y); #endif - prt.m_aos[i_prt].cpu() = cpu; - prt.m_aos[i_prt].id() = id; + prt.m_idcpu[i_prt] = amrex::SetParticleIDandCPU(id, cpu); - // initialize the real components - for (int j = 0; j < PartData::NAR; ++j) + // initialize the real components after position + for (int j = AMREX_SPACEDIM; j < PartData::NAR; ++j) { prt.m_rdata[j][i_prt] = initializeRealValue(m_policy_real[j], engine); - for (int j = 0; j < prt.m_num_runtime_real; ++j) + } + for (int j = 0; j < prt.m_num_runtime_real; ++j) { prt.m_runtime_rdata[j][i_prt] = initializeRealValue(m_policy_real[j+PartData::NAR], engine); + } // initialize the int components - for (int j = 0; j < PartData::NAI; ++j) + for (int j = 0; j < PartData::NAI; ++j) { prt.m_idata[j][i_prt] = initializeIntValue(m_policy_int[j]); - for (int j = 0; j < prt.m_num_runtime_int; ++j) + } + for (int j = 0; j < prt.m_num_runtime_int; ++j) { prt.m_runtime_idata[j][i_prt] = initializeIntValue(m_policy_int[j+PartData::NAI]); + } } }; diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index 732a12bb729..dbac563ca28 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -48,7 +49,7 @@ SmartCopyTag getSmartCopyTag (const NameMap& src, const NameMap& dst) noexcept; * \param num_added the number of particles to set the ids for. */ template -void setNewParticleIDs (PTile& ptile, int old_size, int num_added) +void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_added) { amrex::Long pid; #ifdef AMREX_USE_OMP @@ -60,12 +61,12 @@ void setNewParticleIDs (PTile& ptile, int old_size, int num_added) } const int cpuid = amrex::ParallelDescriptor::MyProc(); - auto pp = ptile.GetArrayOfStructs()().data() + old_size; + auto ptd = ptile.getParticleTileData(); amrex::ParallelFor(num_added, [=] AMREX_GPU_DEVICE (int ip) noexcept { - auto& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + auto const lip = static_cast(ip); + auto const new_id = lip + old_size; + ptd.m_idcpu[new_id] = amrex::SetParticleIDandCPU(pid+lip, cpuid); }); } diff --git a/Source/Particles/PhotonParticleContainer.H b/Source/Particles/PhotonParticleContainer.H index d4498785e49..34afac53482 100644 --- a/Source/Particles/PhotonParticleContainer.H +++ b/Source/Particles/PhotonParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_PhotonParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -25,7 +26,7 @@ /** * Photon particles have no mass, they deposit no charge, and see specific QED * effects. For these reasons, they are stored in the separate particle - * container PhotonParticleContainer, that inherts from + * container PhotonParticleContainer, that inherits from * PhysicalParticleContainer. The particle pusher and current deposition, in * particular, are overriden in this container. */ @@ -69,7 +70,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX(WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -103,7 +105,7 @@ public: amrex::MultiFab* /*rho*/, int /*icomp*/, const long /*offset*/, - const long /*np_to_depose*/, + const long /*np_to_deposit*/, int /*thread_num*/, int /*lev*/, int /*depos_lev*/) override {} @@ -119,12 +121,13 @@ public: amrex::MultiFab * const /*jy*/, amrex::MultiFab * const /*jz*/, long const /*offset*/, - long const /*np_to_depose*/, + long const /*np_to_deposit*/, int const /*thread_num*/, int const /*lev*/, int const /*depos_lev*/, amrex::Real const /*dt*/, - amrex::Real const /*relative_time*/) override {} + amrex::Real const /*relative_time*/, + PushType /*push_type*/) override {} }; #endif // #ifndef WARPX_PhotonParticleContainer_H_ diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index 96091b9067f..aa9f04be224 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -185,7 +185,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, np_to_push, [=] AMREX_GPU_DEVICE (long i, auto exteb_control, auto qed_control) { - if (do_copy) copyAttribs(i); + if (do_copy) { copyAttribs(i); } ParticleReal x, y, z; GetPosition(i, x, y, z); @@ -205,17 +205,17 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; // workaround for nvcc if constexpr (exteb_control == has_exteb) { getExternalEB(i, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } #ifdef WARPX_QED - [[maybe_unused]] auto& evolve_opt_tmp = evolve_opt; - [[maybe_unused]] auto p_optical_depth_BW_tmp = p_optical_depth_BW; - [[maybe_unused]] auto ux_tmp = ux; // for nvhpc - [[maybe_unused]] auto uy_tmp = uy; - [[maybe_unused]] auto uz_tmp = uz; + [[maybe_unused]] const auto& evolve_opt_tmp = evolve_opt; + [[maybe_unused]] auto *p_optical_depth_BW_tmp = p_optical_depth_BW; + [[maybe_unused]] auto *ux_tmp = ux; // for nvhpc + [[maybe_unused]] auto *uy_tmp = uy; + [[maybe_unused]] auto *uz_tmp = uz; [[maybe_unused]] auto dt_tmp = dt; if constexpr (qed_control == has_qed) { evolve_opt(ux[i], uy[i], uz[i], Exp, Eyp, Ezp, Bxp, Byp, Bzp, @@ -240,10 +240,11 @@ PhotonParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { - // This does gather, push and depose. - // Push and depose have been re-written for photons + // This does gather, push and deposit. + // Push and deposit have been re-written for photons PhysicalParticleContainer::Evolve (lev, Ex, Ey, Ez, Bx, By, Bz, @@ -252,6 +253,6 @@ PhotonParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index ec9834c4e7e..b77c1147a15 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -11,6 +11,7 @@ #define WARPX_PhysicalParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED @@ -72,9 +73,9 @@ public: void InitIonizationModule () override; /* - * \brief Returns a pointer to the plasma injector. + * \brief Returns a pointer to the i'th plasma injector. */ - PlasmaInjector* GetPlasmaInjector () override; + PlasmaInjector* GetPlasmaInjector (int i) override; /** * \brief Evolve is the central function PhysicalParticleContainer that @@ -105,6 +106,7 @@ public: * \param dt time step by which particles are advanced * \param a_dt_type type of time step (used for sub-cycling) * \param skip_deposition Skip the charge and current deposition. + * \param push_type Type of particle push, explicit or implicit. Defaults to explicit * * Evolve iterates over particle iterator (each box) and performs filtering, * field gather, particle push and current deposition for all particles @@ -134,7 +136,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; virtual void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -150,6 +153,20 @@ public: amrex::Real dt, ScaleFields scaleFields, DtType a_dt_type=DtType::Full); + void ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type=DtType::Full); + void PushP (int lev, amrex::Real dt, const amrex::MultiFab& Ex, const amrex::MultiFab& Ey, @@ -189,35 +206,41 @@ public: * number of particles per cell (in the cells of `part_realbox`). * The new particles are only created inside the intersection of `part_realbox` * with the local grid for the current proc. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] lev the index of the refinement level * @param[in] part_realbox the box in which new particles should be created * (this box should correspond to an integer number of cells in each direction, * but its boundaries need not be aligned with the actual cells of the simulation) */ - void AddPlasma (int lev, amrex::RealBox part_realbox = amrex::RealBox()); + void AddPlasma (PlasmaInjector const& plasma_injector, int lev, amrex::RealBox part_realbox = amrex::RealBox()); /** * Create new macroparticles for this species, with a fixed * number of particles per cell in a plane. + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] dt time step size, used to partially advance the particles */ - void AddPlasmaFlux (amrex::Real dt); + void AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt); void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz, - amrex::Real t_lab = 0.); + amrex::Real t_lab = 0.) const; void AddGaussianBeam ( + PlasmaInjector const& plasma_injector, amrex::Real x_m, amrex::Real y_m, amrex::Real z_m, amrex::Real x_rms, amrex::Real y_rms, amrex::Real z_rms, amrex::Real x_cut, amrex::Real y_cut, amrex::Real z_cut, - amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order); + amrex::Real q_tot, long npart, int do_symmetrize, int symmetrization_order, + amrex::Real focal_distance); /** Load a particle beam from an external file + * @param[in] the PlasmaInjector instance holding the input parameters * @param[in] q_tot total charge of the particle species to be initialized * @param[in] z_shift optional shift to the z position of particles (useful for boosted frame runs) */ - void AddPlasmaFromFile (amrex::ParticleReal q_tot, + void AddPlasmaFromFile (PlasmaInjector & plasma_injector, + amrex::ParticleReal q_tot, amrex::ParticleReal z_shift); void CheckAndAddParticle ( @@ -231,7 +254,7 @@ public: amrex::Gpu::HostVector& particle_uy, amrex::Gpu::HostVector& particle_uz, amrex::Gpu::HostVector& particle_w, - amrex::Real t_lab= 0.); + amrex::Real t_lab= 0.) const; /** * \brief Default initialize runtime attributes in a tile. This routine does not initialize the @@ -246,12 +269,9 @@ public: * @param[in] engine the random engine, used in initialization of QED optical depths */ void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) final; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) final; /** * \brief Apply NCI Godfrey filter to all components of E and B before gather @@ -281,12 +301,12 @@ public: * \param Bx Field array before filtering (not modified) * \param By Field array before filtering (not modified) * \param Bz Field array before filtering (not modified) - * \param exfab pointer to the Ex field (modified) - * \param eyfab pointer to the Ey field (modified) - * \param ezfab pointer to the Ez field (modified) - * \param bxfab pointer to the Bx field (modified) - * \param byfab pointer to the By field (modified) - * \param bzfab pointer to the Bz field (modified) + * \param ex_ptr pointer to the Ex field (modified) + * \param ey_ptr pointer to the Ey field (modified) + * \param ez_ptr pointer to the Ez field (modified) + * \param bx_ptr pointer to the Bx field (modified) + * \param by_ptr pointer to the By field (modified) + * \param bz_ptr pointer to the Bz field (modified) * * The NCI Godfrey filter is applied on Ex, the result is stored in filtered_Ex * and the pointer exfab is modified (before this function is called, it points to Ex @@ -302,9 +322,9 @@ public: const amrex::FArrayBox& Ex, const amrex::FArrayBox& Ey, const amrex::FArrayBox& Ez, const amrex::FArrayBox& Bx, const amrex::FArrayBox& By, const amrex::FArrayBox& Bz, - amrex::FArrayBox const * & exfab, amrex::FArrayBox const * & eyfab, - amrex::FArrayBox const * & ezfab, amrex::FArrayBox const * & bxfab, - amrex::FArrayBox const * & byfab, amrex::FArrayBox const * & bzfab); + amrex::FArrayBox const * & ex_ptr, amrex::FArrayBox const * & ey_ptr, + amrex::FArrayBox const * & ez_ptr, amrex::FArrayBox const * & bx_ptr, + amrex::FArrayBox const * & by_ptr, amrex::FArrayBox const * & bz_ptr); /** * \brief This function determines if resampling should be done for the current species, and @@ -348,14 +368,38 @@ public: (std::shared_ptr ptr) override; //__________ + BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { + return m_shr_p_bw_engine.get(); + } + + QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const override { + return m_shr_p_qs_engine.get(); + } + PhotonEmissionFilterFunc getPhotonEmissionFilterFunc (); PairGenerationFilterFunc getPairGenerationFilterFunc (); #endif + std::vector getUserIntAttribs () const override { + return m_user_int_attribs; + } + + std::vector getUserRealAttribs () const override { + return m_user_real_attribs; + } + + amrex::Vector< amrex::Parser* > getUserIntAttribParser () const override { + return GetVecOfPtrs(m_user_int_attrib_parser); + } + + amrex::Vector< amrex::Parser* > getUserRealAttribParser () const override { + return GetVecOfPtrs(m_user_real_attrib_parser); + } + protected: std::string species_name; - std::unique_ptr plasma_injector; + std::vector> plasma_injectors; // When true, adjust the transverse particle positions accounting // for the difference between the Lorentz transformed time of the @@ -403,9 +447,9 @@ protected: /* Vector of user-defined real attributes for species, species_name */ std::vector m_user_real_attribs; /* Vector of user-defined parser for initializing user-defined integer attributes */ - std::vector< std::unique_ptr > m_user_int_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_int_attrib_parser; /* Vector of user-defined parser for initializing user-defined real attributes */ - std::vector< std::unique_ptr > m_user_real_attrib_parser; + amrex::Vector< std::unique_ptr > m_user_real_attrib_parser; }; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 810aa73417b..f01a5ccf0fd 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -21,6 +21,7 @@ #endif #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" +#include "Particles/ParticleCreation/DefaultInitialization.H" #include "Particles/Pusher/CopyParticleAttribs.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/PushSelector.H" @@ -38,6 +39,10 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #include "WarpX.H" #include @@ -46,7 +51,6 @@ #include #include #include -#include #include #include #include @@ -193,8 +197,8 @@ namespace * and avoid any possible undefined behavior before the next call to redistribute) and sets * the particle id to -1 so that it can be effectively deleted. * - * \param p particle aos data - * \param pa particle soa data + * \param idcpu particle id soa data + * \param pa particle real soa data * \param ip index for soa data * \param do_field_ionization whether species has ionization * \param pi ionization level data @@ -205,20 +209,21 @@ namespace */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ZeroInitializeAndSetNegativeID ( - ParticleType& p, const GpuArray& pa, long& ip, + uint64_t * AMREX_RESTRICT idcpu, + const GpuArray& pa, long& ip, const bool& do_field_ionization, int* pi #ifdef WARPX_QED - ,const bool& has_quantum_sync, amrex::ParticleReal* p_optical_depth_QSR - ,const bool& has_breit_wheeler, amrex::ParticleReal* p_optical_depth_BW + ,const bool& has_quantum_sync, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR + ,const bool& has_breit_wheeler, amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_BW #endif ) noexcept { - p.pos(0) = 0._rt; + pa[PIdx::z][ip] = 0._rt; #if (AMREX_SPACEDIM >= 2) - p.pos(1) = 0._rt; + pa[PIdx::x][ip] = 0._rt; #endif #if defined(WARPX_DIM_3D) - p.pos(2) = 0._rt; + pa[PIdx::y][ip] = 0._rt; #endif pa[PIdx::w ][ip] = 0._rt; pa[PIdx::ux][ip] = 0._rt; @@ -233,7 +238,7 @@ namespace if (has_breit_wheeler) {p_optical_depth_BW[ip] = 0._rt;} #endif - p.id() = -1; + idcpu[ip] = amrex::ParticleIdCpus::Invalid; } } @@ -244,13 +249,81 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp { BackwardCompatibility(); - plasma_injector = std::make_unique(species_id, species_name, amr_core->Geom(0)); - physical_species = plasma_injector->getPhysicalSpecies(); - charge = plasma_injector->getCharge(); - mass = plasma_injector->getMass(); - const ParmParse pp_species_name(species_name); + std::string injection_style = "none"; + pp_species_name.query("injection_style", injection_style); + if (injection_style != "none") { + // The base plasma injector, whose input parameters have no source prefix. + // Only created if needed + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0))); + } + + std::vector injection_sources; + pp_species_name.queryarr("injection_sources", injection_sources); + for (auto &source_name : injection_sources) { + plasma_injectors.push_back(std::make_unique(species_id, species_name, amr_core->Geom(0), + source_name)); + } + + // Setup the charge and mass. There are multiple ways that they can be specified, so checks are needed to + // ensure that a value is specified and warnings given if multiple values are specified. + // The ordering is that species.charge and species.mass take precedence over all other values. + // Next is charge and mass determined from species_type. + // Last is charge and mass from the plasma injector setup + bool charge_from_source = false; + bool mass_from_source = false; + for (auto const& plasma_injector : plasma_injectors) { + // For now, use the last value for charge and mass that is found. + // A check could be added for consistency of multiple values, but it'll probably never be needed + charge_from_source |= plasma_injector->queryCharge(charge); + mass_from_source |= plasma_injector->queryMass(mass); + } + + std::string physical_species_s; + const bool species_is_specified = pp_species_name.query("species_type", physical_species_s); + if (species_is_specified) { + const auto physical_species_from_string = species::from_string( physical_species_s ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(physical_species_from_string, + physical_species_s + " does not exist!"); + physical_species = physical_species_from_string.value(); + charge = species::get_charge( physical_species ); + mass = species::get_mass( physical_species ); + } + + // parse charge and mass (overriding values above) + const bool charge_is_specified = utils::parser::queryWithParser(pp_species_name, "charge", charge); + const bool mass_is_specified = utils::parser::queryWithParser(pp_species_name, "mass", mass); + + if (charge_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".charge' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".charge' will take precedence.\n"); + } + if (mass_is_specified && species_is_specified) { + ablastr::warn_manager::WMRecordWarning("Species", + "Both '" + species_name + ".mass' and " + + species_name + ".species_type' are specified.\n" + + species_name + ".mass' will take precedence.\n"); + } + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + charge_from_source || + charge_is_specified || + species_is_specified, + "Need to specify at least one of species_type or charge for species '" + + species_name + "'." + ); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass_from_source || + mass_is_specified || + species_is_specified, + "Need to specify at least one of species_type or mass for species '" + + species_name + "'." + ); + pp_species_name.query("boost_adjust_transverse_positions", boost_adjust_transverse_positions); pp_species_name.query("do_backward_propagation", do_backward_propagation); pp_species_name.query("random_theta", m_rz_random_theta); @@ -275,7 +348,7 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_species_name.query("do_field_ionization", do_field_ionization); pp_species_name.query("do_resampling", do_resampling); - if (do_resampling) m_resampler = Resampling(species_name); + if (do_resampling) { m_resampler = Resampling(species_name); } //check if Radiation Reaction is enabled and do consistency checks pp_species_name.query("do_classical_radiation_reaction", do_classical_radiation_reaction); @@ -297,12 +370,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp #ifdef WARPX_QED pp_species_name.query("do_qed_quantum_sync", m_do_qed_quantum_sync); - if (m_do_qed_quantum_sync) + if (m_do_qed_quantum_sync) { AddRealComp("opticalDepthQSR"); + } pp_species_name.query("do_qed_breit_wheeler", m_do_qed_breit_wheeler); - if (m_do_qed_breit_wheeler) + if (m_do_qed_breit_wheeler) { AddRealComp("opticalDepthBW"); + } if(m_do_qed_quantum_sync){ pp_species_name.get("qed_quantum_sync_phot_product_species", @@ -374,7 +449,6 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) : WarpXParticleContainer(amr_core, 0) { - plasma_injector = std::make_unique(); } void @@ -401,7 +475,7 @@ void PhysicalParticleContainer::InitData () } void PhysicalParticleContainer::MapParticletoBoostedFrame ( - ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) + ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, Real t_lab) const { // Map the particles from the lab frame to the boosted frame. // This boosts the particle to the lab frame and calculates @@ -448,12 +522,13 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame ( void PhysicalParticleContainer::AddGaussianBeam ( + PlasmaInjector const& plasma_injector, const Real x_m, const Real y_m, const Real z_m, const Real x_rms, const Real y_rms, const Real z_rms, const Real x_cut, const Real y_cut, const Real z_cut, const Real q_tot, long npart, const int do_symmetrize, - const int symmetrization_order) { + const int symmetrization_order, const Real focal_distance) { // Declare temporary vectors on the CPU Gpu::HostVector particle_x; @@ -474,28 +549,65 @@ PhysicalParticleContainer::AddGaussianBeam ( for (long i = 0; i < npart; ++i) { #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) const Real weight = q_tot/(npart*charge); - const Real x = amrex::RandomNormal(x_m, x_rms); - const Real y = amrex::RandomNormal(y_m, y_rms); - const Real z = amrex::RandomNormal(z_m, z_rms); + Real x = amrex::RandomNormal(x_m, x_rms); + Real y = amrex::RandomNormal(y_m, y_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #elif defined(WARPX_DIM_XZ) const Real weight = q_tot/(npart*charge*y_rms); - const Real x = amrex::RandomNormal(x_m, x_rms); + Real x = amrex::RandomNormal(x_m, x_rms); constexpr Real y = 0._prt; - const Real z = amrex::RandomNormal(z_m, z_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #elif defined(WARPX_DIM_1D_Z) const Real weight = q_tot/(npart*charge*x_rms*y_rms); constexpr Real x = 0._prt; constexpr Real y = 0._prt; - const Real z = amrex::RandomNormal(z_m, z_rms); + Real z = amrex::RandomNormal(z_m, z_rms); #endif - if (plasma_injector->insideBounds(x, y, z) && + if (plasma_injector.insideBounds(x, y, z) && std::abs( x - x_m ) <= x_cut * x_rms && std::abs( y - y_m ) <= y_cut * y_rms && std::abs( z - z_m ) <= z_cut * z_rms ) { - XDim3 u = plasma_injector->getMomentum(x, y, z); + XDim3 u = plasma_injector.getMomentum(x, y, z); + + if (plasma_injector.do_focusing){ + XDim3 u_bulk = plasma_injector.getInjectorMomentumHost()->getBulkMomentum(x,y,z); + Real u_bulk_norm = std::sqrt( u_bulk.x*u_bulk.x+u_bulk.y*u_bulk.y+u_bulk.z*u_bulk.z ); + + // Compute the position of the focal plane + // (it is located at a distance `focal_distance` from the beam centroid, in the direction of the bulk velocity) + Real n_x = u_bulk.x/u_bulk_norm; + Real n_y = u_bulk.y/u_bulk_norm; + Real n_z = u_bulk.z/u_bulk_norm; + Real x_f = x_m + focal_distance * n_x; + Real y_f = y_m + focal_distance * n_y; + Real z_f = z_m + focal_distance * n_z; + Real gamma = std::sqrt( 1._rt + (u.x*u.x+u.y*u.y+u.z*u.z) ); + + Real v_x = u.x / gamma * PhysConst::c; + Real v_y = u.y / gamma * PhysConst::c; + Real v_z = u.z / gamma * PhysConst::c; + + // Compute the time at which the particle will cross the focal plane + Real v_dot_n = v_x * n_x + v_y * n_y + v_z * n_z; + Real t = ((x_f-x)*n_x + (y_f-y)*n_y + (z_f-z)*n_z) / v_dot_n; + + // Displace particles in the direction orthogonal to the beam bulk momentum + // i.e. orthogonal to (n_x, n_y, n_z) +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + x = x - (v_x - v_dot_n*n_x) * t; + y = y - (v_y - v_dot_n*n_y) * t; + z = z - (v_z - v_dot_n*n_z) * t; +#elif defined(WARPX_DIM_XZ) + x = x - (v_x - v_dot_n*n_x) * t; + z = z - (v_z - v_dot_n*n_z) * t; +#elif defined(WARPX_DIM_1D_Z) + z = z - (v_z - v_dot_n*n_z) * t; +#endif + } u.x *= PhysConst::c; u.y *= PhysConst::c; u.z *= PhysConst::c; + if (do_symmetrize && symmetrization_order == 8){ // Add eight particles to the beam: CheckAndAddParticle(x, y, z, u.x, u.y, u.z, weight/8._rt, @@ -559,6 +671,7 @@ PhysicalParticleContainer::AddGaussianBeam ( } // Add the temporary CPU vectors to the particle structure auto const np = static_cast(particle_z.size()); + amrex::Vector xp(particle_x.data(), particle_x.data() + np); amrex::Vector yp(particle_y.data(), particle_y.data() + np); amrex::Vector zp(particle_z.data(), particle_z.data() + np); @@ -577,7 +690,8 @@ PhysicalParticleContainer::AddGaussianBeam ( } void -PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, +PhysicalParticleContainer::AddPlasmaFromFile(PlasmaInjector & plasma_injector, + ParticleReal q_tot, ParticleReal z_shift) { // Declare temporary vectors on the CPU @@ -592,10 +706,8 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #ifdef WARPX_USE_OPENPMD //TODO: Make changes for read/write in multiple MPI ranks if (ParallelDescriptor::IOProcessor()) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(plasma_injector, - "AddPlasmaFromFile: plasma injector not initialized.\n"); // take ownership of the series and close it when done - auto series = std::move(plasma_injector->m_openpmd_input_series); + auto series = std::move(plasma_injector.m_openpmd_input_series); // assumption asserts: see PlasmaInjector openPMD::Iteration it = series->iterations.begin()->second; @@ -663,7 +775,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, #endif ParticleReal const z = ptr_z.get()[i]*position_unit_z + ptr_offset_z.get()[i]*position_offset_unit_z + z_shift; - if (plasma_injector->insideBounds(x, y, z)) { + if (plasma_injector.insideBounds(x, y, z)) { ParticleReal const ux = ptr_ux.get()[i]*momentum_unit_x/mass; ParticleReal const uz = ptr_uz.get()[i]*momentum_unit_z/mass; ParticleReal uy = 0.0_prt; @@ -701,119 +813,28 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot, 1, attr, 0, attr_int, 1); #endif // WARPX_USE_OPENPMD - ignore_unused(q_tot, z_shift); + ignore_unused(plasma_injector, q_tot, z_shift); } void PhysicalParticleContainer::DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - const int n_external_attr_real, - const int n_external_attr_int, - const amrex::RandomEngine& engine) + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) { - using namespace amrex::literals; - - const int np = pinned_tile.numParticles(); - - // Preparing data needed for user defined attributes - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto get_position = GetParticlePosition(pinned_tile); - const auto soa = pinned_tile.getParticleTileData(); - const amrex::ParticleReal* AMREX_RESTRICT ux = soa.m_rdata[PIdx::ux]; - const amrex::ParticleReal* AMREX_RESTRICT uy = soa.m_rdata[PIdx::uy]; - const amrex::ParticleReal* AMREX_RESTRICT uz = soa.m_rdata[PIdx::uz]; - constexpr int lev = 0; - const amrex::Real t = WarpX::GetInstance().gett_new(lev); - -#ifndef WARPX_QED - amrex::ignore_unused(engine); -#endif - - // Initialize the last NumRuntimeRealComps() - n_external_attr_real runtime real attributes - for (int j = PIdx::nattribs + n_external_attr_real; j < NumRealComps() ; ++j) - { - amrex::Vector attr_temp(np, 0.0_prt); + ParticleCreation::DefaultInitializeRuntimeAttributes(pinned_tile, + n_external_attr_real, n_external_attr_int, + m_user_real_attribs, m_user_int_attribs, + particle_comps, particle_icomps, + amrex::GetVecOfPtrs(m_user_real_attrib_parser), + amrex::GetVecOfPtrs(m_user_int_attrib_parser), #ifdef WARPX_QED - // Current runtime comp is quantum synchrotron optical depth - if (particle_comps.find("opticalDepthQSR") != particle_comps.end() && - particle_comps["opticalDepthQSR"] == j) - { - const QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt = - m_shr_p_qs_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = quantum_sync_get_opt(engine); - } - } - - // Current runtime comp is Breit-Wheeler optical depth - if (particle_comps.find("opticalDepthBW") != particle_comps.end() && - particle_comps["opticalDepthBW"] == j) - { - const BreitWheelerGetOpticalDepth breit_wheeler_get_opt = - m_shr_p_bw_engine->build_optical_depth_functor();; - for (int i = 0; i < np; ++i) { - attr_temp[i] = breit_wheeler_get_opt(engine); - } - } + true, + m_shr_p_bw_engine.get(), + m_shr_p_qs_engine.get(), #endif - - for (int ia = 0; ia < n_user_real_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_comps.find(m_user_real_attribs[ia]) != particle_comps.end() && - particle_comps[m_user_real_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_real_attrib_parserexec = - m_user_real_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = user_real_attrib_parserexec(xp, yp, zp, - ux[i], uy[i], uz[i], t); - } - } - } - - pinned_tile.push_back_real(j, attr_temp.data(), attr_temp.data() + np); - } - - // Initialize the last NumRuntimeIntComps() - n_external_attr_int runtime int attributes - for (int j = n_external_attr_int; j < NumIntComps() ; ++j) - { - amrex::Vector attr_temp(np, 0); - - // Current runtime comp is ionization level - if (particle_icomps.find("ionizationLevel") != particle_icomps.end() && - particle_icomps["ionizationLevel"] == j) - { - for (int i = 0; i < np; ++i) { - attr_temp[i] = ionization_initial_level; - } - } - - for (int ia = 0; ia < n_user_int_attribs; ++ia) - { - // Current runtime comp is ia-th user defined attribute - if (particle_icomps.find(m_user_int_attribs[ia]) != particle_icomps.end() && - particle_icomps[m_user_int_attribs[ia]] == j) - { - amrex::ParticleReal xp, yp, zp; - const amrex::ParserExecutor<7> user_int_attrib_parserexec = - m_user_int_attrib_parser[ia]->compile<7>(); - for (int i = 0; i < np; ++i) { - get_position(i, xp, yp, zp); - attr_temp[i] = static_cast( - user_int_attrib_parserexec(xp, yp, zp, ux[i], uy[i], uz[i], t)); - } - } - } - - pinned_tile.push_back_int(j, attr_temp.data(), attr_temp.data() + np); - } - + ionization_initial_level, + 0,pinned_tile.numParticles()); } @@ -829,7 +850,7 @@ PhysicalParticleContainer::CheckAndAddParticle ( Gpu::HostVector& particle_uy, Gpu::HostVector& particle_uz, Gpu::HostVector& particle_w, - Real t_lab) + Real t_lab) const { if (WarpX::gamma_boost > 1.) { MapParticletoBoostedFrame(x, y, z, ux, uy, uz, t_lab); @@ -848,95 +869,96 @@ PhysicalParticleContainer::AddParticles (int lev) { WARPX_PROFILE("PhysicalParticleContainer::AddParticles()"); - if (plasma_injector->add_single_particle) { - if (WarpX::gamma_boost > 1.) { - MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], - plasma_injector->single_particle_pos[1], - plasma_injector->single_particle_pos[2], - plasma_injector->single_particle_u[0], - plasma_injector->single_particle_u[1], - plasma_injector->single_particle_u[2]); + for (auto const& plasma_injector : plasma_injectors) { + + if (plasma_injector->add_single_particle) { + if (WarpX::gamma_boost > 1.) { + MapParticletoBoostedFrame(plasma_injector->single_particle_pos[0], + plasma_injector->single_particle_pos[1], + plasma_injector->single_particle_pos[2], + plasma_injector->single_particle_u[0], + plasma_injector->single_particle_u[1], + plasma_injector->single_particle_u[2]); + } + amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; + amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; + amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; + amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; + amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; + amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; + amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; + amrex::Vector> attr_int; + AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, + 1, attr, 0, attr_int, 0); + return; } - amrex::Vector xp = {plasma_injector->single_particle_pos[0]}; - amrex::Vector yp = {plasma_injector->single_particle_pos[1]}; - amrex::Vector zp = {plasma_injector->single_particle_pos[2]}; - amrex::Vector uxp = {plasma_injector->single_particle_u[0]}; - amrex::Vector uyp = {plasma_injector->single_particle_u[1]}; - amrex::Vector uzp = {plasma_injector->single_particle_u[2]}; - amrex::Vector> attr = {{plasma_injector->single_particle_weight}}; - amrex::Vector> attr_int; - AddNParticles(lev, 1, xp, yp, zp, uxp, uyp, uzp, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->add_multiple_particles) { - if (WarpX::gamma_boost > 1.) { - for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { - MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], - plasma_injector->multiple_particles_pos_y[i], - plasma_injector->multiple_particles_pos_z[i], - plasma_injector->multiple_particles_ux[i], - plasma_injector->multiple_particles_uy[i], - plasma_injector->multiple_particles_uz[i]); + if (plasma_injector->add_multiple_particles) { + if (WarpX::gamma_boost > 1.) { + for (int i=0 ; i < plasma_injector->multiple_particles_pos_x.size() ; i++) { + MapParticletoBoostedFrame(plasma_injector->multiple_particles_pos_x[i], + plasma_injector->multiple_particles_pos_y[i], + plasma_injector->multiple_particles_pos_z[i], + plasma_injector->multiple_particles_ux[i], + plasma_injector->multiple_particles_uy[i], + plasma_injector->multiple_particles_uz[i]); + } } + amrex::Vector> attr; + attr.push_back(plasma_injector->multiple_particles_weight); + amrex::Vector> attr_int; + AddNParticles(lev, static_cast(plasma_injector->multiple_particles_pos_x.size()), + plasma_injector->multiple_particles_pos_x, + plasma_injector->multiple_particles_pos_y, + plasma_injector->multiple_particles_pos_z, + plasma_injector->multiple_particles_ux, + plasma_injector->multiple_particles_uy, + plasma_injector->multiple_particles_uz, + 1, attr, 0, attr_int, 0); } - amrex::Vector> attr; - attr.push_back(plasma_injector->multiple_particles_weight); - amrex::Vector> attr_int; - AddNParticles(lev, static_cast(plasma_injector->multiple_particles_pos_x.size()), - plasma_injector->multiple_particles_pos_x, - plasma_injector->multiple_particles_pos_y, - plasma_injector->multiple_particles_pos_z, - plasma_injector->multiple_particles_ux, - plasma_injector->multiple_particles_uy, - plasma_injector->multiple_particles_uz, - 1, attr, 0, attr_int, 0); - return; - } - if (plasma_injector->gaussian_beam) { - AddGaussianBeam(plasma_injector->x_m, - plasma_injector->y_m, - plasma_injector->z_m, - plasma_injector->x_rms, - plasma_injector->y_rms, - plasma_injector->z_rms, - plasma_injector->x_cut, - plasma_injector->y_cut, - plasma_injector->z_cut, - plasma_injector->q_tot, - plasma_injector->npart, - plasma_injector->do_symmetrize, - plasma_injector->symmetrization_order); - - - return; - } + if (plasma_injector->gaussian_beam) { + AddGaussianBeam(*plasma_injector, + plasma_injector->x_m, + plasma_injector->y_m, + plasma_injector->z_m, + plasma_injector->x_rms, + plasma_injector->y_rms, + plasma_injector->z_rms, + plasma_injector->x_cut, + plasma_injector->y_cut, + plasma_injector->z_cut, + plasma_injector->q_tot, + plasma_injector->npart, + plasma_injector->do_symmetrize, + plasma_injector->symmetrization_order, + plasma_injector->focal_distance); + } - if (plasma_injector->external_file) { - AddPlasmaFromFile(plasma_injector->q_tot, - plasma_injector->z_shift); - return; - } + if (plasma_injector->external_file) { + AddPlasmaFromFile(*plasma_injector, + plasma_injector->q_tot, + plasma_injector->z_shift); + } - if ( plasma_injector->doInjection() ) { - AddPlasma( lev ); + if ( plasma_injector->doInjection() ) { + AddPlasma(*plasma_injector, lev); + } } } void -PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) +PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int lev, RealBox part_realbox) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasma()"); // If no part_realbox is provided, initialize particles in the whole domain const Geometry& geom = Geom(lev); - if (!part_realbox.ok()) part_realbox = geom.ProbDomain(); + if (!part_realbox.ok()) { part_realbox = geom.ProbDomain(); } - const int num_ppc = plasma_injector->num_particles_per_cell; + const int num_ppc = plasma_injector.num_particles_per_cell; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, part_realbox.hi(0)); + Real rmax = std::min(plasma_injector.xmax, part_realbox.hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -963,18 +985,18 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) fine_injection_box.coarsen(rrfac); } - InjectorPosition* inj_pos = plasma_injector->getInjectorPosition(); - InjectorDensity* inj_rho = plasma_injector->getInjectorDensity(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* inj_pos = plasma_injector.getInjectorPosition(); + InjectorDensity* inj_rho = plasma_injector.getInjectorDensity(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); const Real gamma_boost = WarpX::gamma_boost; const Real beta_boost = WarpX::beta_boost; const Real t = WarpX::GetInstance().gett_new(lev); - const Real density_min = plasma_injector->density_min; - const Real density_max = plasma_injector->density_max; + const Real density_min = plasma_injector.density_min; + const Real density_max = plasma_injector.density_max; #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -1038,9 +1060,9 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) overlap_realbox.lo(2))}; // count the number of particles that each cell in overlap_box could add - Gpu::DeviceVector counts(overlap_box.numPts(), 0); - Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + Gpu::DeviceVector counts(overlap_box.numPts(), 0); + Gpu::DeviceVector offset(overlap_box.numPts()); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1058,7 +1080,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) if (inj_pos->overlapsWith(lo, hi)) { auto index = overlap_box.index(iv); - const int r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? + const amrex::Long r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? (AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2])) : (1); pcounts[index] = num_ppc*r; // update pcount by checking if cell-corners or cell-center @@ -1068,12 +1090,15 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const auto zlim = GpuArray{lo.z,(lo.z+hi.z)/2._rt,hi.z}; const auto checker = [&](){ - for (const auto& x : xlim) - for (const auto& y : ylim) - for (const auto& z : zlim) + for (const auto& x : xlim) { + for (const auto& y : ylim) { + for (const auto& z : zlim) { if (inj_pos->insideBounds(x,y,z) and (inj_rho->getDensity(x,y,z) > 0) ) { return 1; } + } + } + } return 0; }; const int flag_pcount = checker(); @@ -1093,10 +1118,10 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1105,7 +1130,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "ERROR: overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); @@ -1116,16 +1141,16 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) DefineAndReturnParticleTile(lev, grid_id, tile_id); } - auto old_size = particle_tile.GetArrayOfStructs().size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); @@ -1181,12 +1206,14 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1208,7 +1235,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); #ifdef WARPX_DIM_RZ const bool rz_random_theta = m_rz_random_theta; #endif @@ -1219,7 +1246,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const auto index = overlap_box.index(iv); #ifdef WARPX_DIM_RZ Real theta_offset = 0._rt; - if (rz_random_theta) theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; + if (rz_random_theta) { theta_offset = amrex::Random(engine) * 2._rt * MathConst::pi; } #endif Real scale_fac = 0.0_rt; @@ -1236,9 +1263,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) for (int i_part = 0; i_part < pcounts[index]; ++i_part) { long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? // In the refined injection region: use refinement ratio `lrrfac` inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : @@ -1248,7 +1273,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #if defined(WARPX_DIM_3D) if (!tile_realbox.contains(XDim3{pos.x,pos.y,pos.z})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1259,7 +1284,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!tile_realbox.contains(XDim3{pos.x,pos.z,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1270,7 +1295,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) #else amrex::ignore_unused(j,k); if (!tile_realbox.contains(XDim3{pos.z,0.0_rt,0.0_rt})) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1309,7 +1334,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) const Real z0 = applyBallisticCorrection(pos, inj_mom, gamma_boost, beta_boost, t); if (!inj_pos->insideBounds(xb, yb, z0)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1323,7 +1348,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1341,7 +1366,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) // If the particle is not within the lab-frame zmin, zmax, etc. // go to the next generated particle. if (!inj_pos->insideBounds(xb, yb, z0_lab)) { - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1353,7 +1378,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) dens = inj_rho->getDensity(pos.x, pos.y, z0_lab); // Remove particle if density below threshold if ( dens < density_min ){ - ZeroInitializeAndSetNegativeID(p, pa, ip, loc_do_field_ionization, pi + ZeroInitializeAndSetNegativeID(pa_idcpu, pa, ip, loc_do_field_ionization, pi #ifdef WARPX_QED ,loc_has_quantum_sync, p_optical_depth_QSR ,loc_has_breit_wheeler, p_optical_depth_BW @@ -1420,17 +1445,17 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) pa[PIdx::uz][ip] = u.z; #if defined(WARPX_DIM_3D) - p.pos(0) = pos.x; - p.pos(1) = pos.y; - p.pos(2) = pos.z; + pa[PIdx::x][ip] = pos.x; + pa[PIdx::y][ip] = pos.y; + pa[PIdx::z][ip] = pos.z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) #ifdef WARPX_DIM_RZ pa[PIdx::theta][ip] = theta; #endif - p.pos(0) = xb; - p.pos(1) = pos.z; + pa[PIdx::x][ip] = xb; + pa[PIdx::z][ip] = pos.z; #else - p.pos(0) = pos.z; + pa[PIdx::z][ip] = pos.z; #endif } }); @@ -1444,20 +1469,26 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox) } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // The function that calls this is responsible for redistributing particles. } void -PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) +PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::Real dt) { WARPX_PROFILE("PhysicalParticleContainer::AddPlasmaFlux()"); const Geometry& geom = Geom(0); const amrex::RealBox& part_realbox = geom.ProbDomain(); - const amrex::Real num_ppc_real = plasma_injector->num_particles_per_cell_real; + const amrex::Real num_ppc_real = plasma_injector.num_particles_per_cell_real; #ifdef WARPX_DIM_RZ - Real rmax = std::min(plasma_injector->xmax, geom.ProbDomain().hi(0)); + Real rmax = std::min(plasma_injector.xmax, geom.ProbDomain().hi(0)); #endif const auto dx = geom.CellSizeArray(); @@ -1466,20 +1497,20 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) Real scale_fac = 0._rt; // Scale particle weight by the area of the emitting surface, within one cell #if defined(WARPX_DIM_3D) - scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector->flux_normal_axis]/num_ppc_real; + scale_fac = dx[0]*dx[1]*dx[2]/dx[plasma_injector.flux_normal_axis]/num_ppc_real; #elif defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) scale_fac = dx[0]*dx[1]/num_ppc_real; // When emission is in the r direction, the emitting surface is a cylinder. // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 0) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 0) { scale_fac /= dx[0]; } // When emission is in the z direction, the emitting surface is an annulus // The factor 2*pi*r is added later below. - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[1]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[1]; } // When emission is in the theta direction (flux_normal_axis == 1), // the emitting surface is a rectangle, within the plane of the simulation #elif defined(WARPX_DIM_1D_Z) scale_fac = dx[0]/num_ppc_real; - if (plasma_injector->flux_normal_axis == 2) scale_fac /= dx[0]; + if (plasma_injector.flux_normal_axis == 2) { scale_fac /= dx[0]; } #endif amrex::LayoutData* cost = WarpX::getCosts(0); @@ -1506,16 +1537,16 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) fine_injection_box.coarsen(rrfac); } - InjectorPosition* flux_pos = plasma_injector->getInjectorFluxPosition(); - InjectorFlux* inj_flux = plasma_injector->getInjectorFlux(); - InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice(); + InjectorPosition* flux_pos = plasma_injector.getInjectorFluxPosition(); + InjectorFlux* inj_flux = plasma_injector.getInjectorFlux(); + InjectorMomentum* inj_mom = plasma_injector.getInjectorMomentumDevice(); constexpr int level_zero = 0; const amrex::Real t = WarpX::GetInstance().gett_new(level_zero); #ifdef WARPX_DIM_RZ const int nmodes = WarpX::n_rz_azimuthal_modes; const bool rz_random_theta = m_rz_random_theta; - bool radially_weighted = plasma_injector->radially_weighted; + bool radially_weighted = plasma_injector.radially_weighted; #endif MFItInfo info; @@ -1546,30 +1577,30 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) for (int dir=0; dirflux_normal_axis) { + if (dir == plasma_injector.flux_normal_axis) { #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - if (2*dir == plasma_injector->flux_normal_axis) { + if (2*dir == plasma_injector.flux_normal_axis) { // The above formula captures the following cases: // - flux_normal_axis=0 (emission along x/r) and dir=0 // - flux_normal_axis=2 (emission along z) and dir=1 #elif defined(WARPX_DIM_1D_Z) - if ( (dir==0) && (plasma_injector->flux_normal_axis==2) ) { + if ( (dir==0) && (plasma_injector.flux_normal_axis==2) ) { #endif - if (plasma_injector->flux_direction > 0) { - if (plasma_injector->surface_flux_pos < tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos >= tile_realbox.hi(dir)) { + if (plasma_injector.flux_direction > 0) { + if (plasma_injector.surface_flux_pos < tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos >= tile_realbox.hi(dir)) { no_overlap = true; break; } } else { - if (plasma_injector->surface_flux_pos <= tile_realbox.lo(dir) || - plasma_injector->surface_flux_pos > tile_realbox.hi(dir)) { + if (plasma_injector.surface_flux_pos <= tile_realbox.lo(dir) || + plasma_injector.surface_flux_pos > tile_realbox.hi(dir)) { no_overlap = true; break; } } - overlap_realbox.setLo( dir, plasma_injector->surface_flux_pos ); - overlap_realbox.setHi( dir, plasma_injector->surface_flux_pos ); + overlap_realbox.setLo( dir, plasma_injector.surface_flux_pos ); + overlap_realbox.setHi( dir, plasma_injector.surface_flux_pos ); overlap_box.setSmall( dir, 0 ); overlap_box.setBig( dir, 0 ); shifted[dir] = @@ -1612,7 +1643,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // count the number of particles that each cell in overlap_box could add Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); - auto pcounts = counts.data(); + auto *pcounts = counts.data(); const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1646,10 +1677,10 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // Max number of new particles. All of them are created, // and invalid ones are then discarded - const int max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); + const amrex::Long max_new_particles = Scan::ExclusiveSum(counts.size(), counts.data(), offset.data()); // Update NextID to include particles created in this function - Long pid; + amrex::Long pid; #ifdef AMREX_USE_OMP #pragma omp critical (add_plasma_nextid) #endif @@ -1658,23 +1689,23 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) ParticleType::NextID(pid+max_new_particles); } WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - static_cast(pid + max_new_particles) < LastParticleID, + pid + max_new_particles < LongParticleIds::LastParticleID, "overflow on particle id numbers"); const int cpuid = ParallelDescriptor::MyProc(); auto& particle_tile = tmp_pc.DefineAndReturnParticleTile(0, grid_id, tile_id); - auto old_size = particle_tile.GetArrayOfStructs().size(); - auto new_size = old_size + max_new_particles; + auto const old_size = static_cast(particle_tile.size()); + auto const new_size = old_size + max_new_particles; particle_tile.resize(new_size); - ParticleType* pp = particle_tile.GetArrayOfStructs()().data() + old_size; auto& soa = particle_tile.GetStructOfArrays(); GpuArray pa; for (int ia = 0; ia < PIdx::nattribs; ++ia) { pa[ia] = soa.GetRealData(ia).data() + old_size; } + uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); @@ -1731,12 +1762,14 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // has to be initialized const bool loc_has_quantum_sync = has_quantum_sync(); const bool loc_has_breit_wheeler = has_breit_wheeler(); - if (loc_has_quantum_sync) + if (loc_has_quantum_sync) { p_optical_depth_QSR = soa.GetRealData( particle_comps["opticalDepthQSR"]).data() + old_size; - if(loc_has_breit_wheeler) + } + if(loc_has_breit_wheeler) { p_optical_depth_BW = soa.GetRealData( particle_comps["opticalDepthBW"]).data() + old_size; + } //If needed, get the appropriate functors from the engines QuantumSynchrotronGetOpticalDepth quantum_sync_get_opt; @@ -1754,14 +1787,14 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) const bool loc_do_field_ionization = do_field_ionization; const int loc_ionization_initial_level = ionization_initial_level; #ifdef WARPX_DIM_RZ - int const loc_flux_normal_axis = plasma_injector->flux_normal_axis; + int const loc_flux_normal_axis = plasma_injector.flux_normal_axis; #endif // Loop over all new particles and inject them (creates too many // particles, in particular does not consider xmin, xmax etc.). // The invalid ones are given negative ID and are deleted during the // next redistribute. - const auto poffset = offset.data(); + auto *const poffset = offset.data(); amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { @@ -1770,9 +1803,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) for (int i_part = 0; i_part < pcounts[index]; ++i_part) { const long ip = poffset[index] + i_part; - ParticleType& p = pp[ip]; - p.id() = pid+ip; - p.cpu() = cpuid; + pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); // This assumes the flux_pos is of type InjectorPositionRandomPlane const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? @@ -1797,19 +1828,19 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // the particles will be within the domain. #if defined(WARPX_DIM_3D) if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.y,ppos.z})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.x,ppos.z,0.0_prt})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #else amrex::ignore_unused(j,k); if (!ParticleUtils::containsInclusive(tile_realbox, XDim3{ppos.z,0.0_prt,0.0_prt})) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } #endif @@ -1817,7 +1848,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // If the particle's initial position is not within or on the species's // xmin, xmax, ymin, ymax, zmin, zmax, go to the next generated particle. if (!flux_pos->insideBoundsInclusive(ppos.x, ppos.y, ppos.z)) { - p.id() = -1; + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } @@ -1840,7 +1871,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) // Rotate the momentum // This because, when the flux direction is e.g. "r" // the `inj_mom` objects generates a v*Gaussian distribution - // along the Cartesian "x" directionm by default. This + // along the Cartesian "x" direction by default. This // needs to be rotated along "r". Real ur = pu.x; Real ut = pu.y; @@ -1850,8 +1881,8 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) #endif Real flux = inj_flux->getFlux(ppos.x, ppos.y, ppos.z, t); // Remove particle if flux is negative or 0 - if ( flux <=0 ){ - p.id() = -1; + if (flux <= 0) { + pa_idcpu[ip] = amrex::ParticleIdCpus::Invalid; continue; } @@ -1860,7 +1891,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) } #ifdef WARPX_QED - if(loc_has_quantum_sync){ + if (loc_has_quantum_sync) { p_optical_depth_QSR[ip] = quantum_sync_get_opt(engine); } @@ -1910,18 +1941,18 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) UpdatePosition(ppos.x, ppos.y, ppos.z, pu.x, pu.y, pu.z, t_fract); #if defined(WARPX_DIM_3D) - p.pos(0) = ppos.x; - p.pos(1) = ppos.y; - p.pos(2) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::y][ip] = ppos.y; + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_RZ) pa[PIdx::theta][ip] = std::atan2(ppos.y, ppos.x); - p.pos(0) = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = std::sqrt(ppos.x*ppos.x + ppos.y*ppos.y); + pa[PIdx::z][ip] = ppos.z; #elif defined(WARPX_DIM_XZ) - p.pos(0) = ppos.x; - p.pos(1) = ppos.z; + pa[PIdx::x][ip] = ppos.x; + pa[PIdx::z][ip] = ppos.z; #else - p.pos(0) = ppos.z; + pa[PIdx::z][ip] = ppos.z; #endif } }); @@ -1935,6 +1966,12 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt) } } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + // Redistribute the new particles that were added to the temporary container. // (This eliminates invalid particles, and makes sure that particles // are in the right tile.) @@ -1971,7 +2008,8 @@ PhysicalParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition) + Real /*t*/, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { WARPX_PROFILE("PhysicalParticleContainer::Evolve()"); @@ -1994,8 +2032,9 @@ PhysicalParticleContainer::Evolve (int lev, const auto t_lev = pti.GetLevel(); const auto index = pti.GetPairIndex(); tmp_particle_data.resize(finestLevel()+1); - for (int i = 0; i < TmpIdx::nattribs; ++i) + for (int i = 0; i < TmpIdx::nattribs; ++i) { tmp_particle_data[t_lev][index][i].resize(np); + } } } @@ -2099,10 +2138,17 @@ PhysicalParticleContainer::Evolve (int lev, WARPX_PROFILE_VAR_START(blp_fg); const auto np_to_push = np_gather; const auto gather_lev = lev; - PushPX(pti, exfab, eyfab, ezfab, - bxfab, byfab, bzfab, - Ex.nGrowVect(), e_is_nodal, - 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, exfab, eyfab, ezfab, + bxfab, byfab, bzfab, + Ex.nGrowVect(), e_is_nodal, + 0, np_to_push, lev, gather_lev, dt, ScaleFields(false), a_dt_type); + } if (np_gather < np) { @@ -2133,11 +2179,19 @@ PhysicalParticleContainer::Evolve (int lev, // Field gather and push for particles in gather buffers e_is_nodal = cEx->is_nodal() and cEy->is_nodal() and cEz->is_nodal(); - PushPX(pti, cexfab, ceyfab, cezfab, - cbxfab, cbyfab, cbzfab, - cEx->nGrowVect(), e_is_nodal, - nfine_gather, np-nfine_gather, - lev, lev-1, dt, ScaleFields(false), a_dt_type); + if (push_type == PushType::Explicit) { + PushPX(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } else if (push_type == PushType::Implicit) { + ImplicitPushXP(pti, cexfab, ceyfab, cezfab, + cbxfab, cbyfab, cbzfab, + cEx->nGrowVect(), e_is_nodal, + nfine_gather, np-nfine_gather, + lev, lev-1, dt, ScaleFields(false), a_dt_type); + } } WARPX_PROFILE_VAR_STOP(blp_fg); @@ -2145,6 +2199,9 @@ PhysicalParticleContainer::Evolve (int lev, // Current Deposition if (!skip_deposition) { + //// Deposit at t_{n+1/2} with explicit push + //const amrex::Real relative_time = (push_type == PushType::Explicit ? -0.5_rt * dt : 0.0_rt); + const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; @@ -2152,17 +2209,17 @@ PhysicalParticleContainer::Evolve (int lev, for (int i=0; i(pti); - const amrex::Vector ppc_nd = plasma_injector->num_particles_per_cell_each_dim; + const amrex::Vector ppc_nd = plasma_injectors[0]->num_particles_per_cell_each_dim; const std::array& dx = WarpX::CellSize(lev); amrex::Vector split_offset = {dx[0]/2._rt, dx[1]/2._rt, @@ -2323,20 +2380,22 @@ PhysicalParticleContainer::SplitParticles (int lev) split_offset[1] /= ppc_nd[1]; split_offset[2] /= ppc_nd[2]; } - // particle Array Of Structs data - auto& particles = pti.GetArrayOfStructs(); // particle Struct Of Arrays data auto& attribs = pti.GetAttribs(); auto& wp = attribs[PIdx::w ]; auto& uxp = attribs[PIdx::ux]; auto& uyp = attribs[PIdx::uy]; auto& uzp = attribs[PIdx::uz]; + + ParticleTileType& ptile = ParticlesAt(lev, pti); + auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + const long np = pti.numParticles(); for(int i=0; i> attr_int; pctmp_split.AddNParticles(lev, np_split_to_add, - xp, yp, zp, uxp, uyp, uzp, - 1, attr, + xp, + yp, + zp, + uxp, + uyp, + uzp, + 1, + attr, 0, attr_int, - 1, NoSplitParticleID); + 1, LongParticleIds::NoSplitParticleID); // Copy particles from tmp to current particle container constexpr bool local_flag = true; addParticles(pctmp_split,local_flag); @@ -2482,7 +2547,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("PhysicalParticleContainer::PushP()"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -2588,7 +2653,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, } // Externally applied E and B-field in Cartesian co-ordinates - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } @@ -2631,9 +2696,11 @@ PhysicalParticleContainer::PushP (int lev, Real dt, void PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) { - // Inject plasma on level 0. Paticles will be redistributed. + // Inject plasma on level 0. Particles will be redistributed. const int lev=0; - AddPlasma(lev, injection_box); + for (auto const& plasma_injector : plasma_injectors) { + AddPlasma(*plasma_injector, lev, injection_box); + } } /* \brief Inject a flux of particles during the simulation @@ -2641,13 +2708,15 @@ PhysicalParticleContainer::ContinuousInjection (const RealBox& injection_box) void PhysicalParticleContainer::ContinuousFluxInjection (amrex::Real t, amrex::Real dt) { - if (plasma_injector->doFluxInjection()){ - // Check the optional parameters for start and stop of injection - if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && - ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ + for (auto const& plasma_injector : plasma_injectors) { + if (plasma_injector->doFluxInjection()){ + // Check the optional parameters for start and stop of injection + if ( ((plasma_injector->flux_tmin<0) || (t>=plasma_injector->flux_tmin)) && + ((plasma_injector->flux_tmax<0) || (t< plasma_injector->flux_tmax)) ){ - AddPlasmaFlux(dt); + AddPlasmaFlux(*plasma_injector, dt); + } } } } @@ -2674,7 +2743,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, (gather_lev==(lev )), "Gather buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_push == 0) return; + if (np_to_push == 0) { return; } // Get cell size on gather_lev const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); @@ -2773,7 +2842,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #ifdef WARPX_QED const auto do_sync = m_do_qed_quantum_sync; amrex::Real t_chi_max = 0.0; - if (do_sync) t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } QuantumSynchrotronEvolveOpticalDepth evolve_opt; amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; @@ -2834,45 +2903,308 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, nox, galerkin_interpolation); } - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; + if constexpr (exteb_control == has_exteb) { + getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + } + + scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + +#ifdef WARPX_QED + if (!do_sync) +#endif + { + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, +#ifdef WARPX_QED + t_chi_max, +#endif + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); + } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + + UpdatePosition(xp, yp, zp, ux[ip], uy[ip], uz[ip], dt); + setPosition(ip, xp, yp, zp); + } + } +#endif + +#ifdef WARPX_QED + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } + } +#else + amrex::ignore_unused(qed_control); +#endif + }); +} + +/* \brief Perform the implicit particle push operation in one fused kernel + * The main difference from PushPX is the order of operations: + * - push position by 1/2 dt + * - gather fields + * - push velocity by dt + * - average old and new velocity to get time centered value + * The routines ends with both position and velocity at the half time level. + */ +void +PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, + amrex::FArrayBox const * exfab, + amrex::FArrayBox const * eyfab, + amrex::FArrayBox const * ezfab, + amrex::FArrayBox const * bxfab, + amrex::FArrayBox const * byfab, + amrex::FArrayBox const * bzfab, + amrex::IntVect ngEB, int /*e_is_nodal*/, + long offset, + long np_to_push, + int lev, int gather_lev, + amrex::Real dt, ScaleFields scaleFields, + DtType a_dt_type) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE((gather_lev==(lev-1)) || + (gather_lev==(lev )), + "Gather buffers only work for lev-1"); + // If no particles, do not do anything + if (np_to_push == 0) { return; } + + // Get cell size on gather_lev + const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + + // Get box from which field is gathered. + // If not gathering from the finest level, the box is coarsened. + Box box; + if (lev == gather_lev) { + box = pti.tilebox(); + } else { + const IntVect& ref_ratio = WarpX::RefRatio(gather_lev); + box = amrex::coarsen(pti.tilebox(),ref_ratio); + } + + // Add guard cells to the box. + box.grow(ngEB); + + auto setPosition = SetParticlePosition(pti, offset); + + const auto getExternalEB = GetExternalEBField(pti, offset); + + const amrex::ParticleReal Ex_external_particle = m_E_external_particle[0]; + const amrex::ParticleReal Ey_external_particle = m_E_external_particle[1]; + const amrex::ParticleReal Ez_external_particle = m_E_external_particle[2]; + const amrex::ParticleReal Bx_external_particle = m_B_external_particle[0]; + const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; + const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; + + // Lower corner of tile box physical domain (take into account Galilean shift) + const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + + const Dim3 lo = lbound(box); + + int depos_type = WarpX::current_deposition_algo; + int nox = WarpX::nox; + int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; + + amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + + amrex::Array4 const& ex_arr = exfab->array(); + amrex::Array4 const& ey_arr = eyfab->array(); + amrex::Array4 const& ez_arr = ezfab->array(); + amrex::Array4 const& bx_arr = bxfab->array(); + amrex::Array4 const& by_arr = byfab->array(); + amrex::Array4 const& bz_arr = bzfab->array(); + + amrex::IndexType const ex_type = exfab->box().ixType(); + amrex::IndexType const ey_type = eyfab->box().ixType(); + amrex::IndexType const ez_type = ezfab->box().ixType(); + amrex::IndexType const bx_type = bxfab->box().ixType(); + amrex::IndexType const by_type = byfab->box().ixType(); + amrex::IndexType const bz_type = bzfab->box().ixType(); + + auto& attribs = pti.GetAttribs(); + ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr() + offset; + ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr() + offset; + +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + int do_copy = (m_do_back_transformed_particles && (a_dt_type!=DtType::SecondHalf) ); + CopyParticleAttribs copyAttribs; + if (do_copy) { + copyAttribs = CopyParticleAttribs(pti, tmp_particle_data, offset); + } + + int* AMREX_RESTRICT ion_lev = nullptr; + if (do_field_ionization) { + ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr() + offset; + } + + // Loop over the particles and update their momentum + const amrex::ParticleReal q = this->charge; + const amrex::ParticleReal m = this-> mass; + + const auto pusher_algo = WarpX::particle_pusher_algo; + const auto do_crr = do_classical_radiation_reaction; +#ifdef WARPX_QED + const auto do_sync = m_do_qed_quantum_sync; + amrex::Real t_chi_max = 0.0; + if (do_sync) { t_chi_max = m_shr_p_qs_engine->get_minimum_chi_part(); } + + QuantumSynchrotronEvolveOpticalDepth evolve_opt; + amrex::ParticleReal* AMREX_RESTRICT p_optical_depth_QSR = nullptr; + const bool local_has_quantum_sync = has_quantum_sync(); + if (local_has_quantum_sync) { + evolve_opt = m_shr_p_qs_engine->build_evolve_functor(); + p_optical_depth_QSR = pti.GetAttribs(particle_comps["opticalDepthQSR"]).dataPtr() + offset; + } +#endif + + const auto t_do_not_gather = do_not_gather; + + enum exteb_flags : int { no_exteb, has_exteb }; + enum qed_flags : int { no_qed, has_qed }; + + int exteb_runtime_flag = getExternalEB.isNoOp() ? no_exteb : has_exteb; +#ifdef WARPX_QED + int qed_runtime_flag = (local_has_quantum_sync || do_sync) ? has_qed : no_qed; +#else + int qed_runtime_flag = no_qed; +#endif + + // Using this version of ParallelFor with compile time options + // improves performance when qed or external EB are not used by reducing + // register pressure. + amrex::ParallelFor(TypeList, + CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, + auto qed_control) + { + // Position advance starts from the position at the start of the step + // but uses the most recent velocity. +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal xp = x_n[ip]; + amrex::ParticleReal xp_n = x_n[ip]; +#else + amrex::ParticleReal xp = 0._rt; + amrex::ParticleReal xp_n = 0._rt; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal yp = y_n[ip]; + amrex::ParticleReal yp_n = y_n[ip]; +#else + amrex::ParticleReal yp = 0._rt; + amrex::ParticleReal yp_n = 0._rt; +#endif + amrex::ParticleReal zp = z_n[ip]; + amrex::ParticleReal zp_n = z_n[ip]; + + UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); + setPosition(ip, xp, yp, zp); + + amrex::ParticleReal Exp = Ex_external_particle; + amrex::ParticleReal Eyp = Ey_external_particle; + amrex::ParticleReal Ezp = Ez_external_particle; + amrex::ParticleReal Bxp = Bx_external_particle; + amrex::ParticleReal Byp = By_external_particle; + amrex::ParticleReal Bzp = Bz_external_particle; + + if(!t_do_not_gather){ + // first gather E and B to the particle positions + doGatherShapeNImplicit(xp_n, yp_n, zp_n, xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, + ex_type, ey_type, ez_type, bx_type, by_type, bz_type, + dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, + depos_type ); + } + + // Externally applied E and B-field in Cartesian co-ordinates + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } scaleFields(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp); + if (do_copy) { + // Copy the old x and u for the BTD + copyAttribs(ip); + } + + // The momentum push starts with the velocity at the start of the step + ux[ip] = ux_n[ip]; + uy[ip] = uy_n[ip]; + uz[ip] = uz_n[ip]; + #ifdef WARPX_QED if (!do_sync) #endif { - doParticlePush<0>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, #ifdef WARPX_QED - t_chi_max, + t_chi_max, #endif - dt); + dt); } #ifdef WARPX_QED else { if constexpr (qed_control == has_qed) { - doParticlePush<1>(getPosition, setPosition, copyAttribs, ip, - ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 0, - m, q, pusher_algo, do_crr, do_copy, - t_chi_max, - dt); + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); } } #endif #ifdef WARPX_QED [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto foo_podq = p_optical_depth_QSR; - [[maybe_unused]] auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc if constexpr (qed_control == has_qed) { if (local_has_quantum_sync) { evolve_opt(ux[ip], uy[ip], uz[ip], @@ -2883,13 +3215,19 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, #else amrex::ignore_unused(qed_control); #endif + + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + }); } void PhysicalParticleContainer::InitIonizationModule () { - if (!do_field_ionization) return; + if (!do_field_ionization) { return; } const ParmParse pp_species_name(species_name); if (charge != PhysConst::q_e){ ablastr::warn_manager::WMRecordWarning("Species", @@ -2898,10 +3236,15 @@ PhysicalParticleContainer::InitIonizationModule () "overriding user value and setting charge = q_e."); charge = PhysConst::q_e; } + utils::parser::queryWithParser(pp_species_name, "do_adk_correction", do_adk_correction); + utils::parser::queryWithParser( pp_species_name, "ionization_initial_level", ionization_initial_level); pp_species_name.get("ionization_product_species", ionization_product_name); pp_species_name.get("physical_element", physical_element); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + physical_element == "H" || !do_adk_correction, + "Correction to ADK by Zhang et al., PRA 90, 043410 (2014) only works with Hydrogen"); // Add runtime integer component for ionization level AddIntComp("ionizationLevel"); // Get atomic number and ionization energies from file @@ -2936,6 +3279,18 @@ PhysicalParticleContainer::InitIonizationModule () h_ionization_energies.begin(), h_ionization_energies.end(), ionization_energies.begin()); + adk_correction_factors.resize(4); + if (do_adk_correction) { + Vector h_correction_factors(4); + constexpr int offset_corr = 0; // hard-coded: only Hydrogen + for(int i=0; i<4; i++){ + h_correction_factors[i] = table_correction_factors[i+offset_corr]; + } + Gpu::copyAsync(Gpu::hostToDevice, + h_correction_factors.begin(), h_correction_factors.end(), + adk_correction_factors.begin()); + } + Real const* AMREX_RESTRICT p_ionization_energies = ionization_energies.data(); Real * AMREX_RESTRICT p_adk_power = adk_power.data(); Real * AMREX_RESTRICT p_adk_prefactor = adk_prefactor.data(); @@ -2973,13 +3328,19 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti, adk_prefactor.dataPtr(), adk_exp_prefactor.dataPtr(), adk_power.dataPtr(), + adk_correction_factors.dataPtr(), particle_icomps["ionizationLevel"], - ion_atomic_number}; + ion_atomic_number, + do_adk_correction}; } -PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector () +PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector (int i) { - return plasma_injector.get(); + if (i < 0 || i >= static_cast(plasma_injectors.size())) { + return nullptr; + } else { + return plasma_injectors[i].get(); + } } void PhysicalParticleContainer::resample (const int timestep, const bool verbose) diff --git a/Source/Particles/Pusher/CopyParticleAttribs.H b/Source/Particles/Pusher/CopyParticleAttribs.H index 1f0cee67421..4f961fe94aa 100644 --- a/Source/Particles/Pusher/CopyParticleAttribs.H +++ b/Source/Particles/Pusher/CopyParticleAttribs.H @@ -49,9 +49,9 @@ struct CopyParticleAttribs CopyParticleAttribs (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data, long a_offset = 0) noexcept { - if (tmp_particle_data.empty()) return; + if (tmp_particle_data.empty()) { return; } - auto& attribs = a_pti.GetAttribs(); + const auto& attribs = a_pti.GetAttribs(); uxp = attribs[PIdx::ux].dataPtr() + a_offset; uyp = attribs[PIdx::uy].dataPtr() + a_offset; diff --git a/Source/Particles/Pusher/GetAndSetPosition.H b/Source/Particles/Pusher/GetAndSetPosition.H index e4477a2a60d..44641557756 100644 --- a/Source/Particles/Pusher/GetAndSetPosition.H +++ b/Source/Particles/Pusher/GetAndSetPosition.H @@ -30,24 +30,26 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, amrex::ParticleReal& y, amrex::ParticleReal& z) noexcept { -#ifdef WARPX_DIM_RZ - const amrex::ParticleReal theta = p.rdata(T_PIdx::theta); - const amrex::ParticleReal r = p.pos(0); + using namespace amrex::literals; + +#if defined(WARPX_DIM_RZ) + amrex::ParticleReal const theta = p.rdata(T_PIdx::theta); + amrex::ParticleReal const r = p.pos(T_PIdx::x); x = r*std::cos(theta); y = r*std::sin(theta); - z = p.pos(1); -#elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); -#elif WARPX_DIM_XZ - x = p.pos(0); - y = amrex::ParticleReal(0.0); - z = p.pos(1); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_3D) + x = p.pos(PIdx::x); + y = p.pos(PIdx::y); + z = p.pos(PIdx::z); +#elif defined(WARPX_DIM_XZ) + x = p.pos(PIdx::x); + y = 0_prt; + z = p.pos(PIdx::z); #else - x = amrex::ParticleReal(0.0); - y = amrex::ParticleReal(0.0); - z = p.pos(0); + x = 0_prt; + y = 0_prt; + z = p.pos(PIdx::z); #endif } @@ -59,10 +61,19 @@ void get_particle_position (const WarpXParticleContainer::SuperParticleType& p, template struct GetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - const PType* AMREX_RESTRICT m_structs = nullptr; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_3D) + const RType* AMREX_RESTRICT m_x = nullptr; + const RType* AMREX_RESTRICT m_y = nullptr; + const RType* AMREX_RESTRICT m_z = nullptr; +#elif defined(WARPX_DIM_1D_Z) + const RType* AMREX_RESTRICT m_z = nullptr; +#endif + #if defined(WARPX_DIM_RZ) const RType* m_theta = nullptr; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) @@ -84,10 +95,19 @@ struct GetParticlePosition template GetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - const auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) const auto& soa = a_pti.GetStructOfArrays(); + +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -98,24 +118,23 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void operator() (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - const RType r = p.pos(0); + RType const r = m_x[i]; x = r*std::cos(m_theta[i]); y = r*std::sin(m_theta[i]); - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } @@ -127,23 +146,22 @@ struct GetParticlePosition AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void AsStored (const long i, RType& x, RType& y, RType& z) const noexcept { - const PType& p = m_structs[i]; #ifdef WARPX_DIM_RZ - x = p.pos(0); + x = m_x[i]; y = m_theta[i]; - z = p.pos(1); + z = m_z[i]; #elif WARPX_DIM_3D - x = p.pos(0); - y = p.pos(1); - z = p.pos(2); + x = m_x[i]; + y = m_y[i]; + z = m_z[i]; #elif WARPX_DIM_XZ - x = p.pos(0); + x = m_x[i]; y = m_y_default; - z = p.pos(1); + z = m_z[i]; #else x = m_x_default; y = m_y_default; - z = p.pos(0); + z = m_z[i]; #endif } }; @@ -158,10 +176,18 @@ struct GetParticlePosition template struct SetParticlePosition { - using PType = WarpXParticleContainer::ParticleType; using RType = amrex::ParticleReal; - PType* AMREX_RESTRICT m_structs; +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_3D) + RType* AMREX_RESTRICT m_x; + RType* AMREX_RESTRICT m_y; + RType* AMREX_RESTRICT m_z; +#elif defined(WARPX_DIM_1D_Z) + RType* AMREX_RESTRICT m_z; +#endif #if defined(WARPX_DIM_RZ) RType* AMREX_RESTRICT m_theta; #endif @@ -169,10 +195,18 @@ struct SetParticlePosition template SetParticlePosition (const ptiType& a_pti, long a_offset = 0) noexcept { - auto& aos = a_pti.GetArrayOfStructs(); - m_structs = aos().dataPtr() + a_offset; -#if defined(WARPX_DIM_RZ) auto& soa = a_pti.GetStructOfArrays(); +#if defined(WARPX_DIM_RZ) || defined(WARPX_DIM_XZ) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_3D) + m_x = soa.GetRealData(PIdx::x).dataPtr() + a_offset; + m_y = soa.GetRealData(PIdx::y).dataPtr() + a_offset; + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#elif defined(WARPX_DIM_1D_Z) + m_z = soa.GetRealData(PIdx::z).dataPtr() + a_offset; +#endif +#if defined(WARPX_DIM_RZ) m_theta = soa.GetRealData(T_PIdx::theta).dataPtr() + a_offset; #endif } @@ -190,17 +224,17 @@ struct SetParticlePosition #endif #ifdef WARPX_DIM_RZ m_theta[i] = std::atan2(y, x); - m_structs[i].pos(0) = std::sqrt(x*x + y*y); - m_structs[i].pos(1) = z; + m_x[i] = std::sqrt(x*x + y*y); + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } @@ -218,18 +252,18 @@ struct SetParticlePosition amrex::ignore_unused(x,y); #endif #ifdef WARPX_DIM_RZ - m_structs[i].pos(0) = x; + m_x[i] = x; m_theta[i] = y; - m_structs[i].pos(1) = z; + m_z[i] = z; #elif WARPX_DIM_3D - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = y; - m_structs[i].pos(2) = z; + m_x[i] = x; + m_y[i] = y; + m_z[i] = z; #elif WARPX_DIM_XZ - m_structs[i].pos(0) = x; - m_structs[i].pos(1) = z; + m_x[i] = x; + m_z[i] = z; #else - m_structs[i].pos(0) = z; + m_z[i] = z; #endif } }; diff --git a/Source/Particles/Pusher/PushSelector.H b/Source/Particles/Pusher/PushSelector.H index f6fa3ba164f..4a82e582bfb 100644 --- a/Source/Particles/Pusher/PushSelector.H +++ b/Source/Particles/Pusher/PushSelector.H @@ -12,7 +12,6 @@ #include "Particles/Pusher/UpdateMomentumBorisWithRadiationReaction.H" #include "Particles/Pusher/UpdateMomentumHigueraCary.H" #include "Particles/Pusher/UpdateMomentumVay.H" -#include "Particles/Pusher/UpdatePosition.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -21,56 +20,45 @@ #include /** - * \brief Push position and momentum for a single particle + * \brief Push momentum for a single particle * * \tparam do_sync Whether to include quantum synchrotron radiation (QSR) - * \param GetPosition A functor for returning the particle position. - * \param SetPosition A functor for setting the particle position. - * \param copyAttribs A functor for storing the old u and x - * \param i The index of the particle to work on * \param ux, uy, uz Particle momentum * \param Ex, Ey, Ez Electric field on particles. * \param Bx, By, Bz Magnetic field on particles. - * \param ion_lev Ionization level of this particle (0 if ioniziation not on) + * \param ion_lev Ionization level of this particle (0 if ionization not on) * \param m Mass of this species. * \param a_q Charge of this species. * \param pusher_algo 0: Boris, 1: Vay, 2: HigueraCary * \param do_crr Whether to do the classical radiation reaction - * \param do_copy Whether to copy the old x and u for the BTD * \param t_chi_max Cutoff chi for QSR * \param dt Time step size */ template AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void doParticlePush(const GetParticlePosition& GetPosition, - const SetParticlePosition& SetPosition, - const CopyParticleAttribs& copyAttribs, - const long i, - amrex::ParticleReal& ux, - amrex::ParticleReal& uy, - amrex::ParticleReal& uz, - const amrex::ParticleReal Ex, - const amrex::ParticleReal Ey, - const amrex::ParticleReal Ez, - const amrex::ParticleReal Bx, - const amrex::ParticleReal By, - const amrex::ParticleReal Bz, - const int ion_lev, - const amrex::ParticleReal m, - const amrex::ParticleReal a_q, - const int pusher_algo, - const int do_crr, - const int do_copy, +void doParticleMomentumPush(amrex::ParticleReal& ux, + amrex::ParticleReal& uy, + amrex::ParticleReal& uz, + const amrex::ParticleReal Ex, + const amrex::ParticleReal Ey, + const amrex::ParticleReal Ez, + const amrex::ParticleReal Bx, + const amrex::ParticleReal By, + const amrex::ParticleReal Bz, + const int ion_lev, + const amrex::ParticleReal m, + const amrex::ParticleReal a_q, + const int pusher_algo, + const int do_crr, #ifdef WARPX_QED - const amrex::Real t_chi_max, + const amrex::Real t_chi_max, #endif - const amrex::Real dt) + const amrex::Real dt) { amrex::ParticleReal qp = a_q; - if (ion_lev) { qp *= ion_lev; } + qp *= ion_lev; - if (do_copy) copyAttribs(i); if (do_crr) { #ifdef WARPX_QED amrex::ignore_unused(t_chi_max); @@ -88,10 +76,6 @@ void doParticlePush(const GetParticlePosition& GetPosition, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); } - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else #endif { @@ -99,35 +83,19 @@ void doParticlePush(const GetParticlePosition& GetPosition, UpdateMomentumBorisWithRadiationReaction(ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } } else if (pusher_algo == ParticlePusherAlgo::Boris) { UpdateMomentumBoris( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::Vay) { UpdateMomentumVay( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } else if (pusher_algo == ParticlePusherAlgo::HigueraCary) { UpdateMomentumHigueraCary( ux, uy, uz, Ex, Ey, Ez, Bx, By, Bz, qp, m, dt); - amrex::ParticleReal x, y, z; - GetPosition(i, x, y, z); - UpdatePosition(x, y, z, ux, uy, uz, dt ); - SetPosition(i, x, y, z); } //else { // amrex::Abort("Unknown particle pusher"); // } diff --git a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H index dc998caee34..8da3a8cfb2c 100644 --- a/Source/Particles/Pusher/UpdateMomentumHigueraCary.H +++ b/Source/Particles/Pusher/UpdateMomentumHigueraCary.H @@ -46,7 +46,7 @@ void UpdateMomentumHigueraCary( const T sigma = gamma - betam; // Get u* const T ust = (umx*betax + umy*betay + umz*betaz)*invclight; - // Get new gamma inversed + // Get new gamma inverse gamma = 1._prt/std::sqrt(0.5_prt*(sigma + std::sqrt(sigma*sigma + 4._prt*(betam + ust*ust)) )); // Compute t const T tx = gamma*betax; diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 4bce5c10211..6e2e82632ee 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -15,8 +15,12 @@ #include + /** \brief Push the particle's positions over one timestep, - * given the value of its momenta `ux`, `uy`, `uz` */ + * given the value of its momenta `ux`, `uy`, `uz`. + * This uses the standard leapfrog algorithm + * x^{n+1} - x^{n} = dt*u^{n+1/2}/gamma^{n+1/2} + */ AMREX_GPU_HOST_DEVICE AMREX_INLINE void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, @@ -42,4 +46,43 @@ void UpdatePosition(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::Parti z += uz * inv_gamma * dt; } +/** \brief Push the particle's positions over one timestep, + * given the value of its momenta `ux`, `uy`, `uz`. + * The implicit version is the Crank-Nicolson scheme, + * x^{n+1} - x^{n} = dt*(u^{n+1} + u^{n})/(gamma^{n+1} + gamma^{n}) + * See Eqs. 15 and 17 in Chen, JCP 407 (2020) 109228. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z, + const amrex::ParticleReal ux_n, const amrex::ParticleReal uy_n, const amrex::ParticleReal uz_n, + const amrex::ParticleReal ux, const amrex::ParticleReal uy, const amrex::ParticleReal uz, + const amrex::Real dt ) +{ + using namespace amrex::literals; + + constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); + + // Compute inverse Lorentz factor, the average of gamma at time levels n and n+1 + // The ux,uy,uz are the velocities at time level n+1/2 + const amrex::ParticleReal ux_np1 = 2._prt*ux - ux_n; + const amrex::ParticleReal uy_np1 = 2._prt*uy - uy_n; + const amrex::ParticleReal uz_np1 = 2._prt*uz - uz_n; + const amrex::ParticleReal gamma_n = std::sqrt(1._prt + (ux_n*ux_n + uy_n*uy_n + uz_n*uz_n)*inv_c2); + const amrex::ParticleReal gamma_np1 = std::sqrt(1._prt + (ux_np1*ux_np1 + uy_np1*uy_np1 + uz_np1*uz_np1)*inv_c2); + const amrex::ParticleReal inv_gamma = 2.0_prt/(gamma_n + gamma_np1); + + // Update positions over one time step +#if (AMREX_SPACEDIM >= 2) + x += ux * inv_gamma * dt; +#else + amrex::ignore_unused(x); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) // RZ pushes particles in 3D + y += uy * inv_gamma * dt; +#else + amrex::ignore_unused(y); +#endif + z += uz * inv_gamma * dt; +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index e3910286c91..5dc6a458f97 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -60,8 +60,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto& ptile = pc->ParticlesAt(lev, pti); auto& soa = ptile.GetStructOfArrays(); amrex::ParticleReal * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); - WarpXParticleContainer::ParticleType * const AMREX_RESTRICT - particle_ptr = ptile.GetArrayOfStructs()().data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); // Using this function means that we must loop over the cells in the ParallelFor. In the case // of the leveling thinning algorithm, it would have possibly been more natural and more @@ -71,8 +70,8 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); const auto n_cells = static_cast(bins.numBins()); - const auto indices = bins.permutationPtr(); - const auto cell_offsets = bins.offsetsPtr(); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); const amrex::Real target_ratio = m_target_ratio; const int min_ppc = m_min_ppc; @@ -89,8 +88,9 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // do nothing for cells with less particles than min_ppc // (this intentionally includes skipping empty cells, too) - if (cell_numparts < min_ppc) + if (cell_numparts < min_ppc) { return; + } amrex::Real average_weight = 0._rt; // First loop over cell particles to compute average particle weight in the cell @@ -113,7 +113,7 @@ void LevelingThinning::operator() (WarpXParIter& pti, const int lev, // Remove particle with probability 1 - particle_weight/level_weight if (random_number > w[indices[i]]/level_weight) { - particle_ptr[indices[i]].id() = -1; + idcpu[indices[i]] = amrex::ParticleIdCpus::Invalid; } // Set particle weight to level weight otherwise else diff --git a/Source/Particles/RigidInjectedParticleContainer.H b/Source/Particles/RigidInjectedParticleContainer.H index acbe3a0493d..bc20420ea6e 100644 --- a/Source/Particles/RigidInjectedParticleContainer.H +++ b/Source/Particles/RigidInjectedParticleContainer.H @@ -9,6 +9,7 @@ #define WARPX_RigidInjectedParticleContainer_H_ #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Particles/Gather/ScaleFields.H" #include "PhysicalParticleContainer.H" @@ -84,7 +85,8 @@ public: amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, - bool skip_deposition=false ) override; + bool skip_deposition=false, + PushType push_type=PushType::Explicit) override; void PushPX (WarpXParIter& pti, amrex::FArrayBox const * exfab, @@ -122,7 +124,7 @@ private: amrex::Vector zinject_plane_levels; - // Temporary quantites + // Temporary quantities amrex::ParticleReal zinject_plane_lev; amrex::ParticleReal zinject_plane_lev_previous; bool done_injecting_lev; diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 4e4d644123b..d50851ffb1b 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -117,9 +117,9 @@ RigidInjectedParticleContainer::RemapParticles() for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { const auto& attribs = pti.GetAttribs(); - const auto uxp = attribs[PIdx::ux].dataPtr(); - const auto uyp = attribs[PIdx::uy].dataPtr(); - const auto uzp = attribs[PIdx::uz].dataPtr(); + const auto *const uxp = attribs[PIdx::ux].dataPtr(); + const auto *const uyp = attribs[PIdx::uy].dataPtr(); + const auto *const uzp = attribs[PIdx::uz].dataPtr(); const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -299,7 +299,8 @@ RigidInjectedParticleContainer::Evolve (int lev, MultiFab* rho, MultiFab* crho, const MultiFab* cEx, const MultiFab* cEy, const MultiFab* cEz, const MultiFab* cBx, const MultiFab* cBy, const MultiFab* cBz, - Real t, Real dt, DtType a_dt_type, bool skip_deposition) + Real t, Real dt, DtType a_dt_type, bool skip_deposition, + PushType push_type) { // Update location of injection plane in the boosted frame @@ -307,7 +308,7 @@ RigidInjectedParticleContainer::Evolve (int lev, zinject_plane_levels[lev] -= dt*WarpX::beta_boost*PhysConst::c; zinject_plane_lev = zinject_plane_levels[lev]; - // Set the done injecting flag whan the inject plane moves out of the + // Set the done injecting flag when the inject plane moves out of the // simulation domain. // It is much easier to do this check, rather than checking if all of the // particles have crossed the inject plane. @@ -324,7 +325,7 @@ RigidInjectedParticleContainer::Evolve (int lev, rho, crho, cEx, cEy, cEz, cBx, cBy, cBz, - t, dt, a_dt_type, skip_deposition); + t, dt, a_dt_type, skip_deposition, push_type); } void @@ -334,7 +335,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, { WARPX_PROFILE("RigidInjectedParticleContainer::PushP"); - if (do_not_push) return; + if (do_not_push) { return; } const std::array& dx = WarpX::CellSize(std::max(lev,0)); @@ -447,7 +448,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); - [[maybe_unused]] auto& getExternalEB_tmp = getExternalEB; + [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; if constexpr (exteb_control == has_exteb) { getExternalEB(ip, Exp, Eyp, Ezp, Bxp, Byp, Bzp); } diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 60e54b5d59b..7c56da457ed 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -16,7 +16,7 @@ /** * Compute shape factor and return index of leftmost cell where * particle writes. - * Specializations are defined for orders 0 to 3 (using "if constexpr"). + * Specializations are defined for orders 0 to 4 (using "if constexpr"). * Shape factor functors may be evaluated with double arguments * in current deposition to ensure that current deposited by * particles that move only a small distance is still resolved. @@ -64,6 +64,17 @@ struct Compute_shape_factor // index of the leftmost cell where particle deposits return j-1; } + else if constexpr (depos_order == 4){ + const auto j = static_cast(xmid + T(0.5)); + const T xint = xmid - T(j); + sx[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + // index of the leftmost cell where particle deposits + return j-2; + } else{ WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shape_factor"); amrex::ignore_unused(sx, xmid); @@ -77,7 +88,7 @@ struct Compute_shape_factor /** * Compute shifted shape factor and return index of leftmost cell where * particle writes, for Esirkepov algorithm. - * Specializations are defined below for orders 1, 2 and 3 (using "if constexpr"). + * Specializations are defined below for orders 1, 2, 3, and 4 (using "if constexpr"). */ template struct Compute_shifted_shape_factor @@ -89,8 +100,14 @@ struct Compute_shifted_shape_factor const T x_old, const int i_new) const { - if constexpr (depos_order == 1){ - const auto i = static_cast(x_old); + if constexpr (depos_order == 0){ + const auto i = static_cast(std::floor(x_old + T(0.5))); + const int i_shift = i - i_new; + sx[1+i_shift] = T(1.0); + return i; + } + else if constexpr (depos_order == 1){ + const auto i = static_cast(std::floor(x_old)); const int i_shift = i - i_new; const T xint = x_old - T(i); sx[1+i_shift] = T(1.0) - xint; @@ -110,7 +127,7 @@ struct Compute_shifted_shape_factor else if constexpr (depos_order == 3){ const auto i = static_cast(x_old); const int i_shift = i - (i_new + 1); - const T xint = x_old - i; + const T xint = x_old - T(i); sx[1+i_shift] = (T(1.0))/(T(6.0))*(T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - xint); sx[2+i_shift] = (T(2.0))/(T(3.0)) - xint*xint*(T(1.0) - xint/(T(2.0))); sx[3+i_shift] = (T(2.0))/(T(3.0)) - (T(1.0) - xint)*(T(1.0) - xint)*(T(1.0) - T(0.5)*(T(1.0) - xint)); @@ -118,6 +135,18 @@ struct Compute_shifted_shape_factor // index of the leftmost cell where particle deposits return i - 1; } + else if constexpr (depos_order == 4){ + const auto i = static_cast(x_old + T(0.5)); + const int i_shift = i - (i_new + 2); + const T xint = x_old - T(i); + sx[1+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint)*(T(0.5) - xint); + sx[2+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) + xint - xint*xint)); + sx[3+i_shift] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint*xint*(xint*xint - T(2.5))); + sx[4+i_shift] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint + T(4.0)*xint*xint*(T(1.5) - xint - xint*xint)); + sx[5+i_shift] = (T(1.0))/(T(24.0))*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5) + xint)*(T(0.5)+xint); + // index of the leftmost cell where particle deposits + return i - 2; + } else{ WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shifted_shape_factor"); amrex::ignore_unused(sx, x_old, i_new); @@ -126,4 +155,92 @@ struct Compute_shifted_shape_factor } }; +/** + * Compute shape factors for two positions that are within + * half a grid cell of the same cell interface and return the common + * index of the leftmost cell where particle writes, which is correctly + * determined by the average of the positions. + * This is used for computing the segment weights transverse to the + * current density direction in the Villasenor deposition algorithm. + */ +template +struct Compute_shape_factor_pair +{ + template< typename T > + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + int operator()( + T* const sx_old, + T* const sx_new, + T xold, + T xnew) const + { + const T xmid = T(0.5)*(xnew + xold); + if constexpr (depos_order == 1){ + const auto j = static_cast(xmid); + const T xint_old = xold - T(j); + sx_old[0] = T(1.0) - xint_old; + sx_old[1] = xint_old; + // + const T xint_new = xnew - T(j); + sx_new[0] = T(1.0) - xint_new; + sx_new[1] = xint_new; + return j; + } + else if constexpr (depos_order == 2){ + const auto j = static_cast(xmid + T(0.5)); + const T xint_old = xold - T(j); + sx_old[0] = T(0.5)*(T(0.5) - xint_old)*(T(0.5) - xint_old); + sx_old[1] = T(0.75) - xint_old*xint_old; + sx_old[2] = T(0.5)*(T(0.5) + xint_old)*(T(0.5) + xint_old); + // + const T xint_new = xnew - T(j); + sx_new[0] = T(0.5)*(T(0.5) - xint_new)*(T(0.5) - xint_new); + sx_new[1] = T(0.75) - xint_new*xint_new; + sx_new[2] = T(0.5)*(T(0.5) + xint_new)*(T(0.5) + xint_new); + // index of the leftmost cell where particle deposits + return j-1; + } + else if constexpr (depos_order == 3){ + const auto j = static_cast(xmid); + const T xint_old = xold - T(j); + sx_old[0] = T(1.0)/T(6.0)*(T(1.0) - xint_old)*(T(1.0) - xint_old)*(T(1.0) - xint_old); + sx_old[1] = T(2.0)/T(3.0) - xint_old*xint_old*(T(1.0) - xint_old/(T(2.0))); + sx_old[2] = T(2.0)/T(3.0) - (T(1.0) - xint_old)*(T(1.0) - xint_old)*(T(1.0) - T(0.5)*(T(1.0) - xint_old)); + sx_old[3] = T(1.0)/T(6.0)*xint_old*xint_old*xint_old; + // + const T xint_new = xnew - T(j); + sx_new[0] = T(1.0)/T(6.0)*(T(1.0) - xint_new)*(T(1.0) - xint_new)*(T(1.0) - xint_new); + sx_new[1] = T(2.0)/T(3.0) - xint_new*xint_new*(T(1.0) - xint_new/(T(2.0))); + sx_new[2] = T(2.0)/T(3.0) - (T(1.0) - xint_new)*(T(1.0) - xint_new)*(T(1.0) - T(0.5)*(T(1.0) - xint_new)); + sx_new[3] = T(1.0)/T(6.0)*xint_new*xint_new*xint_new; + // index of the leftmost cell where particle deposits + return j-1; + } + else if constexpr (depos_order == 4){ + const auto j = static_cast(xmid + T(0.5)); + const T xint_old = xold - T(j); + sx_old[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint_old)*(T(0.5) - xint_old)*(T(0.5) - xint_old)*(T(0.5) - xint_old); + sx_old[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint_old + T(4.0)*xint_old*xint_old*(T(1.5) + xint_old - xint_old*xint_old)); + sx_old[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint_old*xint_old*(xint_old*xint_old - T(2.5))); + sx_old[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint_old + T(4.0)*xint_old*xint_old*(T(1.5) - xint_old - xint_old*xint_old)); + sx_old[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint_old)*(T(0.5) + xint_old)*(T(0.5) + xint_old)*(T(0.5)+xint_old); + // + const T xint_new = xnew - T(j); + sx_new[0] = (T(1.0))/(T(24.0))*(T(0.5) - xint_new)*(T(0.5) - xint_new)*(T(0.5) - xint_new)*(T(0.5) - xint_new); + sx_new[1] = (T(1.0))/(T(24.0))*(T(4.75) - T(11.0)*xint_new + T(4.0)*xint_new*xint_new*(T(1.5) + xint_new - xint_new*xint_new)); + sx_new[2] = (T(1.0))/(T(24.0))*(T(14.375) + T(6.0)*xint_new*xint_new*(xint_new*xint_new - T(2.5))); + sx_new[3] = (T(1.0))/(T(24.0))*(T(4.75) + T(11.0)*xint_new + T(4.0)*xint_new*xint_new*(T(1.5) - xint_new - xint_new*xint_new)); + sx_new[4] = (T(1.0))/(T(24.0))*(T(0.5) + xint_new)*(T(0.5) + xint_new)*(T(0.5) + xint_new)*(T(0.5)+xint_new); + // + // index of the leftmost cell where particle deposits + return j-2; + } + else{ + WARPX_ABORT_WITH_MESSAGE("Unknown particle shape selected in Compute_shape_factor_pair"); + amrex::ignore_unused(sx_old, sx_new, xold, xnew); + } + return 0; + } +}; + #endif // SHAPEFACTORS_H_ diff --git a/Source/Particles/Sorting/Partition.cpp b/Source/Particles/Sorting/Partition.cpp index 18bebed185f..9f92739a5d5 100644 --- a/Source/Particles/Sorting/Partition.cpp +++ b/Source/Particles/Sorting/Partition.cpp @@ -10,7 +10,6 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include #include #include #include @@ -61,7 +60,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // Initialize temporary arrays Gpu::DeviceVector inexflag; inexflag.resize(np); - Gpu::DeviceVector pid; + Gpu::DeviceVector pid; pid.resize(np); // First, partition particles into the larger buffer @@ -76,7 +75,7 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - Find the indices that reorder particles so that the last particles // are in the larger buffer fillWithConsecutiveIntegers( pid ); - auto const sep = stablePartition( pid.begin(), pid.end(), inexflag ); + auto *const sep = stablePartition( pid.begin(), pid.end(), inexflag ); // At the end of this step, `pid` contains the indices that should be used to // reorder the particles, and `sep` is the position in the array that // separates the particles that deposit/gather on the fine patch (first part) @@ -109,8 +108,8 @@ PhysicalParticleContainer::PartitionParticlesInBuffers( // - For each particle in the large buffer, find whether it is in // the smaller buffer, by looking up the mask. Store the answer in `inexflag`. amrex::ParallelFor( np - n_fine, - fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, n_fine) ); - auto const sep2 = stablePartition( sep, pid.end(), inexflag ); + fillBufferFlagRemainingParticles(pti, bmasks, inexflag, Geom(lev), pid, int(n_fine)) ); + auto *const sep2 = stablePartition( sep, pid.end(), inexflag ); if (bmasks == gather_masks) { nfine_gather = iteratorDistance(pid.begin(), sep2); diff --git a/Source/Particles/Sorting/SortingUtils.H b/Source/Particles/Sorting/SortingUtils.H index ac2c63e88f8..ba7761bf48a 100644 --- a/Source/Particles/Sorting/SortingUtils.H +++ b/Source/Particles/Sorting/SortingUtils.H @@ -12,6 +12,7 @@ #include #include +#include /** \brief Fill the elements of the input vector with consecutive integer, @@ -19,7 +20,7 @@ * * \param[inout] v Vector of integers, to be filled by this routine */ -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ); /** \brief Find the indices that would reorder the elements of `predicate` * so that the elements with non-zero value precede the other elements @@ -41,7 +42,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, int const* AMREX_RESTRICT predicate_ptr = predicate.dataPtr(); int N = static_cast(std::distance(index_begin, index_end)); auto num_true = amrex::StablePartition(&(*index_begin), N, - [predicate_ptr] AMREX_GPU_DEVICE (long i) { return predicate_ptr[i]; }); + [predicate_ptr] AMREX_GPU_DEVICE (int i) { return predicate_ptr[i]; }); ForwardIterator sep = index_begin; std::advance(sep, num_true); @@ -49,7 +50,7 @@ ForwardIterator stablePartition(ForwardIterator const index_begin, // On CPU: Use std library ForwardIterator const sep = std::stable_partition( index_begin, index_end, - [&predicate](long i) { return predicate[i]; } + [&predicate](int i) { return predicate[i]; } ); #endif return sep; @@ -88,7 +89,7 @@ class fillBufferFlag // Extract simple structure that can be used directly on the GPU m_domain{geom.Domain()}, m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()} { for (int idim=0; idim m_buffer_mask; amrex::GpuArray m_prob_lo; amrex::GpuArray m_inv_cell_size; @@ -141,12 +140,12 @@ class fillBufferFlagRemainingParticles amrex::iMultiFab const* bmasks, amrex::Gpu::DeviceVector& inexflag, amrex::Geometry const& geom, - amrex::Gpu::DeviceVector const& particle_indices, - long const start_index ) : + amrex::Gpu::DeviceVector const& particle_indices, + int start_index ) : m_domain{geom.Domain()}, // Extract simple structure that can be used directly on the GPU m_inexflag_ptr{inexflag.dataPtr()}, - m_particles{pti.GetArrayOfStructs().data()}, + m_ptd{pti.GetParticleTile().getConstParticleTileData()}, m_buffer_mask{(*bmasks)[pti].array()}, m_start_index{start_index}, m_indices_ptr{particle_indices.dataPtr()} @@ -159,11 +158,11 @@ class fillBufferFlagRemainingParticles AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - void operator()( const long i ) const { + void operator()( const int i ) const { // Select a particle - auto const& p = m_particles[m_indices_ptr[i+m_start_index]]; + auto const j = m_indices_ptr[i+m_start_index]; // Find the index of the cell where this particle is located - amrex::IntVect const iv = amrex::getParticleCell( p, + amrex::IntVect const iv = amrex::getParticleCell( m_ptd, j, m_prob_lo, m_inv_cell_size, m_domain ); // Find the value of the buffer flag in this cell and // store it at the corresponding particle position in the array `inexflag` @@ -175,10 +174,10 @@ class fillBufferFlagRemainingParticles amrex::GpuArray m_inv_cell_size; amrex::Box m_domain; int* m_inexflag_ptr; - WarpXParticleContainer::ParticleType const* m_particles; + WarpXParticleContainer::ParticleTileType::ConstParticleTileDataType const m_ptd; amrex::Array4 m_buffer_mask; - long const m_start_index; - long const* m_indices_ptr; + int const m_start_index; + int const* m_indices_ptr; }; /** \brief Functor that copies the elements of `src` into `dst`, @@ -195,7 +194,7 @@ class copyAndReorder copyAndReorder( amrex::Gpu::DeviceVector const& src, amrex::Gpu::DeviceVector& dst, - amrex::Gpu::DeviceVector const& indices ): + amrex::Gpu::DeviceVector const& indices ): // Extract simple structure that can be used directly on the GPU m_src_ptr{src.dataPtr()}, m_dst_ptr{dst.dataPtr()}, @@ -203,14 +202,14 @@ class copyAndReorder {} AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void operator()( const long ip ) const { + void operator()( const int ip ) const { m_dst_ptr[ip] = m_src_ptr[ m_indices_ptr[ip] ]; } private: T const* m_src_ptr; T* m_dst_ptr; - long const* m_indices_ptr; + int const* m_indices_ptr; }; #endif // WARPX_PARTICLES_SORTING_SORTINGUTILS_H_ diff --git a/Source/Particles/Sorting/SortingUtils.cpp b/Source/Particles/Sorting/SortingUtils.cpp index 699119e8e18..cd4b6a13c76 100644 --- a/Source/Particles/Sorting/SortingUtils.cpp +++ b/Source/Particles/Sorting/SortingUtils.cpp @@ -8,7 +8,7 @@ #include "SortingUtils.H" -void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) +void fillWithConsecutiveIntegers( amrex::Gpu::DeviceVector& v ) { #ifdef AMREX_USE_GPU // On GPU: Use amrex diff --git a/Source/Particles/SpeciesPhysicalProperties.cpp b/Source/Particles/SpeciesPhysicalProperties.cpp index 118db98f2c2..20ca5ed48b9 100644 --- a/Source/Particles/SpeciesPhysicalProperties.cpp +++ b/Source/Particles/SpeciesPhysicalProperties.cpp @@ -126,8 +126,9 @@ namespace { constexpr auto quiet_NaN = std::numeric_limits::quiet_NaN(); - // The atomic mass data below is from this NIST page + // The atomic mass data below is from these NIST pages // https://physics.nist.gov/cgi-bin/Compositions/stand_alone.pl?ele=&ascii=ascii2&isotype=some + // https://physics.nist.gov/cgi-bin/cuu/Value?malu const std::map species_to_properties { @@ -177,7 +178,7 @@ namespace { amrex::Real(4.00260325413) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::alpha, Properties{ - amrex::Real(4.00260325413) * PhysConst::m_u - amrex::Real(2) * PhysConst::m_e, + amrex::Real(4.001506179127) * PhysConst::m_u, amrex::Real(2) * PhysConst::q_e}}, {PhysicalSpecies::lithium, Properties{ amrex::Real(6.967) * PhysConst::m_u, diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 8ae6e4ef64b..7d2d5619da9 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -13,6 +13,7 @@ #include "WarpXParticleContainer_fwd.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "Initialization/PlasmaInjector.H" #include "Particles/ParticleBoundaries.H" #include "SpeciesPhysicalProperties.H" @@ -48,10 +49,10 @@ class WarpXParIter - : public amrex::ParIter<0,0,PIdx::nattribs> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs>::ParIter; + using amrex::ParIterSoA::ParIterSoA; WarpXParIter (ContainerType& pc, int level); @@ -88,13 +89,14 @@ public: * particle container classes (that store a collection of particles) derive. Derived * classes can be used for plasma particles, photon particles, or non-physical * particles (e.g., for the laser antenna). - * It derives from amrex::ParticleContainer<0,0,PIdx::nattribs>, where the - * template arguments stand for the number of int and amrex::Real SoA and AoS - * data in amrex::Particle. - * - AoS amrex::Real: x, y, z (default), 0 additional (first template - * parameter) - * - AoS int: id, cpu (default), 0 additional (second template parameter) - * - SoA amrex::Real: PIdx::nattribs (third template parameter), see PIdx for + * It derives from amrex::ParticleContainerPureSoA, where the + * template arguments stand for the number of int and amrex::Real SoA + * data in amrex::SoAParticle. + * - SoA amrex::Real: positions x, y, z, momentum ux, uy, uz, ... see PIdx for details; + * more can be added at runtime + * - SoA int: 0 attributes by default, but can be added at runtime + * - SoA uint64_t: idcpu, a global 64bit index, with a 40bit local id and a 24bit cpu id + * (both set at creation) * the list. * * WarpXParticleContainer contains the main functions for initialization, @@ -136,7 +138,7 @@ public: * \brief Virtual function that returns a pointer to the plasma injector, * for derived classes that define one (PhysicalParticleContainer). */ - virtual PlasmaInjector* GetPlasmaInjector () { return nullptr; } + virtual PlasmaInjector* GetPlasmaInjector (const int /*i*/) { return nullptr; } /** * Evolve is the central WarpXParticleContainer function that advances @@ -151,7 +153,8 @@ public: amrex::MultiFab* rho, amrex::MultiFab* crho, const amrex::MultiFab* cEx, const amrex::MultiFab* cEy, const amrex::MultiFab* cEz, const amrex::MultiFab* cBx, const amrex::MultiFab* cBy, const amrex::MultiFab* cBz, - amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false) = 0; + amrex::Real t, amrex::Real dt, DtType a_dt_type=DtType::Full, bool skip_deposition=false, + PushType push_type=PushType::Explicit) = 0; virtual void PostRestart () = 0; @@ -162,16 +165,13 @@ public: * class. */ virtual void DefaultInitializeRuntimeAttributes ( - amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>& pinned_tile, - int n_external_attr_real, - int n_external_attr_int, - const amrex::RandomEngine& engine) = 0; + typename ContainerLike::ParticleTileType& pinned_tile, + int n_external_attr_real, + int n_external_attr_int) = 0; /// /// This pushes the particle positions by one half time step. - /// It is used to desynchronize the particles after initializaton + /// It is used to desynchronize the particles after initialization /// or when restarting from a checkpoint. /// void PushX ( amrex::Real dt); @@ -230,7 +230,7 @@ public: amrex::MultiFab* rho, int icomp, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev); @@ -245,12 +245,13 @@ public: amrex::MultiFab* jy, amrex::MultiFab* jz, long offset, - long np_to_depose, + long np_to_deposit, int thread_num, int lev, int depos_lev, amrex::Real dt, - amrex::Real relative_time); + amrex::Real relative_time, + PushType push_type); // If particles start outside of the domain, ContinuousInjection // makes sure that they are initialized when they enter the domain, and @@ -404,6 +405,19 @@ public: */ void defineAllParticleTiles () noexcept; + virtual std::vector getUserIntAttribs () const { return std::vector{}; } + + virtual std::vector getUserRealAttribs () const { return std::vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserIntAttribParser () const { return amrex::Vector{}; } + + virtual amrex::Vector< amrex::Parser* > getUserRealAttribParser () const { return amrex::Vector{}; } + +#ifdef WARPX_QED + virtual BreitWheelerEngine* get_breit_wheeler_engine_ptr () const {return nullptr;} + virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} +#endif + protected: int species_id; @@ -430,6 +444,7 @@ protected: int do_continuous_injection = 0; int do_field_ionization = 0; + int do_adk_correction = 0; int ionization_product; std::string ionization_product_name; int ion_atomic_number; @@ -438,6 +453,8 @@ protected: amrex::Gpu::DeviceVector adk_power; amrex::Gpu::DeviceVector adk_prefactor; amrex::Gpu::DeviceVector adk_exp_prefactor; + /** for correction in Zhang et al., PRA 90, 043410 (2014). a1, a2, a3, Ecrit. */ + amrex::Gpu::DeviceVector adk_correction_factors; std::string physical_element; int do_resampling = 0; @@ -473,6 +490,9 @@ public: using TmpParticles = amrex::Vector >; TmpParticles getTmpParticleData () const noexcept {return tmp_particle_data;} + + int getIonizationInitialLevel () const noexcept {return ionization_initial_level;} + protected: TmpParticles tmp_particle_data; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 98868d1cf14..6e6c0b27c85 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -60,7 +60,10 @@ #include #include #include - +#ifdef AMREX_USE_EB +# include "EmbeddedBoundary/ParticleBoundaryProcess.H" +# include "EmbeddedBoundary/ParticleScraper.H" +#endif #ifdef AMREX_USE_OMP # include @@ -72,13 +75,13 @@ using namespace amrex; WarpXParIter::WarpXParIter (ContainerType& pc, int level) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, MFItInfo().SetDynamic(WarpX::do_dynamic_scheduling)) { } WarpXParIter::WarpXParIter (ContainerType& pc, int level, MFItInfo& info) - : amrex::ParIter<0,0,PIdx::nattribs>(pc, level, + : amrex::ParIterSoA(pc, level, info.SetDynamic(WarpX::do_dynamic_scheduling)) { } @@ -195,52 +198,53 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Redistribute() will move them to proper places. auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); - using PinnedTile = amrex::ParticleTile, - NArrayReal, NArrayInt, - amrex::PinnedArenaAllocator>; + using PinnedTile = typename ContainerLike::ParticleTileType; PinnedTile pinned_tile; pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps()); const std::size_t np = iend-ibegin; #ifdef WARPX_DIM_RZ + amrex::Vector r(np); amrex::Vector theta(np); #endif for (auto i = ibegin; i < iend; ++i) { - ParticleType p; - if (id==-1) - { - p.id() = ParticleType::NextID(); - } else { - p.id() = id; + auto & idcpu_data = pinned_tile.GetStructOfArrays().GetIdCPUData(); + + amrex::Long current_id = id; // copy input + if (id == -1) { + current_id = ParticleType::NextID(); } - p.cpu() = amrex::ParallelDescriptor::MyProc(); + idcpu_data.push_back(amrex::SetParticleIDandCPU(current_id, ParallelDescriptor::MyProc())); + +#ifdef WARPX_DIM_RZ + r[i-ibegin] = std::sqrt(x[i]*x[i] + y[i]*y[i]); + theta[i-ibegin] = std::atan2(y[i], x[i]); +#endif + } + + if (np > 0) + { #if defined(WARPX_DIM_3D) - p.pos(0) = x[i]; - p.pos(1) = y[i]; - p.pos(2) = z[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); + pinned_tile.push_back_real(PIdx::y, y.data() + ibegin, y.data() + iend); + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::ignore_unused(y); #ifdef WARPX_DIM_RZ - theta[i-ibegin] = std::atan2(y[i], x[i]); - p.pos(0) = std::sqrt(x[i]*x[i] + y[i]*y[i]); + pinned_tile.push_back_real(PIdx::x, r.data(), r.data() + np); #else - p.pos(0) = x[i]; + pinned_tile.push_back_real(PIdx::x, x.data() + ibegin, x.data() + iend); #endif - p.pos(1) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #else //AMREX_SPACEDIM == 1 amrex::ignore_unused(x,y); - p.pos(0) = z[i]; + pinned_tile.push_back_real(PIdx::z, z.data() + ibegin, z.data() + iend); #endif - pinned_tile.push_back(p); - } - - if (np > 0) - { - pinned_tile.push_back_real(PIdx::w , attr_real[0].data() + ibegin, attr_real[0].data() + iend); + pinned_tile.push_back_real(PIdx::w, attr_real[0].data() + ibegin, attr_real[0].data() + iend); pinned_tile.push_back_real(PIdx::ux, ux.data() + ibegin, ux.data() + iend); pinned_tile.push_back_real(PIdx::uy, uy.data() + ibegin, uy.data() + iend); pinned_tile.push_back_real(PIdx::uz, uz.data() + ibegin, uz.data() + iend); @@ -279,9 +283,9 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, pinned_tile.push_back_int(j, attr_int[j].data() + ibegin, attr_int[j].data() + iend); } + pinned_tile.resize(np); // Default initialize the other real and integer runtime attributes - DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int, - amrex::RandomEngine{}); + DefaultInitializeRuntimeAttributes(pinned_tile, nattr_real - 1, nattr_int); auto old_np = particle_tile.numParticles(); auto new_np = old_np + pinned_tile.numParticles(); @@ -291,26 +295,32 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Remove particles that are inside the embedded boundaries +#ifdef AMREX_USE_EB + auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); + scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); +#endif + Redistribute(); } /* \brief Current Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param uxp uyp uzp : Array of particle momenta - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param jx jy jz : Full array of current density - * \param offset : Index of first particle for which current is deposited - * \param np_to_depose: Number of particles for which current is deposited. - Particles [offset,offset+np_tp_depose] deposit current - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) - * \param dt : Time step for particle level - * \param relative_time: Time at which to deposit J, relative to the time of the + * \param pti Particle iterator + * \param wp Array of particle weights + * \param uxp uyp uzp Array of particle momenta + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param jx jy jz Full array of current density + * \param offset Index of first particle for which current is deposited + * \param np_to_deposit Number of particles for which current is deposited. + Particles [offset,offset+np_to_deposit] deposit current + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) + * \param dt Time step for particle level + * \param relative_time Time at which to deposit J, relative to the time of the * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. @@ -321,19 +331,19 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, RealVector const & uyp, RealVector const & uzp, int const * const ion_lev, amrex::MultiFab * const jx, amrex::MultiFab * const jy, amrex::MultiFab * const jz, - long const offset, long const np_to_depose, + long const offset, long const np_to_deposit, int const thread_num, const int lev, int const depos_lev, - amrex::Real const dt, amrex::Real const relative_time) + amrex::Real const dt, amrex::Real const relative_time, PushType push_type) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || (depos_lev==(lev )), "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // If user decides not to deposit - if (do_not_deposit) return; + if (do_not_deposit) { return; } // Number of guard cells for local deposition of J const WarpX& warpx = WarpX::GetInstance(); @@ -443,9 +453,10 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // Take into account Galilean shift const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); - if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { + if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { if (WarpX::grid_type == GridType::Collocated) { - WARPX_ABORT_WITH_MESSAGE("The Esirkepov algorithm cannot be used with a collocated grid."); + WARPX_ABORT_WITH_MESSAGE("Charge-conserving current depositions (Esirkepov and Villasenor) cannot be used with a collocated grid."); } } @@ -466,15 +477,14 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, //sort particles by bin WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, + bins.build(ptile.numParticles(), ptd, ntiles, [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int { Box tbox; @@ -502,9 +512,15 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_STOP(blp_get_max_tilesize); // Now pick current deposition algorithm + if (push_type == PushType::Implicit) { + amrex::Abort("Cannot do shared memory deposition with implicit algorithm"); + } if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Esirkepov algorithm"); } + else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Villasenor algorithm"); + } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { WARPX_ABORT_WITH_MESSAGE("Cannot do shared memory deposition with Vay algorithm"); } @@ -512,25 +528,32 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, WARPX_PROFILE_VAR_START(direct_current_dep_kernel); if (WarpX::nox == 1){ doDepositionSharedShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + } else if (WarpX::nox == 4){ + doDepositionSharedShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -538,73 +561,257 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // If not doing shared memory deposition, call normal kernels else { if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov) { - if (WarpX::nox == 1){ - doEsirkepovDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doEsirkepovDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doEsirkepovDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doEsirkepovDepositionShapeN<1>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doEsirkepovDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doEsirkepovDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doEsirkepovDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doChargeConservingDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doChargeConservingDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doChargeConservingDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doChargeConservingDepositionShapeNImplicit<4>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } + } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + if (push_type == PushType::Implicit) { +#if (AMREX_SPACEDIM >= 2) + auto& xp_n = pti.GetAttribs(particle_comps["x_n"]); + const ParticleReal* xp_n_data = xp_n.dataPtr() + offset; +#else + const ParticleReal* xp_n_data = nullptr; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + auto& yp_n = pti.GetAttribs(particle_comps["y_n"]); + const ParticleReal* yp_n_data = yp_n.dataPtr() + offset; +#else + const ParticleReal* yp_n_data = nullptr; +#endif + auto& zp_n = pti.GetAttribs(particle_comps["z_n"]); + const ParticleReal* zp_n_data = zp_n.dataPtr() + offset; + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doVillasenorDepositionShapeNImplicit<1>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doVillasenorDepositionShapeNImplicit<2>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doVillasenorDepositionShapeNImplicit<3>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doVillasenorDepositionShapeNImplicit<4>( + xp_n_data, yp_n_data, zp_n_data, + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } + else { + WARPX_ABORT_WITH_MESSAGE("The Villasenor algorithm can only be used with implicit algorithm."); } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay) { + if (push_type == PushType::Implicit) { + WARPX_ABORT_WITH_MESSAGE("The Vay algorithm cannot be used with implicit algorithm."); + } if (WarpX::nox == 1){ doVayDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doVayDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); } } else { // Direct deposition - if (WarpX::nox == 1){ - doDepositionShapeN<1>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 2){ - doDepositionShapeN<2>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); - } else if (WarpX::nox == 3){ - doDepositionShapeN<3>( - GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, - uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_depose, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + if (push_type == PushType::Explicit) { + if (WarpX::nox == 1){ + doDepositionShapeN<1>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeN<2>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeN<3>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doDepositionShapeN<4>( + GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, + uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } + } else if (push_type == PushType::Implicit) { + auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); + auto& uyp_n = pti.GetAttribs(particle_comps["uy_n"]); + auto& uzp_n = pti.GetAttribs(particle_comps["uz_n"]); + if (WarpX::nox == 1){ + doDepositionShapeNImplicit<1>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 2){ + doDepositionShapeNImplicit<2>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 3){ + doDepositionShapeNImplicit<3>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } else if (WarpX::nox == 4){ + doDepositionShapeNImplicit<4>( + GetPosition, wp.dataPtr() + offset, + uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, + uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, + ion_lev, + jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo); + } } } } @@ -613,9 +820,9 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_j into j WARPX_PROFILE_VAR_START(blp_accumulate); - (*jx)[pti].atomicAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp()); - (*jy)[pti].atomicAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp()); - (*jz)[pti].atomicAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp()); + (*jx)[pti].lockAdd(local_jx[thread_num], tbx, tbx, 0, 0, jx->nComp()); + (*jy)[pti].lockAdd(local_jy[thread_num], tby, tby, 0, 0, jy->nComp()); + (*jz)[pti].lockAdd(local_jz[thread_num], tbz, tbz, 0, 0, jz->nComp()); WARPX_PROFILE_VAR_STOP(blp_accumulate); #endif } @@ -653,7 +860,7 @@ WarpXParticleContainer::DepositCurrent ( DepositCurrent(pti, wp, uxp, uyp, uzp, ion_lev, J[lev][0].get(), J[lev][1].get(), J[lev][2].get(), - 0, np, thread_num, lev, lev, dt, relative_time); + 0, np, thread_num, lev, lev, dt, relative_time, PushType::Explicit); } #ifdef AMREX_USE_OMP } @@ -662,28 +869,28 @@ WarpXParticleContainer::DepositCurrent ( } /* \brief Charge Deposition for thread thread_num - * \param pti : Particle iterator - * \param wp : Array of particle weights - * \param ion_lev : Pointer to array of particle ionization level. This is - required to have the charge of each macroparticle - since q is a scalar. For non-ionizable species, - ion_lev is a null pointer. - * \param rho : Full array of charge density - * \param icomp : Component of rho into which charge is deposited. - 0: old value (before particle push). - 1: new value (after particle push). - * \param offset : Index of first particle for which charge is deposited - * \param np_to_depose: Number of particles for which charge is deposited. - Particles [offset,offset+np_tp_depose) deposit charge - * \param thread_num : Thread number (if tiling) - * \param lev : Level of box that contains particles - * \param depos_lev : Level on which particles deposit (if buffers are used) + * \param pti Particle iterator + * \param wp Array of particle weights + * \param ion_lev Pointer to array of particle ionization level. This is + required to have the charge of each macroparticle + since q is a scalar. For non-ionizable species, + ion_lev is a null pointer. + * \param rho Full array of charge density + * \param icomp Component of rho into which charge is deposited. + 0: old value (before particle push). + 1: new value (after particle push). + * \param offset Index of first particle for which charge is deposited + * \param np_to_deposit Number of particles for which charge is deposited. + Particles [offset,offset+np_to_deposit) deposit charge + * \param thread_num Thread number (if tiling) + * \param lev Level of box that contains particles + * \param depos_lev Level on which particles deposit (if buffers are used) */ void WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const int * const ion_lev, amrex::MultiFab* rho, const int icomp, - const long offset, const long np_to_depose, + const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { if (WarpX::do_shared_mem_charge_deposition) @@ -693,7 +900,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Deposition buffers only work for lev-1"); // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -786,7 +993,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); - amrex::DenseBins bins; + amrex::DenseBins bins; { const Geometry& geom = Geom(lev); const auto dxi = geom.InvCellSizeArray(); @@ -794,16 +1001,15 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; const int ntiles = numTilesInBox(box, true, bin_size); - bins.build(ptile.numParticles(), pstruct_ptr, ntiles, - [=] AMREX_GPU_HOST_DEVICE (const ParticleType& p) -> unsigned int + bins.build(ptile.numParticles(), ptd, ntiles, + [=] AMREX_GPU_HOST_DEVICE (ParticleType const & p) -> unsigned int { Box tbx; auto iv = getParticleCell(p, plo, dxi, domain); @@ -823,24 +1029,24 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const auto domain = geom.Domain(); auto& ptile = ParticlesAt(lev, pti); - auto& aos = ptile.GetArrayOfStructs(); - auto pstruct_ptr = aos().dataPtr(); + auto ptd = ptile.getParticleTileData(); Box box = pti.validbox(); box.grow(ng_rho); const amrex::IntVect bin_size = WarpX::shared_tilesize; - const auto offsets_ptr = bins.offsetsPtr(); - auto tbox_ptr = tboxes.dataPtr(); - auto permutation = bins.permutationPtr(); + auto *const offsets_ptr = bins.offsetsPtr(); + auto *tbox_ptr = tboxes.dataPtr(); + auto *permutation = bins.permutationPtr(); amrex::ParallelFor(bins.numBins(), [=] AMREX_GPU_DEVICE (int ibin) { const auto bin_start = offsets_ptr[ibin]; const auto bin_stop = offsets_ptr[ibin+1]; if (bin_start < bin_stop) { - auto p = pstruct_ptr[permutation[bin_start]]; + // static_cast until https://github.com/AMReX-Codes/amrex/pull/3684 + auto const i = static_cast(permutation[bin_start]); Box tbx; - auto iv = getParticleCell(p, plo, dxi, domain); + auto iv = getParticleCell(ptd, i, plo, dxi, domain); AMREX_ASSERT(box.contains(iv)); [[maybe_unused]] auto tid = getTileIndex(iv, box, true, bin_size, tbx); AMREX_ASSERT(tid == ibin); @@ -857,7 +1063,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - const auto boxes_ptr = tboxes.dataPtr(); + auto *const boxes_ptr = tboxes.dataPtr(); reduce_op.eval(tboxes.size(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { @@ -888,19 +1094,25 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_depose, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + WarpX::n_rz_azimuthal_modes, cost, + WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::shared_tilesize); + } else if (WarpX::nox == 4){ + doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, + rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -908,7 +1120,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho WARPX_PROFILE_VAR_START(blp_accumulate); - (*rho)[pti].atomicAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc); + (*rho)[pti].lockAdd(local_rho[thread_num], tb, tb, 0, icomp*nc, nc); WARPX_PROFILE_VAR_STOP(blp_accumulate); #endif } else { @@ -954,13 +1166,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); ablastr::particles::deposit_charge( - pti, wp, this->charge, ion_lev, - rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, - ng_rho, depos_lev, ref_ratio, - offset, np_to_depose, - icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + pti, wp, this->charge, ion_lev, + rho, local_rho[thread_num], + WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + ng_rho, depos_lev, ref_ratio, + offset, np_to_deposit, + icomp, nc, + cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize ); } } @@ -1014,7 +1226,7 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, { // Reset the rho array if reset is True int const nc = WarpX::ncomps; - if (reset) rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); + if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } // Loop over particle tiles and deposit charge on each level #ifdef AMREX_USE_OMP @@ -1081,8 +1293,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) #else const bool is_PSATD_RZ = false; #endif - if( !is_PSATD_RZ ) + if( !is_PSATD_RZ ) { nba.surroundingNodes(); + } // Number of guard cells for local deposition of rho const WarpX& warpx = WarpX::GetInstance(); @@ -1107,7 +1320,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { for (int lev = 0; lev <= nLevels; ++lev) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { - const auto wp = pti.GetAttribs(PIdx::w).data(); + auto *const wp = pti.GetAttribs(PIdx::w).data(); reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int ip) @@ -1117,7 +1330,7 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { total_charge = get<0>(reduce_data.value()); - if (!local) ParallelDescriptor::ReduceRealSum(total_charge); + if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } total_charge *= this->charge; return total_charge; } @@ -1198,7 +1411,7 @@ std::array WarpXParticleContainer::meanParticleVelocity(bool lo ParallelDescriptor::ReduceLongSum(np_total); } - std::array mean_v; + std::array mean_v = {0,0,0}; if (np_total > 0) { mean_v[0] = vx_total / np_total; mean_v[1] = vy_total / np_total; @@ -1230,7 +1443,7 @@ amrex::ParticleReal WarpXParticleContainer::maxParticleVelocity(bool local) { } } - if (!local) ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); + if (!local) { ParallelAllReduce::Max(max_v, ParallelDescriptor::Communicator()); } return max_v; } @@ -1248,7 +1461,7 @@ WarpXParticleContainer::PushX (int lev, amrex::Real dt) { WARPX_PROFILE("WarpXParticleContainer::PushX()"); - if (do_not_push) return; + if (do_not_push) { return; } amrex::LayoutData* costs = WarpX::getCosts(lev); @@ -1323,15 +1536,15 @@ WarpXParticleContainer::particlePostLocate(ParticleType& p, const ParticleLocData& pld, const int lev) { - if (not do_splitting) return; + if (not do_splitting) { return; } // Tag particle if goes to higher level. // It will be split later in the loop if (pld.m_lev == lev+1 - and p.id() != NoSplitParticleID + and p.id() != amrex::LongParticleIds::NoSplitParticleID and p.id() >= 0) { - p.id() = DoSplitParticleID; + p.id() = amrex::LongParticleIds::DoSplitParticleID; } if (pld.m_lev == lev-1){ @@ -1345,7 +1558,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ WARPX_PROFILE("WarpXParticleContainer::ApplyBoundaryConditions()"); // Periodic boundaries are handled in AMReX code - if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) return; + if (m_boundary_conditions.CheckAll(ParticleBoundaryType::Periodic)) { return; } auto boundary_conditions = m_boundary_conditions.data; @@ -1370,9 +1583,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ const Real zmax = Geom(lev).ProbHi(WARPX_ZINDEX); ParticleTileType& ptile = ParticlesAt(lev, pti); - ParticleType * const pp = ptile.GetArrayOfStructs()().data(); auto& soa = ptile.GetStructOfArrays(); + uint64_t * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); amrex::ParticleReal * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); amrex::ParticleReal * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); amrex::ParticleReal * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); @@ -1381,10 +1594,9 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ amrex::ParallelForRNG( pti.numParticles(), [=] AMREX_GPU_DEVICE (long i, amrex::RandomEngine const& engine) { - ParticleType& p = pp[i]; - // skip particles that are already flagged for removal - if (p.id() < 0) return; + auto pidw = amrex::ParticleIDWrapper{idcpu[i]}; + if (!pidw.is_valid()) { return; } ParticleReal x, y, z; GetPosition.AsStored(i, x, y, z); @@ -1406,7 +1618,7 @@ WarpXParticleContainer::ApplyBoundaryConditions (){ boundary_conditions, engine); if (particle_lost) { - p.id() = -p.id(); + pidw.make_invalid(); } else { SetPosition.AsStored(i, x, y, z); } diff --git a/Source/Python/Particles/ParticleBoundaryBuffer.cpp b/Source/Python/Particles/ParticleBoundaryBuffer.cpp index 2a35faece9b..b04ac75e600 100644 --- a/Source/Python/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Python/Particles/ParticleBoundaryBuffer.cpp @@ -10,13 +10,13 @@ namespace warpx { class BoundaryBufferParIter - : public amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + : public amrex::ParIterSoA { public: - using amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>::ParIter; + using amrex::ParIterSoA::ParIterSoA; BoundaryBufferParIter(ContainerType& pc, int level) : - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator>(pc, level) {} + amrex::ParIterSoA(pc, level) {} }; } @@ -24,9 +24,9 @@ void init_BoundaryBufferParIter (py::module& m) { py::class_< warpx::BoundaryBufferParIter, - amrex::ParIter<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParIterSoA >(m, "BoundaryBufferParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level") ) ; diff --git a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp index d048b4d9c8f..21dd6a9d364 100644 --- a/Source/Python/Particles/PinnedMemoryParticleContainer.cpp +++ b/Source/Python/Particles/PinnedMemoryParticleContainer.cpp @@ -13,11 +13,19 @@ void init_PinnedMemoryParticleContainer (py::module& m) { py::class_< PinnedMemoryParticleContainer, - amrex::ParticleContainer<0,0,PIdx::nattribs,0,amrex::PinnedArenaAllocator> + amrex::ParticleContainerPureSoA > pmpc (m, "PinnedMemoryParticleContainer"); pmpc - .def("num_int_comps", - [](PinnedMemoryParticleContainer& pc) { return pc.NumIntComps(); } + .def_property_readonly("real_comp_names", + [](PinnedMemoryParticleContainer& pc) + { + return pc.getParticleComps(); + } ) - ; + .def_property_readonly("int_comp_names", + [](PinnedMemoryParticleContainer& pc) + { + return pc.getParticleiComps(); + } + ); } diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 80bb68cfa5b..07793a373f3 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -12,11 +12,11 @@ void init_WarpXParIter (py::module& m) { py::class_< - WarpXParIter, amrex::ParIter<0,0,PIdx::nattribs> + WarpXParIter, amrex::ParIterSoA >(m, "WarpXParIter") - .def(py::init::ContainerType&, int>(), + .def(py::init::ContainerType&, int>(), py::arg("particle_container"), py::arg("level")) - .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), + .def(py::init::ContainerType&, int, amrex::MFItInfo&>(), py::arg("particle_container"), py::arg("level"), py::arg("info")) ; @@ -26,11 +26,11 @@ void init_WarpXParticleContainer (py::module& m) { py::class_< WarpXParticleContainer, - amrex::ParticleContainer<0, 0, PIdx::nattribs, 0> + amrex::ParticleContainerPureSoA > wpc (m, "WarpXParticleContainer"); wpc .def("add_real_comp", - [](WarpXParticleContainer& pc, const std::string& name, bool const comm) { pc.AddRealComp(name, comm); }, + [](WarpXParticleContainer& pc, const std::string& name, bool comm) { pc.AddRealComp(name, comm); }, py::arg("name"), py::arg("comm") ) .def("add_n_particles", @@ -85,7 +85,6 @@ void init_WarpXParticleContainer (py::module& m) py::arg("nattr_int"), py::arg("attr_int"), py::arg("uniqueparticles"), py::arg("id")=-1 ) - .def("num_real_comps", &WarpXParticleContainer::NumRealComps) .def("get_comp_index", [](WarpXParticleContainer& pc, std::string comp_name) { @@ -94,6 +93,14 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("comp_name") ) + .def("get_icomp_index", + [](WarpXParticleContainer& pc, std::string comp_name) + { + auto particle_comps = pc.getParticleiComps(); + return particle_comps.at(comp_name); + }, + py::arg("comp_name") + ) .def("num_local_tiles_at_level", &WarpXParticleContainer::numLocalTilesAtLevel, py::arg("level") diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 1a5b319577c..e42b8268fe1 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -25,6 +25,7 @@ #endif // use PSATD ifdef #include #include +#include #include #include #include @@ -85,7 +86,15 @@ void init_WarpX (py::module& m) "Evolve the simulation the specified number of steps" ) - // from AmrCore->AmrMesh + // from amrex::AmrCore / amrex::AmrMesh + .def_property_readonly("max_level", + [](WarpX const & wx){ return wx.maxLevel(); }, + "The maximum mesh-refinement level for the simulation." + ) + .def_property_readonly("finest_level", + [](WarpX const & wx){ return wx.finestLevel(); }, + "The currently finest level of mesh-refinement used. This is always less or equal to max_level." + ) .def("Geom", //[](WarpX const & wx, int const lev) { return wx.Geom(lev); }, py::overload_cast< int >(&WarpX::Geom, py::const_), @@ -111,7 +120,15 @@ void init_WarpX (py::module& m) }, py::arg("multifab_name"), py::return_value_policy::reference_internal, - "Return MultiFabs by name, e.g., 'Efield_aux[x][l=0]', 'Efield_cp[x][l=0]', ..." + R"doc(Return MultiFabs by name, e.g., ``\"Efield_aux[x][level=0]\"``, ``\"Efield_cp[x][level=0]\"``, ... + +The physical fields in WarpX have the following naming: + +- ``_fp`` are the "fine" patches, the regular resolution of a current mesh-refinement level +- ``_aux`` are temporary (auxiliar) patches at the same resolution as ``_fp``. + They usually include contributions from other levels and can be interpolated for gather routines of particles. +- ``_cp`` are "coarse" patches, at the same resolution (but not necessary values) as the ``_fp`` of ``level - 1`` + (only for level 1 and higher).)doc" ) .def("multi_particle_container", [](WarpX& wx){ return &wx.GetPartContainer(); }, @@ -139,22 +156,26 @@ void init_WarpX (py::module& m) // Expose functions to get the current simulation step and time .def("getistep", [](WarpX const & wx, int lev){ return wx.getistep(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current step on mesh-refinement level ``lev``." ) .def("gett_new", [](WarpX const & wx, int lev){ return wx.gett_new(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time on mesh-refinement level ``lev``." ) .def("getdt", [](WarpX const & wx, int lev){ return wx.getdt(lev); }, - py::arg("lev") + py::arg("lev"), + "Get the current physical time step size on mesh-refinement level ``lev``." ) .def("set_potential_on_eb", [](WarpX& wx, std::string potential) { wx.m_poisson_boundary_handler.setPotentialEB(potential); }, - py::arg("potential") + py::arg("potential"), + "Sets the EB potential string and updates the function parser." ) ; diff --git a/Source/Utils/Parser/IntervalsParser.cpp b/Source/Utils/Parser/IntervalsParser.cpp index 6af4ce6a8c3..6564003abdf 100644 --- a/Source/Utils/Parser/IntervalsParser.cpp +++ b/Source/Utils/Parser/IntervalsParser.cpp @@ -92,7 +92,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto insplit = ablastr::utils::text::split_string>( inconcatenated, m_separator); @@ -102,7 +102,7 @@ utils::parser::IntervalsParser::IntervalsParser ( const SliceParser temp_slice(inslc); m_slices.push_back(temp_slice); if ((temp_slice.getPeriod() > 0) && - (temp_slice.getStop() >= temp_slice.getStart())) m_activated = true; + (temp_slice.getStop() >= temp_slice.getStart())) { m_activated = true; } } } @@ -155,7 +155,7 @@ utils::parser::BTDIntervalsParser::BTDIntervalsParser ( const std::vector& instr_vec) { std::string inconcatenated; - for (const auto& instr_element : instr_vec) inconcatenated +=instr_element; + for (const auto& instr_element : instr_vec) { inconcatenated +=instr_element; } auto const insplit = ablastr::utils::text::split_string>( inconcatenated, std::string(1,m_separator)); diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index fb86996bcd9..dbc4845d1bf 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -63,9 +63,28 @@ namespace utils::parser * \param query_string ParmParse.query will look for this string * \param stored_string variable in which the string to parse is stored */ + void Store_parserString( + amrex::ParmParse const& pp, + std::string const& query_string, + std::string& stored_string); + + /** + * \brief Parse a string (typically a mathematical expression) from the + * input file and store it into a variable. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] pp used to read the query_string `pp.=string` + * \param[in] group name of the optional group + * \param[in] query_string ParmParse.query will look for this string + * \param[out] stored_string variable in which the string to parse is stored + */ void Store_parserString( const amrex::ParmParse &pp, - std::string query_string, + std::string const& group, + std::string const& query_string, std::string& stored_string); @@ -299,6 +318,208 @@ namespace utils::parser } } + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + */ + template + int queryWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryWithParser(a_pp, grp_str.c_str(), val); + } + } + + + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::query, but also supports math expressions for the value. + * + * amrex::ParmParse::query reads a name and a value from the input file. This function does the + * same, and applies the amrex::Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * queryArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored, either a scalar or vector + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + int queryArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return queryArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return queryArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::query, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + int query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val); + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getWithParser(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + template + void getWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, T& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getWithParser(a_pp, grp_str.c_str(), val); + } + } + + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val); + } + } + + + /** Similar to amrex::ParmParse::get, but also supports math expressions for the value. + * + * amrex::ParmParse::get reads a name and a value from the input file. This function does the + * same, and applies the Parser to the value, so the user has the choice to specify a value or + * a math expression (including user-defined constants). + * Works for amrex::Real numbers and integers. + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * getArrWithParser(pp, "group", "name", val, start_ix, num_val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + * \param[in] start_ix start index in the list of inputs values (optional with arrays, default is + * amrex::ParmParse::FIRST for starting with the first input value) + * \param[in] num_val number of input values to use (optional with arrays, default is + * amrex::ParmParse::LAST for reading until the last input value) + */ + template + void getArrWithParser (const amrex::ParmParse& a_pp, std::string const& group, char const * const str, std::vector& val, + const int start_ix, const int num_val) + { + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + getArrWithParser(a_pp, str, val, start_ix, num_val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + getArrWithParser(a_pp, grp_str.c_str(), val, start_ix, num_val); + } + } + + /** Wraps around amrex::ParmParse::get, but also supports an option group name + * + * The group name specified is optional part of the parameter name. A parameter that includes the group + * name takes precedence over one without the group name. If this routine is called like + * get(pp, "group", "name", val), it will query both "group.name" or "name" and return + * the value of "group.name" if found, otherwise the value of "name". + * + * \param[in] a_pp amrex::ParmParse object + * \param[in] group name of the optional group + * \param[in] str name of the parameter to read + * \param[out] val where the value queried and parsed is stored + */ + void get (amrex::ParmParse const& a_pp, std::string const& group, char const * str, std::string& val); + } #endif // WARPX_UTILS_PARSER_PARSERUTILS_H_ diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index aeacedc5b4f..c2da9577947 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -19,8 +19,8 @@ #include void utils::parser::Store_parserString( - const amrex::ParmParse& pp, - std::string query_string, + amrex::ParmParse const& pp, + std::string const& query_string, std::string& stored_string) { std::vector f; @@ -32,6 +32,54 @@ void utils::parser::Store_parserString( f.clear(); } +void utils::parser::Store_parserString( + amrex::ParmParse const& a_pp, + std::string const& group, + std::string const& query_string, + std::string& stored_string) +{ + const bool is_specified_without_group = a_pp.contains(query_string.c_str()); + const std::string grp_str = group + "." + query_string; + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + utils::parser::Store_parserString(a_pp, query_string, stored_string); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + utils::parser::Store_parserString(a_pp, grp_str, stored_string); + } +} + +int utils::parser::query (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + return a_pp.query(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + return a_pp.query(grp_str.c_str(), val); + } +} + +void utils::parser::get (const amrex::ParmParse& a_pp, std::string const& group, char const * str, std::string& val) +{ + const bool is_specified_without_group = a_pp.contains(str); + const std::string grp_str = group + "." + std::string(str); + const bool is_specified_with_group = (group.empty() ? false : a_pp.contains(grp_str.c_str())); + + if (is_specified_without_group && !is_specified_with_group) { + // If found without the group but not with the group, then use the one without the group. + a_pp.get(str, val); + } else { + // Otherwise, use the one with the group even if not found, in which case an exception may be raised. + a_pp.get(grp_str.c_str(), val); + } +} namespace { template< typename int_type > @@ -87,7 +135,7 @@ amrex::Parser utils::parser::makeParser ( parser.registerVariables(varnames); std::set symbols = parser.symbols(); - for (auto const& v : varnames) symbols.erase(v); + for (auto const& v : varnames) { symbols.erase(v); } // User can provide inputs under this name, through which expressions // can be provided for arbitrary variables. PICMI inputs are aware of diff --git a/Source/Utils/ParticleUtils.H b/Source/Utils/ParticleUtils.H index ecd6b7c2739..7e3c89228ea 100644 --- a/Source/Utils/ParticleUtils.H +++ b/Source/Utils/ParticleUtils.H @@ -28,9 +28,10 @@ namespace ParticleUtils { * @param[in] mfi the MultiFAB iterator. * @param[in] ptile the particle tile. */ - amrex::DenseBins - findParticlesInEachCell(int lev, amrex::MFIter const& mfi, - WarpXParticleContainer::ParticleTileType const& ptile); + amrex::DenseBins + findParticlesInEachCell (int lev, + amrex::MFIter const & mfi, + WarpXParticleContainer::ParticleTileType & ptile); /** * \brief Return (relativistic) particle energy given velocity and mass. @@ -181,8 +182,8 @@ namespace ParticleUtils { */ AMREX_GPU_HOST_DEVICE AMREX_INLINE bool containsInclusive (amrex::RealBox const& tilebox, amrex::XDim3 const point) { - const auto xlo = tilebox.lo(); - const auto xhi = tilebox.hi(); + const auto *const xlo = tilebox.lo(); + const auto *const xhi = tilebox.hi(); return AMREX_D_TERM((xlo[0] <= point.x) && (point.x <= xhi[0]), && (xlo[1] <= point.y) && (point.y <= xhi[1]), && (xlo[2] <= point.z) && (point.z <= xhi[2])); diff --git a/Source/Utils/ParticleUtils.cpp b/Source/Utils/ParticleUtils.cpp index 60e04f12b86..b8207b61fa0 100644 --- a/Source/Utils/ParticleUtils.cpp +++ b/Source/Utils/ParticleUtils.cpp @@ -22,24 +22,28 @@ #include #include -namespace ParticleUtils { +namespace ParticleUtils +{ using namespace amrex; + // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleBins = DenseBins; - using index_type = ParticleBins::index_type; + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = DenseBins; + using index_type = typename ParticleBins::index_type; /* Find the particles and count the particles that are in each cell. Note that this does *not* rearrange particle arrays */ ParticleBins - findParticlesInEachCell( int const lev, MFIter const& mfi, - ParticleTileType const& ptile) { + findParticlesInEachCell (int lev, + MFIter const & mfi, + ParticleTileType & ptile) { // Extract particle structures for this tile int const np = ptile.numParticles(); - ParticleType const* particle_ptr = ptile.GetArrayOfStructs()().data(); + auto ptd = ptile.getParticleTileData(); // Extract box properties Geometry const& geom = WarpX::GetInstance().Geom(lev); @@ -51,9 +55,9 @@ namespace ParticleUtils { // Find particles that are in each cell; // results are stored in the object `bins`. ParticleBins bins; - bins.build(np, particle_ptr, cbx, + bins.build(np, ptd, cbx, // Pass lambda function that returns the cell index - [=] AMREX_GPU_DEVICE (const ParticleType& p) noexcept + [=] AMREX_GPU_DEVICE (ParticleType const & p) noexcept -> amrex::IntVect { return IntVect{AMREX_D_DECL( static_cast((p.pos(0)-plo[0])*dxi[0] - lo.x), @@ -64,4 +68,4 @@ namespace ParticleUtils { return bins; } -} +} // namespace ParticleUtils diff --git a/Source/Utils/Physics/IonizationEnergiesTable.H b/Source/Utils/Physics/IonizationEnergiesTable.H index f956e308dbe..b7c82e88247 100644 --- a/Source/Utils/Physics/IonizationEnergiesTable.H +++ b/Source/Utils/Physics/IonizationEnergiesTable.H @@ -1963,4 +1963,11 @@ namespace utils::physics } +constexpr int correction_factors_length = 4; +// a1, a2, a3, Ecrit from Zhang et al., PRA 90, 043410 (2014). +constexpr amrex::Real table_correction_factors[correction_factors_length]{ + // H + amrex::Real(0.11714), amrex::Real(-0.90933), amrex::Real(-0.06034), amrex::Real(32125000000.) +}; + #endif // #ifndef WARPX_UTILS_PHYSICS_IONIZATION_TABLE_H_ diff --git a/Source/Utils/RelativeCellPosition.cpp b/Source/Utils/RelativeCellPosition.cpp index 1a108b54b56..d30da1841cb 100644 --- a/Source/Utils/RelativeCellPosition.cpp +++ b/Source/Utils/RelativeCellPosition.cpp @@ -22,8 +22,9 @@ utils::getRelativeCellPosition(amrex::MultiFab const& mf) // WarpX::grid_type==GridType::Collocated means: all indices/directions on CellIndex::NODE for (int d = 0; d < AMREX_SPACEDIM; d++) { - if (idx_type.cellCentered(d)) + if (idx_type.cellCentered(d)) { relative_position.at(d) = 0.5; + } } return relative_position; diff --git a/Source/Utils/SpeciesUtils.H b/Source/Utils/SpeciesUtils.H index f95525ecb4c..3f5d42910c4 100644 --- a/Source/Utils/SpeciesUtils.H +++ b/Source/Utils/SpeciesUtils.H @@ -21,11 +21,11 @@ namespace SpeciesUtils { std::string const& injection_style, amrex::Real& charge, amrex::Real& mass, PhysicalSpecies& physical_species); - void parseDensity (std::string const& species_name, + void parseDensity (std::string const& species_name, std::string const& source_name, std::unique_ptr& h_inj_rho, std::unique_ptr& density_parser); - void parseMomentum (std::string const& species_name, const std::string& style, + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, std::unique_ptr& h_inj_mom, std::unique_ptr& ux_parser, std::unique_ptr& uy_parser, diff --git a/Source/Utils/SpeciesUtils.cpp b/Source/Utils/SpeciesUtils.cpp index 50729188e33..f8515592a07 100644 --- a/Source/Utils/SpeciesUtils.cpp +++ b/Source/Utils/SpeciesUtils.cpp @@ -77,20 +77,20 @@ namespace SpeciesUtils { // Depending on injection type at runtime, initialize inj_rho // so that inj_rho->getDensity calls // InjectorPosition[Constant or Predefined or etc.].getDensity. - void parseDensity (std::string const& species_name, + void parseDensity (std::string const& species_name, std::string const& source_name, std::unique_ptr& h_inj_rho, std::unique_ptr& density_parser) { - const amrex::ParmParse pp_species_name(species_name); + amrex::ParmParse pp_species(species_name); // parse density information std::string rho_prof_s; - pp_species_name.get("profile", rho_prof_s); + utils::parser::get(pp_species, source_name, "profile", rho_prof_s); std::transform(rho_prof_s.begin(), rho_prof_s.end(), rho_prof_s.begin(), ::tolower); if (rho_prof_s == "constant") { amrex::Real density; - utils::parser::getWithParser(pp_species_name, "density", density); + utils::parser::getWithParser(pp_species, source_name, "density", density); // Construct InjectorDensity with InjectorDensityConstant. h_inj_rho.reset(new InjectorDensity((InjectorDensityConstant*)nullptr, density)); } else if (rho_prof_s == "predefined") { @@ -98,8 +98,7 @@ namespace SpeciesUtils { h_inj_rho.reset(new InjectorDensity((InjectorDensityPredefined*)nullptr,species_name)); } else if (rho_prof_s == "parse_density_function") { std::string str_density_function; - utils::parser::Store_parserString( - pp_species_name, "density_function(x,y,z)", str_density_function); + utils::parser::Store_parserString(pp_species, source_name, "density_function(x,y,z)", str_density_function); // Construct InjectorDensity with InjectorDensityParser. density_parser = std::make_unique( utils::parser::makeParser(str_density_function,{"x","y","z"})); @@ -113,7 +112,7 @@ namespace SpeciesUtils { // Depending on injection type at runtime, initialize inj_mom // so that inj_mom->getMomentum calls // InjectorMomentum[Constant or Gaussian or etc.].getMomentum. - void parseMomentum (std::string const& species_name, const std::string& style, + void parseMomentum (std::string const& species_name, std::string const& source_name, const std::string& style, std::unique_ptr& h_inj_mom, std::unique_ptr& ux_parser, std::unique_ptr& uy_parser, @@ -127,15 +126,15 @@ namespace SpeciesUtils { { using namespace amrex::literals; - const amrex::ParmParse pp_species_name(species_name); + amrex::ParmParse pp_species(species_name); // parse momentum information std::string mom_dist_s; - pp_species_name.get("momentum_distribution_type", mom_dist_s); + utils::parser::get(pp_species, source_name, "momentum_distribution_type", mom_dist_s); std::transform(mom_dist_s.begin(), - mom_dist_s.end(), - mom_dist_s.begin(), - ::tolower); + mom_dist_s.end(), + mom_dist_s.begin(), + ::tolower); if (mom_dist_s == "at_rest") { constexpr amrex::Real ux = 0._rt; constexpr amrex::Real uy = 0._rt; @@ -146,9 +145,9 @@ namespace SpeciesUtils { amrex::Real ux = 0._rt; amrex::Real uy = 0._rt; amrex::Real uz = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux", ux); - utils::parser::queryWithParser(pp_species_name, "uy", uy); - utils::parser::queryWithParser(pp_species_name, "uz", uz); + utils::parser::queryWithParser(pp_species, source_name, "ux", ux); + utils::parser::queryWithParser(pp_species, source_name, "uy", uy); + utils::parser::queryWithParser(pp_species, source_name, "uz", uz); // Construct InjectorMomentum with InjectorMomentumConstant. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumConstant*)nullptr, ux, uy, uz)); } else if (mom_dist_s == "gaussian") { @@ -158,12 +157,12 @@ namespace SpeciesUtils { amrex::Real ux_th = 0._rt; amrex::Real uy_th = 0._rt; amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); // Construct InjectorMomentum with InjectorMomentumGaussian. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussian*)nullptr, ux_m, uy_m, uz_m, ux_th, uy_th, uz_th)); @@ -176,12 +175,12 @@ namespace SpeciesUtils { amrex::Real ux_th = 0._rt; amrex::Real uy_th = 0._rt; amrex::Real uz_th = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_m", ux_m); - utils::parser::queryWithParser(pp_species_name, "uy_m", uy_m); - utils::parser::queryWithParser(pp_species_name, "uz_m", uz_m); - utils::parser::queryWithParser(pp_species_name, "ux_th", ux_th); - utils::parser::queryWithParser(pp_species_name, "uy_th", uy_th); - utils::parser::queryWithParser(pp_species_name, "uz_th", uz_th); + utils::parser::queryWithParser(pp_species, source_name, "ux_m", ux_m); + utils::parser::queryWithParser(pp_species, source_name, "uy_m", uy_m); + utils::parser::queryWithParser(pp_species, source_name, "uz_m", uz_m); + utils::parser::queryWithParser(pp_species, source_name, "ux_th", ux_th); + utils::parser::queryWithParser(pp_species, source_name, "uy_th", uy_th); + utils::parser::queryWithParser(pp_species, source_name, "uz_th", uz_th); // Construct InjectorMomentum with InjectorMomentumGaussianFlux. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumGaussianFlux*)nullptr, ux_m, uy_m, uz_m, ux_th, uy_th, uz_th, @@ -193,32 +192,32 @@ namespace SpeciesUtils { amrex::Real ux_max = 0._rt; amrex::Real uy_max = 0._rt; amrex::Real uz_max = 0._rt; - utils::parser::queryWithParser(pp_species_name, "ux_min", ux_min); - utils::parser::queryWithParser(pp_species_name, "uy_min", uy_min); - utils::parser::queryWithParser(pp_species_name, "uz_min", uz_min); - utils::parser::queryWithParser(pp_species_name, "ux_max", ux_max); - utils::parser::queryWithParser(pp_species_name, "uy_max", uy_max); - utils::parser::queryWithParser(pp_species_name, "uz_max", uz_max); + utils::parser::queryWithParser(pp_species, source_name, "ux_min", ux_min); + utils::parser::queryWithParser(pp_species, source_name, "uy_min", uy_min); + utils::parser::queryWithParser(pp_species, source_name, "uz_min", uz_min); + utils::parser::queryWithParser(pp_species, source_name, "ux_max", ux_max); + utils::parser::queryWithParser(pp_species, source_name, "uy_max", uy_max); + utils::parser::queryWithParser(pp_species, source_name, "uz_max", uz_max); // Construct InjectorMomentum with InjectorMomentumUniform. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumUniform*)nullptr, ux_min, uy_min, uz_min, ux_max, uy_max, uz_max)); } else if (mom_dist_s == "maxwell_boltzmann"){ - h_mom_temp = std::make_unique(pp_species_name); + h_mom_temp = std::make_unique(pp_species, source_name); const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); + h_mom_vel = std::make_unique(pp_species, source_name); const GetVelocity getVel(*h_mom_vel); // Construct InjectorMomentum with InjectorMomentumBoltzmann. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumBoltzmann*)nullptr, getTemp, getVel)); } else if (mom_dist_s == "maxwell_juttner"){ - h_mom_temp = std::make_unique(pp_species_name); + h_mom_temp = std::make_unique(pp_species, source_name); const GetTemperature getTemp(*h_mom_temp); - h_mom_vel = std::make_unique(pp_species_name); + h_mom_vel = std::make_unique(pp_species, source_name); const GetVelocity getVel(*h_mom_vel); // Construct InjectorMomentum with InjectorMomentumJuttner. h_inj_mom.reset(new InjectorMomentum((InjectorMomentumJuttner*)nullptr, getTemp, getVel)); } else if (mom_dist_s == "radial_expansion") { amrex::Real u_over_r = 0._rt; - utils::parser::queryWithParser(pp_species_name, "u_over_r", u_over_r); + utils::parser::queryWithParser(pp_species, source_name, "u_over_r", u_over_r); // Construct InjectorMomentum with InjectorMomentumRadialExpansion. h_inj_mom.reset(new InjectorMomentum ((InjectorMomentumRadialExpansion*)nullptr, u_over_r)); @@ -226,12 +225,9 @@ namespace SpeciesUtils { std::string str_momentum_function_ux; std::string str_momentum_function_uy; std::string str_momentum_function_uz; - utils::parser::Store_parserString(pp_species_name, "momentum_function_ux(x,y,z)", - str_momentum_function_ux); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uy(x,y,z)", - str_momentum_function_uy); - utils::parser::Store_parserString(pp_species_name, "momentum_function_uz(x,y,z)", - str_momentum_function_uz); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux(x,y,z)", str_momentum_function_ux); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy(x,y,z)", str_momentum_function_uy); + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz(x,y,z)", str_momentum_function_uz); // Construct InjectorMomentum with InjectorMomentumParser. ux_parser = std::make_unique( utils::parser::makeParser(str_momentum_function_ux, {"x","y","z"})); @@ -250,17 +246,17 @@ namespace SpeciesUtils { std::string str_momentum_function_ux_th; std::string str_momentum_function_uy_th; std::string str_momentum_function_uz_th; - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux_m(x,y,z)", str_momentum_function_ux_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy_m(x,y,z)", str_momentum_function_uy_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz_m(x,y,z)", str_momentum_function_uz_m); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_ux_th(x,y,z)", str_momentum_function_ux_th); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uy_th(x,y,z)", str_momentum_function_uy_th); - utils::parser::Store_parserString(pp_species_name, + utils::parser::Store_parserString(pp_species, source_name, "momentum_function_uz_th(x,y,z)", str_momentum_function_uz_th); // Construct InjectorMomentum with InjectorMomentumParser. ux_parser = std::make_unique( diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 7ad33409900..735fc7993f1 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -23,6 +23,17 @@ struct MediumForEM { }; }; +/** + * \brief struct to select the overall evolve scheme + */ +struct EvolveScheme { + enum { + Explicit = 0, + ImplicitPicard = 1, + SemiImplicitPicard = 2 + }; +}; + /** * \brief struct to select algorithm for macroscopic Maxwell solver LaxWendroff (semi-implicit) represents sigma*E = sigma*0.5*(E^(n) + E^(n+1)) @@ -76,7 +87,8 @@ struct CurrentDepositionAlgo { enum { Esirkepov = 0, Direct = 1, - Vay = 2 + Vay = 2, + Villasenor = 3 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index ed8186a325c..abaf17f0a2c 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -20,9 +20,16 @@ #include #include -// Define dictionary with correspondance between user-input strings, +// Define dictionary with correspondence between user-input strings, // and corresponding integer for use inside the code +const std::map evolve_scheme_to_int = { + {"explicit", EvolveScheme::Explicit }, + {"implicit_picard", EvolveScheme::ImplicitPicard }, + {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, + {"default", EvolveScheme::Explicit } +}; + const std::map grid_to_int = { {"collocated", GridType::Collocated}, {"staggered", GridType::Staggered}, @@ -56,10 +63,11 @@ const std::map particle_pusher_algo_to_int = { }; const std::map current_deposition_algo_to_int = { - {"esirkepov", CurrentDepositionAlgo::Esirkepov }, - {"direct", CurrentDepositionAlgo::Direct }, - {"vay", CurrentDepositionAlgo::Vay }, - {"default", CurrentDepositionAlgo::Esirkepov } // NOTE: overwritten for PSATD and Hybrid-PIC below + {"esirkepov", CurrentDepositionAlgo::Esirkepov }, + {"direct", CurrentDepositionAlgo::Direct }, + {"vay", CurrentDepositionAlgo::Vay }, + {"villasenor", CurrentDepositionAlgo::Villasenor }, + {"default", CurrentDepositionAlgo::Esirkepov } // NOTE: overwritten for PSATD and Hybrid-PIC below }; const std::map charge_deposition_algo_to_int = { @@ -147,7 +155,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ // Pick the right dictionary std::map algo_to_int; - if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { + if (0 == std::strcmp(pp_search_key, "evolve_scheme")) { + algo_to_int = evolve_scheme_to_int; + } else if (0 == std::strcmp(pp_search_key, "maxwell_solver")) { algo_to_int = electromagnetic_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "grid_type")) { algo_to_int = grid_to_int; @@ -159,8 +169,9 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = current_deposition_algo_to_int; if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD || WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC || - WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) + WarpX::electrostatic_solver_id != ElectrostaticSolverAlgo::None) { algo_to_int["default"] = CurrentDepositionAlgo::Direct; + } } else if (0 == std::strcmp(pp_search_key, "charge_deposition")) { algo_to_int = charge_deposition_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "field_gathering")) { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 60b31af2f14..fdcb7cb8da4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -12,6 +12,7 @@ #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) # include "BoundaryConditions/PML_RZ.H" #endif +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" @@ -81,7 +82,8 @@ WarpX::UpdateInjectionPosition (const amrex::Real a_dt) current_injection_position[dir] = pc.m_current_injection_position; #endif - PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(); + // This only uses the base plasma injector + PlasmaInjector* plasma_injector = pc.GetPlasmaInjector(0); amrex::Real v_shift = 0._rt; if (plasma_injector != nullptr) @@ -142,7 +144,7 @@ WarpX::MoveWindow (const int step, bool move_j) if (step == end_moving_window_step) { amrex::Print() << Utils::TextMsg::Info("Stopping moving window"); } - if (!moving_window_active(step)) return 0; + if (!moving_window_active(step)) { return 0; } // Update the continuous position of the moving window, // and of the plasma injection @@ -163,7 +165,7 @@ WarpX::MoveWindow (const int step, bool move_j) const amrex::Real* cdx = geom[0].CellSize(); const int num_shift_base = static_cast((moving_window_x - current_lo[dir]) / cdx[dir]); - if (num_shift_base == 0) return 0; + if (num_shift_base == 0) { return 0; } // update the problem domain. Note the we only do this on the base level because // amrex::Geometry objects share the same, static RealBox. @@ -217,27 +219,29 @@ WarpX::MoveWindow (const int step, bool move_j) amrex::ParserExecutor<3> Efield_parser; bool use_Bparser = false; bool use_Eparser = false; - if (B_ext_grid_s == "parse_b_ext_grid_function") { + if (m_p_ext_field_params->B_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Bparser = true; - if (dim == 0) Bfield_parser = Bxfield_parser->compile<3>(); - if (dim == 1) Bfield_parser = Byfield_parser->compile<3>(); - if (dim == 2) Bfield_parser = Bzfield_parser->compile<3>(); + if (dim == 0) { Bfield_parser = m_p_ext_field_params->Bxfield_parser->compile<3>(); } + if (dim == 1) { Bfield_parser = m_p_ext_field_params->Byfield_parser->compile<3>(); } + if (dim == 2) { Bfield_parser = m_p_ext_field_params->Bzfield_parser->compile<3>(); } } - if (E_ext_grid_s == "parse_e_ext_grid_function") { + if (m_p_ext_field_params->E_ext_grid_type == + ExternalFieldType::parse_ext_grid_function) { use_Eparser = true; - if (dim == 0) Efield_parser = Exfield_parser->compile<3>(); - if (dim == 1) Efield_parser = Eyfield_parser->compile<3>(); - if (dim == 2) Efield_parser = Ezfield_parser->compile<3>(); + if (dim == 0) { Efield_parser = m_p_ext_field_params->Exfield_parser->compile<3>(); } + if (dim == 1) { Efield_parser = m_p_ext_field_params->Eyfield_parser->compile<3>(); } + if (dim == 2) { Efield_parser = m_p_ext_field_params->Ezfield_parser->compile<3>(); } } shiftMF(*Bfield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params-> E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_fp[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); @@ -259,16 +263,16 @@ WarpX::MoveWindow (const int step, bool move_j) if (lev > 0) { // coarse grid shiftMF(*Bfield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); shiftMF(*Bfield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); shiftMF(*Efield_aux[lev][dim], geom[lev], num_shift, dir, lev, do_update_cost); if (fft_do_time_averaging) { shiftMF(*Bfield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - B_external_grid[dim], use_Bparser, Bfield_parser); + m_p_ext_field_params->B_external_grid[dim], use_Bparser, Bfield_parser); shiftMF(*Efield_avg_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost, - E_external_grid[dim], use_Eparser, Efield_parser); + m_p_ext_field_params->E_external_grid[dim], use_Eparser, Efield_parser); } if (move_j) { shiftMF(*current_cp[lev][dim], geom[lev-1], num_shift_crse, dir, lev, do_update_cost); @@ -600,7 +604,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, // guard region both radially and longitudinally. These are the PML cells in the overlapping // longitudinal region. FillBoundary normally does not update these cells. // This update is needed so that the cells at the end of the FABs are updated appropriately - // with the data shifted from the nieghboring FAB. Without this update, the RZ PML becomes + // with the data shifted from the neighboring FAB. Without this update, the RZ PML becomes // unstable with the moving grid. // This code creates a temporary MultiFab using a BoxList where the radial size of all of // its boxes is increased so that the radial guard cells are included in the boxes valid domain. diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index 8d87dfccc9b..192fc9e9152 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -32,7 +32,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) const auto dx = Geom(lev).CellSizeArray(); amrex::ParserExecutor<3> ref_parser; - if (ref_patch_parser) ref_parser = ref_patch_parser->compile<3>(); + if (ref_patch_parser) { ref_parser = ref_patch_parser->compile<3>(); } const auto ftlo = fine_tag_lo; const auto fthi = fine_tag_hi; #ifdef AMREX_USE_OMP diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index c65603f4d02..5b917a81fdf 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -146,7 +146,7 @@ void ConvertLabParamsToBoost() ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - if (gamma_boost <= 1.) return; + if (gamma_boost <= 1.) { return; } Vector prob_lo(AMREX_SPACEDIM); Vector prob_hi(AMREX_SPACEDIM); @@ -289,11 +289,17 @@ void CheckDims () #endif const ParmParse pp_geometry("geometry"); std::string dims; - pp_geometry.get("dims", dims); std::string dims_error = "The selected WarpX executable was built as '"; dims_error.append(dims_compiled).append("'-dimensional, but the "); - dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); - dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + if (pp_geometry.contains("dims")) { + pp_geometry.get("dims", dims); + dims_error.append("inputs file declares 'geometry.dims = ").append(dims).append("'.\n"); + dims_error.append("Please re-compile with a different WarpX_DIMS option or select the right executable name."); + } else { + dims = "Not specified"; + dims_error.append("inputs file does not declare 'geometry.dims'. Please add 'geometry.dims = "); + dims_error.append(dims_compiled).append("' to inputs file."); + } WARPX_ALWAYS_ASSERT_WITH_MESSAGE(dims == dims_compiled, dims_error); } @@ -307,8 +313,9 @@ void CheckGriddingForRZSpectral () const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); // only check for PSATD in RZ - if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { return; + } int max_level; Vector n_cell(AMREX_SPACEDIM, -1); @@ -413,10 +420,12 @@ void ReadBCParams () const ParmParse pp_boundary("boundary"); pp_boundary.queryarr("field_lo", field_BC_lo, 0, AMREX_SPACEDIM); pp_boundary.queryarr("field_hi", field_BC_hi, 0, AMREX_SPACEDIM); - if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) + if (pp_boundary.queryarr("particle_lo", particle_BC_lo, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; - if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) + } + if (pp_boundary.queryarr("particle_hi", particle_BC_hi, 0, AMREX_SPACEDIM)) { particle_boundary_specified = true; + } AMREX_ALWAYS_ASSERT(field_BC_lo.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(field_BC_hi.size() == AMREX_SPACEDIM); AMREX_ALWAYS_ASSERT(particle_BC_lo.size() == AMREX_SPACEDIM); diff --git a/Source/Utils/WarpXVersion.cpp b/Source/Utils/WarpXVersion.cpp index 43978a4795a..41abfebb38c 100644 --- a/Source/Utils/WarpXVersion.cpp +++ b/Source/Utils/WarpXVersion.cpp @@ -17,10 +17,11 @@ WarpX::Version () #ifdef WARPX_GIT_VERSION version = std::string(WARPX_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } std::string @@ -30,8 +31,9 @@ WarpX::PicsarVersion () #ifdef PICSAR_GIT_VERSION version = std::string(PICSAR_GIT_VERSION); #endif - if( version.empty() ) + if( version.empty() ) { return {"Unknown"}; - else + } else { return version; + } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 0bc2e062971..79352274068 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -20,6 +20,7 @@ #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties_fwd.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel_fwd.H" #include "Filter/NCIGodfreyFilter_fwd.H" +#include "Initialization/ExternalField_fwd.H" #include "Particles/ParticleBoundaryBuffer_fwd.H" #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer_fwd.H" @@ -34,12 +35,13 @@ # include "FieldSolver/SpectralSolver/SpectralSolver_fwd.H" # endif #endif +#include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" -#include "AcceleratorLattice/AcceleratorLattice.H" #include "Utils/Parser/IntervalsParser.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" @@ -119,12 +121,20 @@ public: void Evolve (int numsteps = -1); + void EvolveImplicitPicardInit (int lev); + void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); + void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); + void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, + amrex::Vector, 3 > >& Efield_n); + MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} - +#ifdef AMREX_USE_EB + amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} +#endif ParticleBoundaryBuffer& GetParticleBoundaryBuffer () { return *m_particle_boundary_buffer; } static void shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, @@ -139,34 +149,6 @@ public: */ [[nodiscard]] std::string GetAuthors () const { return m_authors; } - //! Initial electric field on the grid - static amrex::Vector E_external_grid; - //! Initial magnetic field on the grid - static amrex::Vector B_external_grid; - - //! Initialization type for external magnetic field on the grid - static std::string B_ext_grid_s; - //! Initialization type for external electric field on the grid - static std::string E_ext_grid_s; - - //! Whether to apply the effect of an externally-defined electric field - static bool add_external_E_field; - //! Whether to apply the effect of an externally-defined magnetic field - static bool add_external_B_field; - - //! User-defined parser to initialize x-component of the magnetic field on the grid - std::unique_ptr Bxfield_parser; - //! User-defined parser to initialize y-component of the magnetic field on the grid - std::unique_ptr Byfield_parser; - //! User-defined parser to initialize z-component of the magnetic field on the grid - std::unique_ptr Bzfield_parser; - //! User-defined parser to initialize x-component of the electric field on the grid - std::unique_ptr Exfield_parser; - //! User-defined parser to initialize y-component of the electric field on the grid - std::unique_ptr Eyfield_parser; - //! User-defined parser to initialize z-component of the electric field on the grid - std::unique_ptr Ezfield_parser; - /** Maximum level up to which the externally defined electric and magnetic fields are initialized. * The default value is set to the max levels in the simulation. * if lev > maxlevel_extEMfield_init, the fields on those levels will have a default value of 0 @@ -174,7 +156,7 @@ public: int maxlevel_extEMfield_init; // Algorithms - //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay) + //! Integer that corresponds to the current deposition algorithm (Esirkepov, direct, Vay, Villasenor) static short current_deposition_algo; //! Integer that corresponds to the charge deposition algorithm (only standard deposition) static short charge_deposition_algo; @@ -184,6 +166,14 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; + //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + static short evolve_scheme; + //! The maximum number of Picard iterations to do each time step + static int max_picard_iterations; + //! The tolerance for the Picard iteration convergence + static amrex::Real picard_iteration_tolerance; + //! Flags whether the Picard iterations are required to converge + static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). */ @@ -451,6 +441,25 @@ public: const std::string& name, std::optional initial_value); + /** + * \brief + * Allocate the MultiFab so that is like the specified MultiFab (same ba and dm) + * and optionally initialize it. This also adds the MultiFab + * to the map of MultiFabs (used to ease the access to MultiFabs from the Python + * interface + * + * \param mf[out] The MultiFab unique pointer to be allocated + * \param mf_model[in] The MultiFab to model + * \param name[in] The name of the MultiFab to use in the map + * \param initial_value[in] The optional initial value + */ + static void AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + int level, + const std::string& name, + std::optional initial_value = {}); + // Maps of all of the MultiFabs and iMultiFabs used (this can include MFs from other classes) // This is a convenience for the Python interface, allowing all MultiFabs // to be easily referenced from Python. @@ -521,6 +530,9 @@ public: const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} + const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} + const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -553,7 +565,7 @@ public: amrex::Vector mirror_z_width; amrex::Vector mirror_z_npoints; - /// object with all reduced diagnotics, similar to MultiParticleContainer for species. + /// object with all reduced diagnostics, similar to MultiParticleContainer for species. std::unique_ptr reduced_diags; void applyMirrors(amrex::Real time); @@ -724,6 +736,11 @@ public: void ApplyEfieldBoundary (int lev, PatchType patch_type); void ApplyBfieldBoundary (int lev, PatchType patch_type, DtType dt_type); +#ifdef WARPX_DIM_RZ + // Applies the boundary conditions that are specific to the axis when in RZ. + void ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex::MultiFab* Ez, int lev) const; +#endif + /** * \brief When the Ohm's law solver is used, the electron pressure values * on PEC boundaries are set to enforce a zero derivative Neumann condition, @@ -767,8 +784,10 @@ public: void doQEDEvents (int lev); #endif - void PushParticlesandDepose (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false); - void PushParticlesandDepose ( amrex::Real cur_time, bool skip_current=false); + void PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type=DtType::Full, bool skip_current=false, + PushType push_type=PushType::Explicit); + void PushParticlesandDeposit (amrex::Real cur_time, bool skip_current=false, + PushType push_type=PushType::Explicit); // This function does aux(lev) = fp(lev) + I(aux(lev-1)-cp(lev)). // Caller must make sure fp and cp have ghost cells filled. @@ -1012,6 +1031,16 @@ public: char field, int lev, PatchType patch_type); + /** + * \brief Load field values from a user-specified openPMD file, + * for the fields Ex, Ey, Ez, Bx, By, Bz + */ + void LoadExternalFieldsFromFile (int lev); + + /** + * \brief Load field values from a user-specified openPMD file + * for a specific field (specified by `F_name`) + */ void ReadExternalFieldFromFile ( std::string read_fields_from_path, amrex::MultiFab* mf, std::string F_name, std::string F_component); @@ -1181,11 +1210,11 @@ protected: * then, the E and B fields at all the levels are initialized with * user-defined values for E_external_grid and B_external_grid. * If the initialization type for B-field is set to - * "parse_B_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Bx_external_grid_function(x,y,z), By_external_grid_function(x,y,z), * and Bz_external_grid_function(x,y,z). * Similarly, if the E-field initialization type is set to - * "parse_E_ext_grid_function", then, the parser is used to read + * "parse_ext_grid_function", then, the parser is used to read * Ex_external_grid_function(x,y,z), Ey_external_grid_function(x,y,z), * and Ex_external_grid_function(x,y,z). The parser for the E and B * initialization assumes that the function has three independent @@ -1201,8 +1230,8 @@ protected: //! Make a new level from scratch using provided BoxArray and //! DistributionMapping. Only used during initialization. Called //! by AmrCoreInitFromScratch. - void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) final; + void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& new_grids, + const amrex::DistributionMapping& new_dmap) final; //! Make a new level using provided BoxArray and //! DistributionMapping and fill with interpolated coarse level @@ -1251,13 +1280,15 @@ private: void AddExternalFields (); - void OneStep_nosub (amrex::Real t); - void OneStep_sub1 (amrex::Real t); + void OneStep_nosub (amrex::Real cur_time); + void OneStep_sub1 (amrex::Real cur_time); + + void OneStep_ImplicitPicard(amrex::Real cur_time); /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ - void OneStep_multiJ (amrex::Real t); + void OneStep_multiJ (amrex::Real cur_time); void RestrictCurrentFromFineToCoarsePatch ( const amrex::Vector,3>>& J_fp, @@ -1326,8 +1357,8 @@ private: void InitFromScratch (); - void AllocLevelData (int lev, const amrex::BoxArray& new_grids, - const amrex::DistributionMapping& new_dmap); + void AllocLevelData (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm); [[nodiscard]] amrex::DistributionMapping GetRestartDMap (const std::string& chkfile, const amrex::BoxArray& ba, int lev) const; @@ -1461,6 +1492,12 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; + // Implicit, fields at start of step and from the previous iteration + amrex::Vector, 3 > > Efield_n; + amrex::Vector, 3 > > Bfield_n; + amrex::Vector, 3 > > Efield_save; + amrex::Vector, 3 > > Bfield_save; + // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; @@ -1560,6 +1597,9 @@ private: #endif amrex::Real v_particle_pml; + // External fields parameters + std::unique_ptr m_p_ext_field_params; + amrex::Real moving_window_x = std::numeric_limits::max(); // Plasma injection parameters @@ -1628,7 +1668,7 @@ private: std::string restart_chkfile; /** When `true`, write the diagnostics after restart at the time of the restart. */ - bool write_diagonstics_on_restart = false; + bool write_diagnostics_on_restart = false; amrex::VisMF::Header::Version plotfile_headerversion = amrex::VisMF::Header::Version_v1; amrex::VisMF::Header::Version slice_plotfile_headerversion = amrex::VisMF::Header::Version_v1; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 87f540aba4b..1136e29b739 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -29,6 +29,7 @@ #endif // use PSATD ifdef #include "FieldSolver/WarpX_FDTD.H" #include "Filter/NCIGodfreyFilter.H" +#include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Fluids/MultiFluidContainer.H" #include "Fluids/WarpXFluidContainer.H" @@ -85,14 +86,6 @@ using namespace amrex; -Vector WarpX::E_external_grid(3, 0.0); -Vector WarpX::B_external_grid(3, 0.0); - -std::string WarpX::B_ext_grid_s; -std::string WarpX::E_ext_grid_s; -bool WarpX::add_external_E_field = false; -bool WarpX::add_external_B_field = false; - int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; int WarpX::end_moving_window_step = -1; @@ -118,6 +111,10 @@ short WarpX::charge_deposition_algo; short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; +short WarpX::evolve_scheme; +int WarpX::max_picard_iterations = 10; +Real WarpX::picard_iteration_tolerance = 1.e-7; +bool WarpX::require_picard_convergence = true; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -293,26 +290,29 @@ WarpX::WarpX () t_old.resize(nlevs_max, std::numeric_limits::lowest()); dt.resize(nlevs_max, std::numeric_limits::max()); - // Loop over species (particles and lasers) - // and set current injection position per species mypc = std::make_unique(this); - const int n_containers = mypc->nContainers(); - for (int i=0; iGetParticleContainer(i); - // Storing injection position for all species, regardless of whether - // they are continuously injected, since it makes looping over the - // elements of current_injection_position easier elsewhere in the code. - if (moving_window_v > 0._rt) - { - // Inject particles continuously from the right end of the box - pc.m_current_injection_position = geom[0].ProbHi(moving_window_dir); - } - else if (moving_window_v < 0._rt) + // Loop over species (particles and lasers) + // and set current injection position per species + if (do_moving_window){ + const int n_containers = mypc->nContainers(); + for (int i=0; iGetParticleContainer(i); + + // Storing injection position for all species, regardless of whether + // they are continuously injected, since it makes looping over the + // elements of current_injection_position easier elsewhere in the code. + if (moving_window_v > 0._rt) + { + // Inject particles continuously from the right end of the box + pc.m_current_injection_position = geom[0].ProbHi(moving_window_dir); + } + else if (moving_window_v < 0._rt) + { + // Inject particles continuously from the left end of the box + pc.m_current_injection_position = geom[0].ProbLo(moving_window_dir); + } } } @@ -442,6 +442,11 @@ WarpX::WarpX () costs_heuristic_cells_wt = 0.250_rt; costs_heuristic_particles_wt = 0.750_rt; break; + case 4: + // this is only a guess + costs_heuristic_cells_wt = 0.200_rt; + costs_heuristic_particles_wt = 0.800_rt; + break; } } else { // FDTD switch (WarpX::nox) @@ -458,6 +463,11 @@ WarpX::WarpX () costs_heuristic_cells_wt = 0.145_rt; costs_heuristic_particles_wt = 0.855_rt; break; + case 4: + // this is only a guess + costs_heuristic_cells_wt = 0.100_rt; + costs_heuristic_particles_wt = 0.900_rt; + break; } } #else // CPU @@ -549,13 +559,13 @@ WarpX::ReadParameters () if(std::string str_abort_on_warning_threshold; pp_warpx.query("abort_on_warning_threshold", str_abort_on_warning_threshold)){ std::optional abort_on_warning_threshold = std::nullopt; - if (str_abort_on_warning_threshold == "high") + if (str_abort_on_warning_threshold == "high") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::high; - else if (str_abort_on_warning_threshold == "medium" ) + } else if (str_abort_on_warning_threshold == "medium" ) { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::medium; - else if (str_abort_on_warning_threshold == "low") + } else if (str_abort_on_warning_threshold == "low") { abort_on_warning_threshold = ablastr::warn_manager::WarnPriority::low; - else { + } else { WARPX_ABORT_WITH_MESSAGE(str_abort_on_warning_threshold +"is not a valid option for warpx.abort_on_warning_threshold (use: low, medium or high)"); } @@ -617,7 +627,7 @@ WarpX::ReadParameters () } } - pp_warpx.query("write_diagonstics_on_restart", write_diagonstics_on_restart); + pp_warpx.query("write_diagnostics_on_restart", write_diagnostics_on_restart); pp_warpx.queryarr("checkpoint_signals", signals_in); #if defined(__linux__) || defined(__APPLE__) @@ -726,20 +736,11 @@ WarpX::ReadParameters () moving_window_v *= PhysConst::c; } - pp_warpx.query("B_ext_grid_init_style", WarpX::B_ext_grid_s); - pp_warpx.query("E_ext_grid_init_style", WarpX::E_ext_grid_s); - - if (WarpX::B_ext_grid_s == "read_from_file") - { + m_p_ext_field_params = std::make_unique(pp_warpx); + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file || + m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file){ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_B_field = true; - } - if (WarpX::E_ext_grid_s == "read_from_file") - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, - "External field reading is not implemented for more than one level"); - add_external_E_field = true; + "External field reading is not implemented for more than one level"); } maxlevel_extEMfield_init = maxLevel(); @@ -789,7 +790,7 @@ WarpX::ReadParameters () // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) #ifdef WARPX_DIM_RZ - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) WarpX::use_filter = false; + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { WarpX::use_filter = false; } #endif // Read filter and fill IntVect filter_npass_each_dir with @@ -861,8 +862,9 @@ WarpX::ReadParameters () const bool shared_tilesize_is_specified = utils::parser::queryArrWithParser(pp_warpx, "shared_tilesize", vect_shared_tilesize, 0, AMREX_SPACEDIM); if (shared_tilesize_is_specified){ - for (int i=0; i(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering - if (grid_type == GridType::Collocated) galerkin_interpolation = false; + if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } #ifdef WARPX_DIM_RZ // Only needs to be set with WARPX_DIM_RZ, otherwise defaults to 1 @@ -1159,6 +1161,7 @@ WarpX::ReadParameters () current_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "current_deposition")); charge_deposition_algo = static_cast(GetAlgorithmInteger(pp_algo, "charge_deposition")); particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); + evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || @@ -1166,6 +1169,12 @@ WarpX::ReadParameters () "Current centering (nodal deposition) cannot be used with Esirkepov deposition." "Please set warpx.do_current_centering = 0 or algo.current_deposition = direct."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo != CurrentDepositionAlgo::Villasenor || + !do_current_centering, + "Current centering (nodal deposition) cannot be used with Villasenor deposition." + "Please set warpx.do_current_centering = 0 or algo.current_deposition = direct."); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::current_deposition_algo != CurrentDepositionAlgo::Vay || !do_current_centering, @@ -1188,6 +1197,14 @@ WarpX::ReadParameters () "Vay deposition not implemented with multi-J algorithm"); } + if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard, + "Villasenor current deposition can only" + "be used with Implicit evolve schemes."); + } + // Query algo.field_gathering from input, set field_gathering_algo to // "default" if not found (default defined in Utils/WarpXAlgorithmSelection.cpp) field_gathering_algo = static_cast(GetAlgorithmInteger(pp_algo, "field_gathering")); @@ -1220,7 +1237,12 @@ WarpX::ReadParameters () } // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) galerkin_interpolation = false; + if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + + { + const ParmParse pp_interpolation("interpolation"); + pp_interpolation.query("galerkin_scheme",galerkin_interpolation); + } // With the PSATD solver, momentum-conserving field gathering // combined with mesh refinement does not seem to work correctly @@ -1251,6 +1273,33 @@ WarpX::ReadParameters () } } + if (evolve_scheme == EvolveScheme::ImplicitPicard || + evolve_scheme == EvolveScheme::SemiImplicitPicard) { + utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); + utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); + utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + current_deposition_algo == CurrentDepositionAlgo::Esirkepov || + current_deposition_algo == CurrentDepositionAlgo::Villasenor || + current_deposition_algo == CurrentDepositionAlgo::Direct, + "Only Esirkepov, Villasenor, or Direct current deposition supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + electromagnetic_solver_id == ElectromagneticSolverAlgo::Yee || + electromagnetic_solver_id == ElectromagneticSolverAlgo::CKC, + "Only the Yee EM solver is supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + particle_pusher_algo == ParticlePusherAlgo::Boris || + particle_pusher_algo == ParticlePusherAlgo::HigueraCary, + "Only the Boris and Higuera particle pushers are supported with the implicit and semi-implicit schemes"); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_gathering_algo != GatheringAlgo::MomentumConserving, + "With implicit and semi-implicit schemes, the momentum conserving field gather is not supported as it would not conserve energy"); + } + // Load balancing parameters std::vector load_balance_intervals_string_vec = {"0"}; pp_algo.queryarr("load_balance_intervals", load_balance_intervals_string_vec); @@ -1258,8 +1307,9 @@ WarpX::ReadParameters () load_balance_intervals_string_vec); pp_algo.query("load_balance_with_sfc", load_balance_with_sfc); // Knapsack factor only used with non-SFC strategy - if (!load_balance_with_sfc) + if (!load_balance_with_sfc) { pp_algo.query("load_balance_knapsack_factor", load_balance_knapsack_factor); + } utils::parser::queryWithParser(pp_algo, "load_balance_efficiency_ratio_threshold", load_balance_efficiency_ratio_threshold); load_balance_costs_update_algo = static_cast(GetAlgorithmInteger(pp_algo, "load_balance_costs_update")); @@ -1301,10 +1351,9 @@ WarpX::ReadParameters () int particle_shape; if (!species_names.empty() || !lasers_names.empty()) { if (utils::parser::queryWithParser(pp_algo, "particle_shape", particle_shape)){ - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (particle_shape >= 1) && (particle_shape <=3), - "algo.particle_shape can be only 1, 2, or 3" + (particle_shape >= 1) && (particle_shape <=4), + "algo.particle_shape can be only 1, 2, 3, or 4" ); nox = particle_shape; @@ -1314,7 +1363,7 @@ WarpX::ReadParameters () else{ WARPX_ABORT_WITH_MESSAGE( "algo.particle_shape must be set in the input file:" - " please set algo.particle_shape to 1, 2, or 3"); + " please set algo.particle_shape to 1, 2, 3, or 4"); } if ((maxLevel() > 0) && (particle_shape > 1) && (do_pml_j_damping == 1)) @@ -1343,8 +1392,9 @@ WarpX::ReadParameters () pp_warpx, "sort_bin_size", vect_sort_bin_size, 0, AMREX_SPACEDIM); if (sort_bin_size_is_specified){ - for (int i=0; i(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -2237,12 +2290,12 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); // Match external field MultiFabs to fine patch - if (add_external_B_field) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); } - if (add_external_E_field) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); @@ -2681,7 +2734,7 @@ void WarpX::AllocLevelSpectralSolverRZ (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -2734,7 +2787,7 @@ void WarpX::AllocLevelSpectralSolver (amrex::Vector(WarpX::do_multi_J_n_depositions); + if (WarpX::do_multi_J) { solver_dt /= static_cast(WarpX::do_multi_J_n_depositions); } auto pss = std::make_unique(lev, realspace_ba, @@ -3221,8 +3274,8 @@ bool WarpX::isAnyBoundaryPML() { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) return true; - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) return true; + if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PML) { return true; } + if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PML) { return true; } } return false; } @@ -3292,3 +3345,21 @@ WarpX::AliasInitMultiFab ( } multifab_map[name_with_suffix] = mf.get(); } + +void +WarpX::AllocInitMultiFabFromModel ( + std::unique_ptr& mf, + amrex::MultiFab& mf_model, + const int level, + const std::string& name, + std::optional initial_value) +{ + const auto name_with_suffix = TagWithLevelSuffix(name, level); + const auto tag = amrex::MFInfo().SetTag(name_with_suffix); + mf = std::make_unique(mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), mf_model.nGrowVect(), tag); + if (initial_value) { + mf->setVal(*initial_value); + } + multifab_map[name_with_suffix] = mf.get(); +} diff --git a/Source/ablastr/coarsen/average.H b/Source/ablastr/coarsen/average.H index 315938acb48..621b4cb54e2 100644 --- a/Source/ablastr/coarsen/average.H +++ b/Source/ablastr/coarsen/average.H @@ -72,20 +72,22 @@ namespace ablastr::coarsen::average // Compute number of points for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { np[l] = 1; // no coarsening - else + } else { np[l] = cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (2 * (cr[l] - 1) + 1) * sf[l] * sc[l]; // nodal + } } // Compute starting indices of source array (fine) for (int l = 0; l < 3; ++l) { - if (cr[l] == 1) + if (cr[l] == 1) { idx_min[l] = ic[l]; // no coarsening - else + } else { idx_min[l] = ic[l] * cr[l] * (1 - sf[l]) * (1 - sc[l]) // cell-centered + (ic[l] * cr[l] - cr[l] + 1) * sf[l] * sc[l]; // nodal + } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.H b/Source/ablastr/coarsen/sample.H index 6ef0962168a..80eac14e833 100644 --- a/Source/ablastr/coarsen/sample.H +++ b/Source/ablastr/coarsen/sample.H @@ -67,14 +67,14 @@ namespace ablastr::coarsen::sample // Compute number of points for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening - else np[l] = 2-sf[l]; + if ( cr[l] == 1 ) { np[l] = 1+amrex::Math::abs(sf[l]-sc[l]); // no coarsening + } else { np[l] = 2-sf[l]; } } // Compute starting indices of source array (fine) for ( int l = 0; l < 3; ++l ) { - if ( cr[l] == 1 ) idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening - else idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); + if ( cr[l] == 1 ) { idx_min[l] = ic[l]-sc[l]*(1-sf[l]); // no coarsening + } else { idx_min[l] = ic[l]*cr[l]+static_cast(cr[l]/2)*(1-sc[l])-(1-sf[l]); } } // Auxiliary integer variables diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index ab5b135309b..b5661037c7e 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,9 +41,10 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) + if ( crse_ratio > amrex::IntVect(1) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, "source fine MultiFab does not have enough guard cells for this interpolation" ); @@ -118,9 +119,9 @@ namespace ablastr::coarsen::sample "source MultiFab converted to staggering of destination MultiFab is not coarsenable" ); ba_tmp.coarsen( crse_ratio ); - if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) + if ( ba_tmp == mf_dst.boxArray() and mf_src.DistributionMap() == mf_dst.DistributionMap() ) { Loop( mf_dst, mf_src, dcomp, scomp, ncomp, ngrowvect, crse_ratio ); - else + } else { // Cannot coarsen into MultiFab with different BoxArray or DistributionMapping: // 1) create temporary MultiFab on coarsened version of source BoxArray with same DistributionMapping diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 953c67ab54e..eaeadfbb2fb 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -12,21 +12,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) -# include -#endif -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include #include #include @@ -45,12 +30,21 @@ #include #include #include +#include #include +#include +#include #include +#include #include #include #include -#include +#if defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ) +# include +#endif +#ifdef AMREX_USE_EB +# include +#endif #include #include @@ -168,7 +162,7 @@ computePhi (amrex::Vector const & rho, const bool always_use_bnorm = (max_norm_b > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "ElectrostaticSolver", "Max norm of rho is 0", @@ -312,9 +306,11 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) - if (post_phi_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index 75957974fdb..8f9021456eb 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -13,19 +13,6 @@ #include #include -#include -#include -#include -#include -#include -#include -#ifdef AMREX_USE_EB -# include -#endif -#include -#include -#include - #include #include #include @@ -43,12 +30,18 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include #include -#include +#ifdef AMREX_USE_EB +# include +#endif #include #include @@ -129,7 +122,7 @@ computeVectorPotential ( amrex::Vector > co const bool always_use_bnorm = (max_comp_J > 0); if (!always_use_bnorm) { - if (absolute_tolerance == 0.0) absolute_tolerance = amrex::Real(1e-6); + if (absolute_tolerance == 0.0) { absolute_tolerance = amrex::Real(1e-6); } ablastr::warn_manager::WMRecordWarning( "MagnetostaticSolver", "Max norm of J is 0", @@ -137,7 +130,7 @@ computeVectorPotential ( amrex::Vector > co ); } - const amrex::LPInfo info; + const amrex::LPInfo& info = amrex::LPInfo(); // Loop over dimensions of A to solve each component individually for (int lev=0; lev<=finest_level; lev++) { @@ -249,9 +242,11 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) - if (post_A_calculation.has_value()) + if constexpr (!std::is_same::value) { + if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); + } + } } // loop over lev(els) } } // namepace Magnetostatic diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index eecc56f9acc..f43e35c6b0b 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -40,8 +40,8 @@ namespace ablastr::particles * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) * \param depos_lev the level to deposit the particles to (default: lev) * \param rel_ref_ratio mesh refinement ratio between lev and depos_lev (default: 1) - * \param offset index to start at when looping over particles to depose (default: 0) - * \param np_to_depose number of particles to depose (default: pti.numParticles()) + * \param offset index to start at when looping over particles to deposit (default: 0) + * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit * \param cost pointer to (load balancing) cost corresponding to box where present @@ -65,7 +65,7 @@ deposit_charge (typename T_PC::ParIterType& pti, std::optional depos_lev = std::nullopt, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, - std::optional np_to_depose = std::nullopt, + std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, amrex::Real * const AMREX_RESTRICT cost = nullptr, long const load_balance_costs_update_algo = 0, @@ -73,8 +73,9 @@ deposit_charge (typename T_PC::ParIterType& pti, { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); - if (num_rho_deposition_guards.has_value()) + if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), "num_rho_deposition_guards are larger than allocated!"); // particle shape @@ -82,14 +83,16 @@ deposit_charge (typename T_PC::ParIterType& pti, // used for MR when we want to deposit for a subset of the particles on the level in the // current box; with offset, we start at a later particle index - if (!np_to_depose.has_value()) - np_to_depose = pti.numParticles(); - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_depose.value() + offset <= pti.numParticles(), - "np_to_depose + offset are out-of-bounds for particle iterator"); + if (!np_to_deposit.has_value()) { + np_to_deposit = pti.numParticles(); + } + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(np_to_deposit.value() + offset <= pti.numParticles(), + "np_to_deposit + offset are out-of-bounds for particle iterator"); int const lev = pti.GetLevel(); - if (!depos_lev.has_value()) + if (!depos_lev.has_value()) { depos_lev = lev; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev.value() == (lev-1)) || (depos_lev.value() == (lev )), "Deposition buffers only work for lev or lev-1"); @@ -100,7 +103,7 @@ deposit_charge (typename T_PC::ParIterType& pti, } // If no particles, do not do anything - if (np_to_depose == 0) return; + if (np_to_deposit == 0) { return; } // Extract deposition order and check that particles shape fits within the guard cells. // NOTE: In specific situations where the staggering of rho and the charge deposition algorithm @@ -179,17 +182,22 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_depose.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + n_rz_azimuthal_modes, cost, + load_balance_costs_update_algo); + } else if (nox == 4){ + doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, + rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo); } @@ -198,7 +206,7 @@ deposit_charge (typename T_PC::ParIterType& pti, #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize); - (*rho)[pti].atomicAdd(local_rho, tb, tb, 0, icomp*nc, nc); + (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc); ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize); #endif } diff --git a/Source/ablastr/particles/IndexHandling.H b/Source/ablastr/particles/IndexHandling.H deleted file mode 100644 index 0ad5ca60446..00000000000 --- a/Source/ablastr/particles/IndexHandling.H +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2019-2022 Axel Huebl - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#ifndef ABLASTR_INDEX_HANDLING_H -#define ABLASTR_INDEX_HANDLING_H - -#include - - -namespace ablastr::particles { - - /** A helper function to derive a globally unique particle ID - * - * @param[in] id AMReX particle ID (on local cpu/rank), AoS .id - * @param[in] cpu AMReX particle CPU (rank) at creation of the particle, AoS .cpu - * @return global particle ID that is unique and permanent in the whole simulation - */ - constexpr uint64_t - localIDtoGlobal (int const id, int const cpu) - { - static_assert(sizeof(int) * 2u <= sizeof(uint64_t), - "int size might cause collisions in global IDs"); - // implementation: - // - we cast both 32-bit (or smaller) ints to a 64bit unsigned int - // - this will leave half of the "upper range" bits in the 64bit unsigned int zeroed out - // because the corresponding (extended) value range was not part of the value range in - // the int representation - // - we bit-shift the cpu into the upper half of zero bits in the 64 bit unsigned int - // (imagine this step as "adding a per-cpu/rank offset to the local integers") - // - then we add this offset - // note: the add is expressed as bitwise OR (|) since this saves us from writing - // brackets for operator precedence between + and << - return uint64_t(id) | uint64_t(cpu) << 32u; - } - -} // namespace ablastr::particles - -#endif // ABLASTR_INDEX_HANDLING_H diff --git a/Source/ablastr/particles/NodalFieldGather.H b/Source/ablastr/particles/NodalFieldGather.H index b0151627198..5a10c73ae20 100644 --- a/Source/ablastr/particles/NodalFieldGather.H +++ b/Source/ablastr/particles/NodalFieldGather.H @@ -18,30 +18,36 @@ namespace ablastr::particles { /** - * \brief Compute weight of each surrounding node in interpolating a nodal field - * to the given coordinates. + * \brief Compute weight of each surrounding node (or cell-centered nodes) in interpolating a nodal ((or a cell-centered node) field + * to the given coordinates. If nodal=1, then the calculations will be done with respect to the nodes (default). If nodal=0, then the calculations will be done with respect to the cell-centered nodal) * * This currently only does linear order. * * \param xp,yp,zp Particle position coordinates * \param plo Index lower bounds of domain. * \param dxi inverse 3D cell spacing - * \param i,j,k Variables to store indices of position on grid - * \param W 2D array of weights to store each neighbouring node + * \param i,j,k Variables to store indices of position on grid (nodal or cell-centered, depending of the value of `nodal`) + * \param W 2D array of weights to store each neighbouring node (or cell-centered node) + * \param nodal Int that tells if the weights are calculated in respect to the nodes (nodal=1) of the cell-centered nodes (nodal=0) */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -void compute_weights_nodal (const amrex::ParticleReal xp, +void compute_weights (const amrex::ParticleReal xp, const amrex::ParticleReal yp, const amrex::ParticleReal zp, amrex::GpuArray const& plo, amrex::GpuArray const& dxi, - int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2]) noexcept + int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2], int nodal=1) noexcept { using namespace amrex::literals; + +#if !((nodal==0)||(nodal==1)) + ABLASTR_ABORT_WITH_MESSAGE("Error: 'nodal' has to be equal to 0 or 1"); +#endif + #if (defined WARPX_DIM_3D) - const amrex::Real x = (xp - plo[0]) * dxi[0]; - const amrex::Real y = (yp - plo[1]) * dxi[1]; - const amrex::Real z = (zp - plo[2]) * dxi[2]; + const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real y = (yp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; + const amrex::Real z = (zp - plo[2]) * dxi[2] + static_cast(nodal-1)*0.5_rt; i = static_cast(amrex::Math::floor(x)); j = static_cast(amrex::Math::floor(y)); @@ -58,17 +64,17 @@ void compute_weights_nodal (const amrex::ParticleReal xp, #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) # if (defined WARPX_DIM_XZ) - const amrex::Real x = (xp - plo[0]) * dxi[0]; + const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; amrex::ignore_unused(yp); i = static_cast(amrex::Math::floor(x)); W[0][1] = x - i; # elif (defined WARPX_DIM_RZ) - const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0]; + const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; i = static_cast(amrex::Math::floor(r)); W[0][1] = r - i; # endif - const amrex::Real z = (zp - plo[1]) * dxi[1]; + const amrex::Real z = (zp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; j = static_cast(amrex::Math::floor(z)); W[1][1] = z - j; @@ -77,7 +83,7 @@ void compute_weights_nodal (const amrex::ParticleReal xp, k = 0; #else - amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W, nodal); ABLASTR_ABORT_WITH_MESSAGE("Error: compute_weights not yet implemented in 1D"); #endif } @@ -138,7 +144,7 @@ amrex::Real doGatherScalarFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights_nodal(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); return interp_field_nodal(ii, jj, kk, W, scalar_field); } @@ -166,7 +172,7 @@ doGatherVectorFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights_nodal(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); amrex::GpuArray const field_interp = { interp_field_nodal(ii, jj, kk, W, vector_field_x), diff --git a/Source/ablastr/particles/ParticleMoments.H b/Source/ablastr/particles/ParticleMoments.H index e45fb574cce..b648ccb28aa 100644 --- a/Source/ablastr/particles/ParticleMoments.H +++ b/Source/ablastr/particles/ParticleMoments.H @@ -35,7 +35,7 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MinAndMaxPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; // Get min and max for the local rank amrex::ReduceOps< @@ -46,11 +46,11 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(PType const & p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; return amrex::makeTuple(x, y, z, x, y, z); }, @@ -90,7 +90,8 @@ namespace particles { amrex::ParticleReal, amrex::ParticleReal> MeanAndStdPositions (T_PC const & pc) { - using PType = typename T_PC::SuperParticleType; + + using ConstParticleTileDataType = typename T_PC::ParticleTileType::ConstParticleTileDataType; amrex::ReduceOps< amrex::ReduceOpSum, amrex::ReduceOpSum, amrex::ReduceOpSum, @@ -103,12 +104,14 @@ namespace particles { amrex::ParticleReal> >( pc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept + [=] AMREX_GPU_DEVICE(const ConstParticleTileDataType& ptd, const int i) noexcept { - amrex::ParticleReal const x = p.pos(0); - amrex::ParticleReal const y = p.pos(1); - amrex::ParticleReal const z = p.pos(2); - amrex::ParticleReal const w = p.rdata(T_RealSoAWeight); + + const amrex::ParticleReal x = ptd.rdata(0)[i]; + const amrex::ParticleReal y = ptd.rdata(1)[i]; + const amrex::ParticleReal z = ptd.rdata(2)[i]; + + const amrex::ParticleReal w = ptd.rdata(T_RealSoAWeight)[i]; return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); }, diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index f23f13773c1..6b476d78114 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -21,8 +21,9 @@ namespace ablastr::profiler AMREX_FORCE_INLINE void device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) + if (do_device_synchronize) { amrex::Gpu::synchronize(); + } } /** An object that conditionally calls device_synchronize() on destruction diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp index c33e6a1d154..bfe34d7d875 100644 --- a/Source/ablastr/utils/Communication.cpp +++ b/Source/ablastr/utils/Communication.cpp @@ -124,7 +124,7 @@ void FillBoundary (amrex::Vector const &mf, bool do_single_precision_comms, const amrex::Periodicity &period, std::optional nodal_sync) { - for (auto x : mf) { + for (auto *x : mf) { ablastr::utils::communication::FillBoundary(*x, do_single_precision_comms, period, nodal_sync); } } @@ -180,7 +180,7 @@ void OverrideSync (amrex::MultiFab &mf, { BL_PROFILE("ablastr::utils::communication::OverrideSync"); - if (mf.ixType().cellCentered()) return; + if (mf.ixType().cellCentered()) { return; } if (do_single_precision_comms) { diff --git a/Source/ablastr/utils/Serialization.H b/Source/ablastr/utils/Serialization.H index 29173a8eeec..b4dd2efa8ac 100644 --- a/Source/ablastr/utils/Serialization.H +++ b/Source/ablastr/utils/Serialization.H @@ -73,8 +73,9 @@ namespace ablastr::utils::serialization ", except vectors of std::string."); put_in(static_cast(val.size()), vec); - for (const auto &el : val) + for (const auto &el : val) { put_in(el, vec); + } } } @@ -145,8 +146,9 @@ namespace ablastr::utils::serialization const auto length = get_out(it); std::vector res(length); - for (int i = 0; i < length; ++i) + for (int i = 0; i < length; ++i) { res[i] = get_out(it); + } return res; } diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 18d43409ec3..9521e5f9c3d 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -146,8 +146,9 @@ SignalHandling::CheckSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } // We assume that signals will definitely be delivered to rank 0, // and may be delivered to other ranks as well. For coordination, @@ -171,12 +172,12 @@ SignalHandling::CheckSignals () } #if defined(AMREX_USE_MPI) - auto comm = amrex::ParallelDescriptor::Communicator(); // Due to a bug in Cray's MPICH 8.1.13 implementation (CUDA builds on Perlmutter@NERSC in 2022), // we cannot use the MPI_CXX_BOOL C++ datatype here. See WarpX PR #3029 and NERSC INC0183281 static_assert(sizeof(bool) == 1, "We communicate bools as 1 byte-sized type in MPI"); BL_MPI_REQUIRE(MPI_Ibcast(signal_actions_requested, SIGNAL_REQUESTS_SIZE, - MPI_BYTE, 0, comm,&signal_mpi_ibcast_request)); + MPI_BYTE, 0, amrex::ParallelDescriptor::Communicator(), + &signal_mpi_ibcast_request)); #endif } @@ -185,8 +186,9 @@ SignalHandling::WaitSignals () { // Is any signal handling action configured? // If not, we can skip all handling and the MPI communication as well. - if (!m_any_signal_action_active) + if (!m_any_signal_action_active) { return; + } #if defined(AMREX_USE_MPI) BL_MPI_REQUIRE(MPI_Wait(&signal_mpi_ibcast_request, MPI_STATUS_IGNORE)); diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index 4ba36ee82e9..629eab8c25d 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -98,25 +98,27 @@ namespace std::string abl_msg_logger::PriorityToString(const Priority& priority) { - if(priority == Priority::high) + if(priority == Priority::high) { return "high"; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return "medium"; - else + } else { return "low"; + } } Priority abl_msg_logger::StringToPriority(const std::string& priority_string) { - if(priority_string == "high") + if(priority_string == "high") { return Priority::high; - else if (priority_string == "medium") + } else if (priority_string == "medium") { return Priority::medium; - else if (priority_string == "low") + } else if (priority_string == "low") { return Priority::low; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Priority string '" + priority_string + "' not recognized"); + } //this silences a "non-void function does not return a value in all control paths" warning return Priority::low; @@ -223,8 +225,9 @@ std::vector Logger::get_msgs() const { auto res = std::vector{}; - for (const auto& msg_w_counter : m_messages) + for (const auto& msg_w_counter : m_messages) { res.emplace_back(msg_w_counter.first); + } return res; } @@ -233,8 +236,9 @@ std::vector Logger::get_msgs_with_counter() const { auto res = std::vector{}; - for (const auto& msg : m_messages) + for (const auto& msg : m_messages) { res.emplace_back(MsgWithCounter{msg.first, msg.second}); + } return res; } @@ -246,8 +250,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const #ifdef AMREX_USE_MPI // Trivial case of only one rank - if (m_num_procs == 1) + if (m_num_procs == 1) { return one_rank_gather_msgs_with_counter_and_ranks(); + } // Find out who is the "gather rank" and how many messages it has const auto my_msgs = get_msgs(); @@ -256,8 +261,9 @@ Logger::collective_gather_msgs_with_counter_and_ranks() const find_gather_rank_and_its_msgs(how_many_msgs); // If the "gather rank" has zero messages there are no messages at all - if(gather_rank_how_many_msgs == 0) + if(gather_rank_how_many_msgs == 0) { return std::vector{}; + } // All the ranks receive the msgs of the "gather rank" as a byte array const auto serialized_gather_rank_msgs = @@ -347,7 +353,7 @@ Logger::compute_msgs_with_counter_and_ranks( const std::vector& displacements, const int gather_rank) const { - if(m_rank != gather_rank) return std::vector{}; + if(m_rank != gather_rank) { return std::vector{}; } std::vector msgs_with_counter_and_ranks; @@ -369,8 +375,9 @@ Logger::compute_msgs_with_counter_and_ranks( #pragma omp parallel for #endif for(int rr = 0; rr < m_num_procs; ++rr){ //for each rank - if(rr == gather_rank) // (skip gather_rank) + if(rr == gather_rank) { // (skip gather_rank) continue; + } // get counters generated by rank rr auto it = all_data.begin() + displacements[rr]; @@ -461,8 +468,9 @@ void Logger::swap_with_io_rank( if (gather_rank != m_io_rank){ if(m_rank == gather_rank){ auto package = std::vector{}; - for (const auto& el: msgs_with_counter_and_ranks) + for (const auto& el: msgs_with_counter_and_ranks) { abl_ser::put_in_vec(el.serialize(), package); + } auto package_size = static_cast(package.size()); amrex::ParallelDescriptor::Send(&package_size, 1, m_io_rank, 0); @@ -510,9 +518,10 @@ get_serialized_gather_rank_msgs( amrex::ParallelDescriptor::Bcast( &size_serialized_gather_rank_msgs, 1, gather_rank); - if (!is_gather_rank) + if (!is_gather_rank) { serialized_gather_rank_msgs.resize( size_serialized_gather_rank_msgs); + } amrex::ParallelDescriptor::Bcast( serialized_gather_rank_msgs.data(), @@ -554,9 +563,10 @@ compute_package_for_gather_rank( // Add the additional messages seen by the current rank to the package abl_ser::put_in(static_cast(msgs_to_send.size()), package); - for (const auto& el : msgs_to_send) + for (const auto& el : msgs_to_send) { abl_ser::put_in_vec( MsgWithCounter{el.first, el.second}.serialize(), package); + } return package; } diff --git a/Source/ablastr/utils/timer/Timer.H b/Source/ablastr/utils/timer/Timer.H index efbb7d6a2bb..5df37493d61 100644 --- a/Source/ablastr/utils/timer/Timer.H +++ b/Source/ablastr/utils/timer/Timer.H @@ -44,7 +44,7 @@ namespace ablastr::utils::timer * * @return the duration */ - double get_duration () noexcept; + [[nodiscard]] double get_duration () const noexcept; /** @@ -53,7 +53,7 @@ namespace ablastr::utils::timer * * @return the maximum duration across all the MPI ranks */ - double get_global_duration (); + [[nodiscard]] double get_global_duration () const; private: diff --git a/Source/ablastr/utils/timer/Timer.cpp b/Source/ablastr/utils/timer/Timer.cpp index 5682a07c290..096c079fa2a 100644 --- a/Source/ablastr/utils/timer/Timer.cpp +++ b/Source/ablastr/utils/timer/Timer.cpp @@ -24,13 +24,13 @@ Timer::record_stop_time() noexcept } double -Timer::get_duration () noexcept +Timer::get_duration () const noexcept { return m_stop_time - m_start_time; } double -Timer::get_global_duration () +Timer::get_global_duration () const { auto duration = this->get_duration(); amrex::ParallelDescriptor::ReduceRealMax( diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index 889f2df848d..ee052ded299 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -29,15 +29,16 @@ namespace const abl_msg_logger::Priority& priority) { using namespace abl_msg_logger; - if (priority == Priority::low) + if (priority == Priority::low) { return WarnPriority::low; - else if (priority == Priority::medium) + } else if (priority == Priority::medium) { return WarnPriority::medium; - else if (priority == Priority::high) + } else if (priority == Priority::high) { return WarnPriority::high; - else + } else { ABLASTR_ABORT_WITH_MESSAGE( "Parsing Priority to WarnPriority has failed"); + } return WarnPriority::high; } @@ -59,10 +60,11 @@ void WarnManager::RecordWarning( WarnPriority priority) { auto msg_priority = abl_msg_logger::Priority::high; - if(priority == WarnPriority::low) + if(priority == WarnPriority::low) { msg_priority = abl_msg_logger::Priority::low; - else if(priority == WarnPriority::medium) + } else if(priority == WarnPriority::medium) { msg_priority = abl_msg_logger::Priority::medium; + } if(m_always_warn_immediately){ @@ -86,10 +88,11 @@ void WarnManager::RecordWarning( if(m_abort_on_warning_threshold){ auto abort_priority = abl_msg_logger::Priority::high; - if(m_abort_on_warning_threshold == WarnPriority::low) + if(m_abort_on_warning_threshold == WarnPriority::low) { abort_priority = abl_msg_logger::Priority::low; - else if(m_abort_on_warning_threshold == WarnPriority::medium) + } else if(m_abort_on_warning_threshold == WarnPriority::medium) { abort_priority = abl_msg_logger::Priority::medium; + } ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( msg_priority < abort_priority, @@ -130,8 +133,9 @@ std::string WarnManager::PrintGlobalWarnings(const std::string& when) const auto all_warnings = m_p_logger->collective_gather_msgs_with_counter_and_ranks(); - if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) + if(m_rank != amrex::ParallelDescriptor::IOProcessorNumber()) { return "[see I/O rank message]"; + } std::sort(all_warnings.begin(), all_warnings.end(), [](const auto& a, const auto& b){ @@ -217,23 +221,25 @@ std::string WarnManager::PrintWarnMsg( { std::stringstream ss; ss << "* --> "; - if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) + if (msg_with_counter.msg.priority == abl_msg_logger::Priority::high) { ss << "[!!!]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::medium) { ss << "[!! ]"; - else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) + } else if (msg_with_counter.msg.priority == abl_msg_logger::Priority::low) { ss << "[! ]"; - else + } else { ss << "[???]"; + } ss << " [" + msg_with_counter.msg.topic << "] "; - if(msg_with_counter.counter == 2) + if(msg_with_counter.counter == 2) { ss << "[raised twice]\n"; - else if(msg_with_counter.counter == 1) + } else if(msg_with_counter.counter == 1) { ss << "[raised once]\n"; - else + } else { ss << "[raised " << msg_with_counter.counter << " times]\n"; + } ss << MsgFormatter(msg_with_counter.msg.text, warn_line_size, warn_tab_size); @@ -248,8 +254,9 @@ std::string WarnManager::PrintWarnMsg( std::string raised_by = "@ Raised by: "; if (!msg_with_counter_and_ranks.all_ranks){ - for (const auto rr : msg_with_counter_and_ranks.ranks) + for (const auto rr : msg_with_counter_and_ranks.ranks) { raised_by += " " + std::to_string(rr); + } } else{ raised_by += "ALL\n"; @@ -296,8 +303,9 @@ WarnManager::MsgFormatter( msg, line_size-prefix_length); std::stringstream ss_out; - for (const auto& line : wrapped_text) + for (const auto& line : wrapped_text) { ss_out << prefix << line << "\n"; + } return ss_out.str(); } diff --git a/Tools/Algorithms/psatd_pml.ipynb b/Tools/Algorithms/psatd_pml.ipynb index 216be77fd87..897ffa70b9e 100644 --- a/Tools/Algorithms/psatd_pml.ipynb +++ b/Tools/Algorithms/psatd_pml.ipynb @@ -228,7 +228,7 @@ " #print(r'Eigenpairs:')\n", " #display(MX_eigenpairs)\n", "\n", - " # Verify that the eigenpairs satisfy the charcteristic equations\n", + " # Verify that the eigenpairs satisfy the characteristic equations\n", " for ep in MX_eigenpairs:\n", " for j in range(ep[1]):\n", " diff = MX * ep[2][j] - ep[0] * ep[2][j]\n", diff --git a/Tools/DevUtils/compute_domain.py b/Tools/DevUtils/compute_domain.py index d67cc86f53a..b54412639bd 100644 --- a/Tools/DevUtils/compute_domain.py +++ b/Tools/DevUtils/compute_domain.py @@ -11,7 +11,7 @@ The user specifies the minimal size of the physical domain and the resolution in each dimension, and the scripts computes: -- the number of cells and physical domain to satify the user-specified domain +- the number of cells and physical domain to satisfy the user-specified domain size and resolution AND make sure that the number of cells along each direction is a multiple of max_grid_size. - a starting point on how to parallelize on Cori KNL (number of nodes, etc.). diff --git a/Tools/DevUtils/update_benchmarks_from_azure_output.py b/Tools/DevUtils/update_benchmarks_from_azure_output.py index 28c3dc6b5f2..ec7b17d1050 100644 --- a/Tools/DevUtils/update_benchmarks_from_azure_output.py +++ b/Tools/DevUtils/update_benchmarks_from_azure_output.py @@ -51,7 +51,7 @@ # Raw Azure output comes with a prefix at the beginning of each line that we do # not need here. The first line that we will read is the prefix followed by the # "{" character, so we determine how long the prefix is by finding the last - # occurence of the "{" character in this line. + # occurrence of the "{" character in this line. azure_indent = line.rfind('{') first_line_read = True new_file_string += line[azure_indent:] diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py index 1be152821f6..b86c0249d01 100644 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ b/Tools/LibEnsemble/run_libensemble_on_warpx.py @@ -102,7 +102,7 @@ ('ramp_down_2', float, (1,)), # input parameter: position of the focusing lens. ('zlens_1', float, (1,)), - # Relative stength of the lens (1. is from + # Relative strength of the lens (1. is from # back-of-the-envelope calculation) ('adjust_factor', float, (1,)), ], diff --git a/Tools/PerformanceTests/run_alltests.py b/Tools/PerformanceTests/run_alltests.py index e9ff899fd2a..b1083fc6f45 100644 --- a/Tools/PerformanceTests/run_alltests.py +++ b/Tools/PerformanceTests/run_alltests.py @@ -259,7 +259,7 @@ def process_analysis(): log_line = '## year month day run_name compiler architecture n_node n_mpi ' +\ 'n_omp time_initialization time_one_iteration Redistribute '+\ 'FillBoundary ParallelCopy CurrentDeposition FieldGather '+\ - 'ParthiclePush Copy Evolve Checkpoint '+\ + 'ParticlePush Copy Evolve Checkpoint '+\ 'WriteParticles Write_FabArray '+\ 'WriteMultiLevelPlotfile '+\ 'RedistributeMPI(unit: second)\n' diff --git a/Tools/PerformanceTests/run_alltests_1node.py b/Tools/PerformanceTests/run_alltests_1node.py index f2ebb73124e..f112552b36e 100644 --- a/Tools/PerformanceTests/run_alltests_1node.py +++ b/Tools/PerformanceTests/run_alltests_1node.py @@ -26,7 +26,7 @@ # > --architecture=knl --mode=run --input_file=uniform_plasma # > --n_node=1 --log_file='my_performance_log.txt' -# ---- Running the pre-drefined automated tests ---- +# ---- Running the pre-defined automated tests ---- # Compile and run: # > python run_alltests_1node.py --automated --recompile # Just run: diff --git a/Tools/PerformanceTests/run_automated.py b/Tools/PerformanceTests/run_automated.py index 58fc02d6ac3..73ab79b00df 100644 --- a/Tools/PerformanceTests/run_automated.py +++ b/Tools/PerformanceTests/run_automated.py @@ -87,7 +87,7 @@ parser.add_argument('--n_node_list', dest='n_node_list', default=[], - help='list ofnumber of nodes for the runs', type=str) + help='list of number of nodes for the runs', type=str) parser.add_argument('--start_date', dest='start_date' ) parser.add_argument('--compiler', diff --git a/Tools/PostProcessing/plot_distribution_mapping.py b/Tools/PostProcessing/plot_distribution_mapping.py index 35fc2c197fa..19628551f26 100644 --- a/Tools/PostProcessing/plot_distribution_mapping.py +++ b/Tools/PostProcessing/plot_distribution_mapping.py @@ -98,7 +98,7 @@ def _get_costs_reduced_diagnostics(self, directory, prange): j_non_zero = j_blocks[j_blocks != 0] k_non_zero = k_blocks[k_blocks != 0] - # only one block in a dir - or smalles block size + # only one block in a dir - or smallest block size i_blocking_factor = 1 if len(i_non_zero) == 0 else i_non_zero.min() j_blocking_factor = 1 if len(j_non_zero) == 0 else j_non_zero.min() k_blocking_factor = 1 if len(k_non_zero) == 0 else k_non_zero.min() diff --git a/Tools/PostProcessing/plot_parallel.py b/Tools/PostProcessing/plot_parallel.py index e4b75c0e406..a4309b3896e 100644 --- a/Tools/PostProcessing/plot_parallel.py +++ b/Tools/PostProcessing/plot_parallel.py @@ -239,7 +239,7 @@ def reduce_evolved_quantity(z, q): file_list.sort() nfiles = len(file_list) -# Get list of particle speciess to plot +# Get list of particle species to plot pslist = get_species(file_list); rank = 0 diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 23441638893..0d55e869d6a 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -1,30 +1,33 @@ -# please set your project account +# please set your project account and uncomment the following two lines #export proj=your_project_id +#myproject -a $proj # required dependencies +module purge +module load cpe/23.12 module load craype-accel-amd-gfx90a craype-x86-trento module load PrgEnv-cray +module load CCE-GPU-3.0.0 module load amd-mixed/5.2.3 -module load CPE-23.02-cce-15.0.1-GPU-softs # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.81.0-mpi-python3 +module load boost/1.83.0-mpi-python3 # optional: for openPMD support module load cray-hdf5-parallel -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 # fix system defaults: do not escape $ with a \ on tab completion shopt -s direxpand @@ -49,7 +52,4 @@ export AMREX_AMD_ARCH=gfx90a # compiler environment hints export CC=$(which cc) export CXX=$(which CC) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" +export FC=$(which amdflang) diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index 8a4cef4a2ec..b48bf144c2a 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -20,7 +20,7 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # Remove old dependencies ##################################################### # -SW_DIR="${HOME}/sw/adastra/gpu" +SW_DIR="${SHAREDHOMEDIR}/sw/adastra/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} @@ -34,62 +34,62 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d $SHAREDHOMEDIR/src/blaspp ] then - cd $HOME/src/blaspp + cd $SHAREDHOMEDIR/src/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp fi -rm -rf $HOME/src/blaspp-adastra-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-adastra-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-adastra-gpu-build +rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build +CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d $SHAREDHOMEDIR/src/lapackpp ] then - cd $HOME/src/lapackpp + cd $SHAREDHOMEDIR/src/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp fi -rm -rf $HOME/src/lapackpp-adastra-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-adastra-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-adastra-gpu-build +rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build # c-blosc (I/O compression, for OpenPMD) -if [ -d $HOME/src/c-blosc ] +if [ -d $SHAREDHOMEDIR/src/c-blosc ] then # git repository is already there : else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $SHAREDHOMEDIR/src/c-blosc fi -rm -rf $HOME/src/c-blosc-ad-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-ad-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-ad-build +rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build +cmake -S $SHAREDHOMEDIR/src/c-blosc -B $SHAREDHOMEDIR/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build $SHAREDHOMEDIR/src/c-blosc-ad-build --target install --parallel 16 +rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build # ADIOS2 v. 2.8.3 (for OpenPMD) -if [ -d $HOME/src/adios2 ] +if [ -d $SHAREDHOMEDIR/src/adios2 ] then # git repository is already there : else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $SHAREDHOMEDIR/src/adios2 fi -rm -rf $HOME/src/adios2-ad-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/adios2-2.8.3 -cmake --build $HOME/src/adios2-ad-build --target install -j 16 -rm -rf $HOME/src/adios2-ad-build +rm -rf $SHAREDHOMEDIR/src/adios2-ad-build +cmake -S $SHAREDHOMEDIR/src/adios2 -B $SHAREDHOMEDIR/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build $SHAREDHOMEDIR/src/adios2-ad-build --target install -j 16 +rm -rf $SHAREDHOMEDIR/src/adios2-ad-build # Python ###################################################################### @@ -114,9 +114,9 @@ python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r $SHAREDHOMEDIR/src/warpx/requirements.txt # optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh index 0cb75e86e69..15a2b292b58 100644 --- a/Tools/machines/adastra-cines/submit.sh +++ b/Tools/machines/adastra-cines/submit.sh @@ -1,22 +1,26 @@ #!/bin/bash -#SBATCH --job-name=warpx #SBATCH --account= +#SBATCH --job-name=warpx #SBATCH --constraint=MI250 -#SBATCH --ntasks-per-node=8 --cpus-per-task=8 --gpus-per-node=8 -#SBATCH --threads-per-core=1 # --hint=nomultithread +#SBATCH --nodes=2 #SBATCH --exclusive #SBATCH --output=%x-%j.out #SBATCH --time=00:10:00 -#SBATCH --nodes=2 module purge -# Architecture +# A CrayPE environment version +module load cpe/23.12 +# An architecture module load craype-accel-amd-gfx90a craype-x86-trento # A compiler to target the architecture module load PrgEnv-cray # Some architecture related libraries and tools -module load amd-mixed +module load CCE-GPU-3.0.0 +module load amd-mixed/5.2.3 + +date +module list export MPICH_GPU_SUPPORT_ENABLED=1 @@ -36,4 +40,5 @@ export OMP_NUM_THREADS=1 export WARPX_NMPI_PER_NODE=8 export TOTAL_NMPI=$(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} )) srun -N${SLURM_JOB_NUM_NODES} -n${TOTAL_NMPI} --ntasks-per-node=${WARPX_NMPI_PER_NODE} \ + --cpus-per-task=8 --threads-per-core=1 --gpu-bind=closest \ ./warpx inputs > output.txt diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 896cd9edbbc..9460f9a5175 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -90,7 +90,10 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython +# cupy and h5py need an older Cython +# https://github.com/cupy/cupy/issues/4610 +# https://github.com/h5py/h5py/issues/2268 +python3 -m pip install --upgrade "cython<3.0" python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy @@ -100,6 +103,14 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +# cupy for ROCm +# https://docs.cupy.dev/en/stable/install.html#building-cupy-for-rocm-from-source +# https://github.com/cupy/cupy/issues/7830 +CC=cc CXX=CC \ +CUPY_INSTALL_USE_HIP=1 \ +ROCM_HOME=${ROCM_PATH} \ +HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ + python3 -m pip install -v cupy # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example new file mode 100644 index 00000000000..a430579a7d5 --- /dev/null +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -0,0 +1,31 @@ +. /vol0004/apps/oss/spack/share/spack/setup-env.sh + +# required dependencies +spack load cmake@3.24.3%fj@4.10.0 arch=linux-rhel8-a64fx + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + +# optional: faster builds +spack load ninja@1.11.1%fj@4.10.0 + +# optional: for PSATD +spack load fujitsu-fftw + +# optional: for QED lookup table generation support +spack load boost@1.80.0%fj@4.8.1/zc5pwgc + +# optional: for openPMD support +spack load hdf5@1.12.2%fj@4.8.1/im6lxev +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + +# compiler environment hints +export CC=$(which mpifcc) +export CXX=$(which mpiFCC) +export FC=$(which mpifrt) +export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" +export CXXFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH diff --git a/Tools/machines/fugaku-riken/install_dependencies.sh b/Tools/machines/fugaku-riken/install_dependencies.sh new file mode 100755 index 00000000000..3ceb45e4558 --- /dev/null +++ b/Tools/machines/fugaku-riken/install_dependencies.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Luca Fedeli +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Remove old dependencies ##################################################### +# +SRC_DIR="${HOME}/src/" +SW_DIR="${HOME}/sw/fugaku/a64fx/" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} + +# General extra dependencies ################################################## +# + +# c-blosc (I/O compression) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + cmake -S ${SRC_DIR}/c-blosc -B ${SRC_DIR}/c-blosc-fugaku-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1-install + cmake --build ${SRC_DIR}/c-blosc-fugaku-build --target install --parallel 48 + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + +# ADIOS2 (I/O) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${SRC_DIR}/adios2-fugaku-build +cmake -S ${SRC_DIR}/adios2 -B ${SRC_DIR}/adios2-fugaku-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3-install +cmake --build ${SRC_DIR}/adios2-fugaku-build --target install -j 48 +rm -rf ${SRC_DIR}/adios2-fugaku-build diff --git a/Tools/machines/fugaku-riken/submit.sh b/Tools/machines/fugaku-riken/submit.sh index caf254ba236..3768ed62f46 100644 --- a/Tools/machines/fugaku-riken/submit.sh +++ b/Tools/machines/fugaku-riken/submit.sh @@ -18,6 +18,12 @@ export INPUT="i.3d" export XOS_MMM_L_PAGING_POLICY=demand:demand:demand +# Add HDF5 library path to LD_LIBRARY_PATH +# This is done manually to avoid calling spack during the run, +# since this would take a significant amount of time. +export LD_LIBRARY_PATH=/vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/hdf5-1.12.2-im6lxevf76cu6cbzspi4itgz3l4gncjj/lib:$LD_LIBRARY_PATH + +# Broadcast WarpX executable to all the nodes llio_transfer ${EXE} mpiexec -stdout-proc ./output.%j/%/1000r/stdout -stderr-proc ./output.%j/%/1000r/stderr -n ${MPI_RANKS} ${EXE} ${INPUT} diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 9334d0a2287..a2b9c7b3855 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -56,7 +56,7 @@ else fi rm -rf $HOME/src/c-blosc-pm-gpu-build cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/c-blosc-pm-gpu-build # ADIOS2 @@ -71,7 +71,7 @@ else fi rm -rf $HOME/src/adios2-pm-gpu-build cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) @@ -87,7 +87,7 @@ else fi rm -rf $HOME/src/blaspp-pm-gpu-build cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) @@ -103,7 +103,7 @@ else fi rm -rf $HOME/src/lapackpp-pm-gpu-build CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 12 +cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/karolina-it4i/cleanup.sh b/Tools/machines/karolina-it4i/cleanup.sh new file mode 100755 index 00000000000..b4d46edd1e4 --- /dev/null +++ b/Tools/machines/karolina-it4i/cleanup.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +rm -rf ${HOME}/.spack ${WORK}/src/warpx ${WORK}/spack ${HOME}/.local/lib/python3.11 diff --git a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh b/Tools/machines/karolina-it4i/install_cpu_dependencies.sh deleted file mode 100755 index 2695435738f..00000000000 --- a/Tools/machines/karolina-it4i/install_cpu_dependencies.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was karolina_cpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_cpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SW_DIR="${HOME}/sw/karolina/cpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# c-blosc (I/O compression) -if [ -d $HOME/src/c-blosc ] -then - cd $HOME/src/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc -fi -rm -rf $HOME/src/c-blosc-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-cpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build - -# ADIOS2 -if [ -d $HOME/src/adios2 ] -then - cd $HOME/src/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 -fi -rm -rf $HOME/src/adios2-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-cpu-build --target install --parallel 16 -rm -rf $HOME/src/adios2-cpu-build - -# BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] -then - cd $HOME/src/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp -fi -rm -rf $HOME/src/blaspp-cpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-cpu-build -Duse_openmp=ON -Dcpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-cpu-build - -# LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] -then - cd $HOME/src/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp -fi -rm -rf $HOME/src/lapackpp-cpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-cpu-build - - -# Python ###################################################################### -# -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade virtualenv -python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-cpu -python3 -m venv ${SW_DIR}/venvs/warpx-cpu -source ${SW_DIR}/venvs/warpx-cpu/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -python3 -m pip install --upgrade cython -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -python3 -m pip install --upgrade matplotlib -python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh new file mode 100755 index 00000000000..0435b5e2926 --- /dev/null +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ################################# +# +set -eu -o pipefail + +# Check: ########################################################## +# +# Was karolina_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your karolina_warpx.profile file!" + echo "Please edit its line 2 to continue!" + return +fi + +# download and activate spack +# this might take about ~ 1 hour +if [ ! -d "$WORK/spack" ] +then + git clone -c feature.manyFiles=true -b v0.21.0 https://github.com/spack/spack.git $WORK/spack + source $WORK/spack/share/spack/setup-env.sh +else + # If the directory exists, checkout v0.21.0 branch + cd $WORK/spack + git checkout v0.21.0 + git pull origin v0.21.0 + source $WORK/spack/share/spack/setup-env.sh + + # Delete spack env if present + spack env deactivate || true + spack env rm -y warpx-karolina-cuda || true + + cd - +fi + +# create and activate the spack environment +spack env create warpx-karolina-cuda $WORK/src/warpx/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml +spack env activate warpx-karolina-cuda +spack install + +# Python ########################################################## +# +python -m pip install --user --upgrade pandas +python -m pip install --user --upgrade matplotlib +# optional +#python -m pip install --user --upgrade yt + +# install or update WarpX dependencies +python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade lasy + +# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +# python -m pip install --user --upgrade -r $WORK/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example deleted file mode 100644 index c0a3ed53ee3..00000000000 --- a/Tools/machines/karolina-it4i/karolina_cpu_warpx.profile.example +++ /dev/null @@ -1,61 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module load GCCcore/11.3.0 -module load CMake/3.23.1-GCCcore-11.3.0 -module load OpenMPI/4.1.4-GCC-11.3.0 - -# optional: for QED support with detailed tables -module load Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -module load OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/cpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/cpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/cpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#module load ccache - -# optional: for Python bindings or libEnsemble -module load Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/cpu/venvs/warpx-cpu" ] -then - source ${HOME}/sw/karolina/cpu/venvs/warpx-cpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -#alias getNode="salloc -N 1 --ntasks-per-node=4 -t 1:00:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -#alias runNode="srun -N 1 --ntasks-per-node=4 -t 0:30:00 -q interactive -C gpu --gpu-bind=single:1 -c 32 -G 4 -A $proj" - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) diff --git a/Tools/machines/karolina-it4i/karolina_gpu.qsub b/Tools/machines/karolina-it4i/karolina_gpu.qsub deleted file mode 100644 index 274184ed1ca..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu.qsub +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -l - -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Authors: Axel Huebl, Andrei Berceanu -# License: BSD-3-Clause-LBNL - -#PBS -q qgpu -#PBS -N WarpX -# Use two full nodes, 8 GPUs per node, 16 GPUs total -#PBS -l select=2:ncpus=128:ngpus=8:mpiprocs=8:ompthreads=16,walltime=00:10:00 -#PBS -A - -cd ${PBS_O_WORKDIR} - -# executable & inputs file or python interpreter & PICMI script here -EXE=./warpx.rz -INPUTS=inputs_rz - -# OpenMP threads per MPI rank -export OMP_NUM_THREADS=16 - -# run -mpirun -np ${PBS_NP} bash -c " - export CUDA_VISIBLE_DEVICES=\${OMPI_COMM_WORLD_LOCAL_RANK}; - ${EXE} ${INPUTS}" \ - > output.txt diff --git a/Tools/machines/karolina-it4i/karolina_gpu.sbatch b/Tools/machines/karolina-it4i/karolina_gpu.sbatch new file mode 100644 index 00000000000..6171ff03abc --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_gpu.sbatch @@ -0,0 +1,40 @@ +#!/bin/bash -l + +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Authors: Axel Huebl, Andrei Berceanu +# License: BSD-3-Clause-LBNL + +#SBATCH --account= +#SBATCH --partition=qgpu +#SBATCH --time=00:10:00 +#SBATCH --job-name=WarpX +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=16 +#SBATCH --gpus-per-node=8 +#SBATCH --gpu-bind=single:1 + +#SBATCH --mail-type=ALL +# change me! +#SBATCH --mail-user=someone@example.com +#SBATCH --chdir=/scratch/project//it4i-/runs/warpx + +#SBATCH -o stdout_%j +#SBATCH -e stderr_%j + +# OpenMP threads per MPI rank +export OMP_NUM_THREADS=16 +export SRUN_CPUS_PER_TASK=16 + +# set user rights to u=rwx;g=r-x;o=--- +umask 0027 + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx.rz +INPUTS=./inputs_rz + +# run +srun -K1 ${EXE} ${INPUTS} diff --git a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example deleted file mode 100644 index 174598acaac..00000000000 --- a/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example +++ /dev/null @@ -1,65 +0,0 @@ -# please set your project account -export proj="" # change me! - -# remembers the location of this script -export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi - -# required dependencies -module purge -ml GCCcore/11.3.0 -ml CUDA/11.7.0 -ml OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 -ml CMake/3.23.1-GCCcore-11.3.0 - -# optional: for QED support with detailed tables -ml Boost/1.79.0-GCC-11.3.0 - -# optional: for openPMD and PSATD+RZ support -ml OpenBLAS/0.3.20-GCC-11.3.0 -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/karolina/gpu/lapackpp-master:$CMAKE_PREFIX_PATH - -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/karolina/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH - -export PATH=${HOME}/sw/karolina/gpu/hdf5-1.14.1.2/bin:${PATH} -export PATH=${HOME}/sw/karolina/gpu/adios2-2.8.3/bin:${PATH} - -# optional: CCache (not found) -#ml ccache - -# optional: for Python bindings or libEnsemble -ml Python/3.10.4-GCCcore-11.3.0-bare - -if [ -d "${HOME}/sw/karolina/gpu/venvs/warpx-gpu" ] -then - source ${HOME}/sw/karolina/gpu/venvs/warpx-gpu/bin/activate -fi - -# an alias to request an interactive batch node for one hour (TODO) -# for parallel execution, start on the batch node: srun -alias getNode="qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00 -I" -# an alias to run a command on a batch node for up to 1hr -# usage: runNode -alias runNode='echo -e "#!/bin/bash\nmpirun -n 4 $1" | qsub -q qgpu -A $proj -l select=1:ncpus=32:ngpus=4 -l walltime=1:00:00' - -# optimize CUDA compilation for A100 -export AMREX_CUDA_ARCH=8.0 - -# optimize CPU microarchitecture for ... (TODO) -#export CXXFLAGS="-march=abc" -#export CFLAGS="-march=def" - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/karolina_warpx.profile.example b/Tools/machines/karolina-it4i/karolina_warpx.profile.example new file mode 100644 index 00000000000..1a4eda19a23 --- /dev/null +++ b/Tools/machines/karolina-it4i/karolina_warpx.profile.example @@ -0,0 +1,65 @@ +# please set your project account, ie DD-N-N +export proj="" # change me! + +# Name and Path of this Script ################### (DO NOT change!) +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) + +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file!" + echo "Please edit its line 2 to continue!" + return +fi + +# set env variable storing the path to the work directory +# please check if your project ID belongs to proj1, proj2, proj3 etc +export WORK="/mnt/proj/${proj,,}/${USER}" # change me! +mkdir -p WORK + +# clone warpx +# you can also clone your own fork here, eg git@github.com:/WarpX.git +if [ ! -d "$WORK/src/warpx" ] +then + git clone https://github.com/ECP-WarpX/WarpX.git $WORK/src/warpx +fi + +# load required modules +module purge +module load OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + +source $WORK/spack/share/spack/setup-env.sh && spack env activate warpx-karolina-cuda && { + echo "Spack environment 'warpx-karolina-cuda' activated successfully." +} || { + echo "Failed to activate Spack environment 'warpx-karolina-cuda'. Please run install_dependencies.sh." +} + +# Text Editor for Tools ########################## (edit this line) +# examples: "nano", "vim", "emacs -nw" or without terminal: "gedit" +#export EDITOR="nano" # change me! + +# allocate an interactive shell for one hour +# usage: getNode 2 # allocates two interactive nodes (default: 1) +function getNode() { + if [ -z "$1" ] ; then + numNodes=1 + else + numNodes=$1 + fi + export OMP_NUM_THREADS=16 + srun --time=1:00:00 --nodes=$numNodes --ntasks=$((8 * $numNodes)) --ntasks-per-node=8 --cpus-per-task=16 --exclusive --gpus-per-node=8 -p qgpu -A $proj --pty bash +} + +# Environment ##################################################### +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH="8.0" +export SCRATCH="/scratch/project/${proj,,}/${USER}" + +# optimize CPU microarchitecture for AMD EPYC 7763 (zen3) +export CFLAGS="-march=znver3" +export CXXFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml new file mode 100644 index 00000000000..1cb6a4ac209 --- /dev/null +++ b/Tools/machines/karolina-it4i/spack-karolina-cuda.yaml @@ -0,0 +1,80 @@ +spack: + specs: +# Karolina's openssl version is deprecated + - openssl certs=system + - pkgconfig + - ccache + - cmake@3.26.5 + - cuda@11.7.0 + - openmpi@4.1.4 +atomics + - fftw + - hdf5@1.14.0 + - adios2@2.9.2 ~mgard + - blaspp + - lapackpp + - boost@1.81.0 +program_options +atomic ~python + - python@3.11.6 + - py-pip + - py-setuptools + - py-wheel + - py-cython + - py-mpi4py + - py-numpy@1.24.2 + - openpmd-api@0.15.2 +python + - py-periodictable@1.5.0 + - py-h5py +# optional +# - py-libensemble +nlopt + + packages: + openssh: + externals: + - spec: openssh@7.4p1 + prefix: /usr + buildable: False + cuda: + externals: + - spec: cuda@11.7.0 + modules: + - CUDA/11.7.0 + buildable: False + mpi: + buildable: False + openmpi: + externals: + - spec: openmpi@4.1.4 +atomics +cuda %gcc@11.3.0 + modules: + - OpenMPI/4.1.4-GCC-11.3.0-CUDA-11.7.0 + libfabric: + externals: + - spec: libfabric@1.15.1 %gcc@11.3.0 + modules: + - libfabric/1.15.1-GCCcore-11.3.0 + buildable: False + all: + target: [zen3] + compiler: [gcc@11.3.0] + variants: +mpi ~fortran +cuda cuda_arch=80 + providers: + mpi: [openmpi@4.1.4] + cuda: [cuda@11.7.0] + + compilers: + - compiler: + modules: [GCCcore/11.3.0] + operating_system: centos7 + paths: + cc: /apps/all/GCCcore/11.3.0/bin/gcc + cxx: /apps/all/GCCcore/11.3.0/bin/g++ + f77: /apps/all/GCCcore/11.3.0/bin/gfortran + fc: /apps/all/GCCcore/11.3.0/bin/gfortran + spec: gcc@=11.3.0 + target: x86_64 + flags: {} + environment: {} + extra_rpaths: [] + + view: true + concretizer: + reuse: false + unify: true diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index 19c74348a99..abad909943c 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -10,7 +10,7 @@ module load cuda/12.0.0 module load boost/1.70.0 # optional: for openPMD support -SRC_DIR="/usr/workspace/${USER}/lassen-toss3/src" +SRC_DIR="/usr/workspace/${USER}/lassen/src" SW_DIR="/usr/workspace/${USER}/lassen-toss3/gpu" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 74b8aa8df17..2cb44035ce4 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -2,9 +2,9 @@ #export proj= # required dependencies -module load LUMI/23.03 partition/G +module load LUMI/23.09 partition/G module load rocm/5.2.3 # waiting for 5.5 for next bump -module load buildtools/23.03 +module load buildtools/23.09 # optional: just an additional text editor module load nano @@ -16,16 +16,16 @@ export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load Boost/1.81.0-cpeCray-23.03 +module load Boost/1.82.0-cpeCray-23.09 # optional: for openPMD support -module load cray-hdf5/1.12.2.3 +module load cray-hdf5/1.12.2.7 export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.10.10 # an alias to request an interactive batch node for one hour # for paralle execution, start on the batch node: srun @@ -41,9 +41,13 @@ export MPICH_GPU_SUPPORT_ENABLED=1 export AMREX_AMD_ARCH=gfx90a # compiler environment hints -export CC=$(which cc) -export CXX=$(which CC) -export FC=$(which ftn) +# Warning: using the compiler wrappers cc and CC +# instead of amdclang and amdclang++ +# currently results in a significant +# loss of performances +export CC=$(which amdclang) +export CXX=$(which amdclang++) +export FC=$(which amdflang) export CFLAGS="-I${ROCM_PATH}/include" export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/submit.sh index f6be702300a..d784471acd5 100644 --- a/Tools/machines/lumi-csc/submit.sh +++ b/Tools/machines/lumi-csc/submit.sh @@ -10,7 +10,7 @@ #SBATCH --gpus-per-node=8 #SBATCH --time=00:10:00 -export MPICH_GPU_SUPPORT_ENABLED=1 +date # note (12-12-22) # this environment setting is currently needed on LUMI to work-around a @@ -30,7 +30,11 @@ export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # the home directory, which does not scale. export ROCFFT_RTC_CACHE_PATH=/dev/null -export OMP_NUM_THREADS=1 +# Seen since August 2023 +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export FI_CXI_RX_MATCH_MODE=software # LUMI documentation suggests using the following wrapper script # to set the ROCR_VISIBLE_DEVICES to the value of SLURM_LOCALID @@ -47,9 +51,21 @@ chmod +x ./select_gpu sleep 1 # LUMI documentation suggests using the following CPU bind -# so that the node local rank and GPU ID match +# in order to have 6 threads per GPU (blosc compression in adios2 uses threads) # see https://docs.lumi-supercomputer.eu/runjobs/scheduled-jobs/lumig-job/ -CPU_BIND="map_cpu:48,56,16,24,1,8,32,40" +# +# WARNING: the following CPU_BIND options don't work on the dev-g partition. +# If you want to run your simulation on dev-g, please comment them +# out and replace them with CPU_BIND="map_cpu:49,57,17,25,1,9,33,41" +# +CPU_BIND="mask_cpu:7e000000000000,7e00000000000000" +CPU_BIND="${CPU_BIND},7e0000,7e000000" +CPU_BIND="${CPU_BIND},7e,7e00" +CPU_BIND="${CPU_BIND},7e00000000,7e0000000000" + +export OMP_NUM_THREADS=6 + +export MPICH_GPU_SUPPORT_ENABLED=1 srun --cpu-bind=${CPU_BIND} ./select_gpu ./warpx inputs | tee outputs.txt rm -rf ./select_gpu diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 8dd64c56c58..3fe76359953 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -130,9 +130,9 @@ python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda11x # CUDA 11.7 compatible wheel +python3 -m pip install cupy-cuda12x # CUDA 12 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel +python3 -m pip install --upgrade torch # CUDA 12 compatible wheel python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 0150ded4839..67ae36c9f1a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -7,14 +7,14 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cpu -module load cmake/3.22.0 -module load cray-fftw/3.3.10.3 +module load cmake/3.24.3 +module load cray-fftw/3.3.10.6 # optional: for QED support with detailed tables export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.7 +module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -28,10 +28,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/ export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx" ] then diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 13ab2ead605..7bf5dd13116 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -6,13 +6,19 @@ export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi # required dependencies -module load cmake/3.22.0 +module load gpu +module load PrgEnv-gnu +module load craype +module load craype-x86-milan +module load craype-accel-nvidia80 +module load cudatoolkit +module load cmake/3.24.3 # optional: for QED support with detailed tables export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/boost-1.82.0-ow5r5qrgslcwu33grygouajmuluzuzv3 # optional: for openPMD and PSATD+RZ support -module load cray-hdf5-parallel/1.12.2.7 +module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH @@ -26,10 +32,10 @@ export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-mast export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} # optional: CCache -export PATH=/global/common/software/spackecp/perlmutter/e4s-23.05/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8-eqk2d3bipbpkgwxq7ujlp6mckwal4dwz/bin:$PATH +export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/opt/spack/linux-sles15-zen3/gcc-11.2.0/ccache-4.8.2-cvooxdw5wgvv2g3vjxjkrpv6dopginv6/bin:$PATH # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx" ] then diff --git a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh similarity index 53% rename from Tools/machines/karolina-it4i/install_gpu_dependencies.sh rename to Tools/machines/polaris-alcf/install_gpu_dependencies.sh index a03bb008816..dbc66b2ffff 100755 --- a/Tools/machines/karolina-it4i/install_gpu_dependencies.sh +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -1,26 +1,24 @@ #!/bin/bash # -# Copyright 2023 The WarpX Community +# Copyright 2024 The WarpX Community # # This file is part of WarpX. # -# Author: Axel Huebl +# Authors: Axel Huebl, Roelof Groenewald # License: BSD-3-Clause-LBNL # Exit on first error encountered ############################################# # set -eu -o pipefail - # Check: ###################################################################### # -# Was karolina_gpu_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your karolina_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - +# Was polaris_gpu_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your polaris_gpu_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi # Remove old dependencies ##################################################### # -SW_DIR="${HOME}/sw/karolina/gpu" +SW_DIR="/home/${USER}/sw/polaris/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} @@ -29,7 +27,6 @@ python3 -m pip uninstall -qq -y pywarpx python3 -m pip uninstall -qq -y warpx python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - # General extra dependencies ################################################## # @@ -43,25 +40,10 @@ then else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi -rm -rf $HOME/src/c-blosc-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-gpu-build - -# HDF5 -if [ -d $HOME/src/hdf5 ] -then - cd $HOME/src/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git src/hdf5 -fi -rm -rf $HOME/src/hdf5-build -cmake -S $HOME/src/hdf5 -B $HOME/src/hdf5-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build $HOME/src/hdf5-build --target install --parallel 16 -rm -rf $HOME/src/hdf5-build +rm -rf $HOME/src/c-blosc-pm-gpu-build +cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/c-blosc-pm-gpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -73,10 +55,10 @@ then else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi -rm -rf $HOME/src/adios2-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-gpu-build --target install --parallel 12 -rm -rf $HOME/src/adios2-gpu-build +rm -rf $HOME/src/adios2-pm-gpu-build +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build $HOME/src/adios2-pm-gpu-build --target install -j 16 +rm -rf $HOME/src/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -89,10 +71,10 @@ then else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi -rm -rf $HOME/src/blaspp-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/blaspp-gpu-build +rm -rf $HOME/src/blaspp-pm-gpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -105,20 +87,19 @@ then else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi -rm -rf $HOME/src/lapackpp-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 12 -rm -rf $HOME/src/lapackpp-gpu-build - +rm -rf $HOME/src/lapackpp-pm-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 +rm -rf $HOME/src/lapackpp-pm-gpu-build # Python ###################################################################### # python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx-gpu -python3 -m venv ${SW_DIR}/venvs/warpx-gpu -source ${SW_DIR}/venvs/warpx-gpu/bin/activate +rm -rf ${SW_DIR}/venvs/warpx +python3 -m venv ${SW_DIR}/venvs/warpx +source ${SW_DIR}/venvs/warpx/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,14 +109,15 @@ python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas python3 -m pip install --upgrade scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +MPICC="CC -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install cupy-cuda11x # CUDA 11.8 compatible wheel # optional: for libEnsemble python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) -python3 -m pip install --upgrade torch # CUDA 11.7 compatible wheel +python3 -m pip install --upgrade torch # CUDA 11.8 compatible wheel python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/polaris-alcf/polaris_gpu.pbs b/Tools/machines/polaris-alcf/polaris_gpu.pbs new file mode 100644 index 00000000000..178db6ad6a2 --- /dev/null +++ b/Tools/machines/polaris-alcf/polaris_gpu.pbs @@ -0,0 +1,36 @@ +#!/bin/bash -l + +#PBS -A +#PBS -l select=:system=polaris +#PBS -l place=scatter +#PBS -l walltime=0:10:00 +#PBS -l filesystems=home:eagle +#PBS -q debug +#PBS -N test_warpx + +# Set required environment variables +# support gpu-aware-mpi +# export MPICH_GPU_SUPPORT_ENABLED=1 + +# Change to working directory +echo Working directory is $PBS_O_WORKDIR +cd ${PBS_O_WORKDIR} + +echo Jobid: $PBS_JOBID +echo Running on host `hostname` +echo Running on nodes `cat $PBS_NODEFILE` + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=input1d + +# MPI and OpenMP settings +NNODES=`wc -l < $PBS_NODEFILE` +NRANKS_PER_NODE=4 +NDEPTH=1 +NTHREADS=1 + +NTOTRANKS=$(( NNODES * NRANKS_PER_NODE )) +echo "NUM_OF_NODES= ${NNODES} TOTAL_NUM_RANKS= ${NTOTRANKS} RANKS_PER_NODE= ${NRANKS_PER_NODE} THREADS_PER_RANK= ${NTHREADS}" + +mpiexec -np ${NTOTRANKS} ${EXE} ${INPUTS} > output.txt diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example new file mode 100644 index 00000000000..d7635188141 --- /dev/null +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -0,0 +1,55 @@ +# Set the project name +export proj="" # change me! + +# swap to GNU programming environment (with gcc 11.2) +module swap PrgEnv-nvhpc PrgEnv-gnu +module swap gcc/12.2.0 gcc/11.2.0 +module load nvhpc-mixed/22.11 + +# swap to the Milan cray package +module swap craype-x86-rome craype-x86-milan + +# required dependencies +module load cmake/3.23.2 + +# optional: for QED support with detailed tables +# module load boost/1.81.0 + +# optional: for openPMD and PSATD+RZ support +module load cray-hdf5-parallel/1.12.2.3 +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} + +# optional: for Python bindings or libEnsemble +module load cray-python/3.9.13.1 + +if [ -d "/home/${USER}/sw/polaris/gpu/venvs/warpx" ] +then + source /home/${USER}/sw/polaris/gpu/venvs/warpx/bin/activate +fi + +# necessary to use CUDA-Aware MPI and run a job +export CRAY_ACCEL_TARGET=nvidia80 + +# optimize CUDA compilation for A100 +export AMREX_CUDA_ARCH=8.0 + +# optimize CPU microarchitecture for AMD EPYC 3rd Gen (Milan/Zen3) +# note: the cc/CC/ftn wrappers below add those +export CXXFLAGS="-march=znver3" +export CFLAGS="-march=znver3" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index 5d88bb9aeed..e41521b5815 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -11,7 +11,7 @@ module load nano # required dependencies module load cmake/3.20.2 module load gcc/9.3.0 -module load cuda/11.3.1 +module load cuda/11.7.1 # optional: faster re-builds module load ccache diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index d382cf97094..9657cec146d 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,13 +250,17 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 23.11 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.02 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts list(APPEND CMAKE_MODULE_PATH "${AMReX_DIR}/AMReXCMakeModules") message(STATUS "AMReX: Found version '${AMReX_VERSION}'") + + if(WarpX_COMPUTE STREQUAL CUDA) + enable_language(CUDA) + endif() endif() endmacro() @@ -269,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "9e35dc19489dc5d312e92781cb0471d282cf8370" +set(WarpX_amrex_branch "2ecafcff40132f56eb2b494e1a374684ff97117a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 7752a82758b..71831357e66 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 23.11 CONFIG REQUIRED) + find_package(pyAMReX 24.02 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "58dc1ed58226ab683a9bdea858da2a196cd1570f" +set(WarpX_pyamrex_branch "0cbf4b08c9045e1845595c836b99f94bb3c1ac9f" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 8d097d869ff..e65cd206f34 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.10.1 CONFIG REQUIRED) + find_package(pybind11 2.11.1 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.10.1" +set(WarpX_pybind11_branch "v2.11.1" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") diff --git a/run_test.sh b/run_test.sh index 485e07d4e2f..ff41399a9c8 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 9e35dc19489dc5d312e92781cb0471d282cf8370 && cd - +cd amrex && git checkout --detach 2ecafcff40132f56eb2b494e1a374684ff97117a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index bb92d69b417..197a39ce23f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '23.11', + version = '24.02', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',