run

#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
DO_DRY_RUN="false"
HOLOHUB_PY_EXE=${HOLOHUB_PY_EXE:-python3}
HOLOHUB_PY_EXE=$(which ${HOLOHUB_PY_EXE})
HOLOHUB_PY_LIB="${HOLOHUB_PY_EXE/bin/lib}"

#===========================================================================================
# Utilities

YELLOW="\e[1;33m"
RED="\e[1;31m"
CYAN="\e[1;36m"
NOCOLOR="\e[0m"
BOLD="\e[1;37m"

# Compare versions
compare_version () {

    if [[ $1 == $2 ]]
    then
        echo "0"
        return 0
    fi

    local IFS=.
    local i ver1=($1) ver2=($2)
    # fill empty fields in ver1 with zeros
    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
    do
        ver1[i]=0
    done

    for ((i=0; i<${#ver1[@]}; i++))
    do
        if [[ -z ${ver2[i]} ]]
        then
            # fill empty fields in ver2 with zeros
            ver2[i]=0
        fi
        if ((10#${ver1[i]} > 10#${ver2[i]}))
        then
            echo "1"
            return 1
        fi
        if ((10#${ver1[i]} < 10#${ver2[i]}))
        then
            echo "2"
            return 2
        fi
    done
    echo "0"
    return 0
}


run_command() {
    local status=0
    local cmd="$*"

    if [ "${DO_DRY_RUN}" != "true" ]; then
        echo -e "${CYAN}[command]${NOCOLOR} ${cmd}"
    else
        echo -e "${CYAN}[dryrun]${NOCOLOR} ${cmd}"
    fi

    [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available

    if [ "${DO_DRY_RUN}" != "true" ]; then
        eval "$@"
        status=$?
    fi
    return $status
}


get_buildtype_str() {
    local build_type="${1:-}"
    local build_type_str

    case "${build_type}" in
        debug|Debug)
            build_type_str="Debug"
            ;;
        release|Release)
            build_type_str="Release"
            ;;
        rel-debug|RelWithDebInfo)
            build_type_str="RelWithDebInfo"
            ;;
        *)
            build_type_str="${CMAKE_BUILD_TYPE:-Release}"
            ;;
    esac

    echo -n "${build_type_str}"
}


get_app_source_root_dir() {
  # Get source path for a given application according to HoloHub convention.
  # Usage: ./run get_app_source_root_dir <app_name>
  local appname="$1"
  if [[ -z "${appname}" ]]; then
    echo "Missing app name argument for get_app_source_root_dir"
    exit 1
  fi

  holohub_app_source="${SCRIPT_DIR}/applications/${appname}"

  # Check if the application is in a subdirectory
  if [ ! -d "$holohub_app_source" ]; then
    sub_app_path=$(find ${SCRIPT_DIR}/applications -type d -name "${appname}")
    app_rel_path=${sub_app_path#"${SCRIPT_DIR}/applications/"}
    if [[ -n "$app_rel_path" ]]; then
      holohub_app_source="${SCRIPT_DIR}/applications/${app_rel_path}"
    fi
  fi

  # Check if the application is in the benchmarks folder
  if [ ! -d "$holohub_app_source" ] && [ -d "${SCRIPT_DIR}/benchmarks/${appname}" ]; then
    holohub_app_source="${SCRIPT_DIR}/benchmarks/${appname}"
  fi

  if [ ! -d "$holohub_app_source" ]; then
    echo "Could not find project ${appname}"
    exit 1
  fi

  echo -n "${holohub_app_source}"
}

get_app_source_lang_dir() {
  # Get source path for a given application language implementation according to HoloHub convention.
  # If a project defines more than one language implementation it must define one subfolder per language,
  # otherwise the implementation may be provided at the top level or in a subfolder at the contributor's discretion.
  # Usage: ./run get_app_source_lang_dir <app_name> <language>
  local appname="$1"
  local language="$2"

  local holohub_app_source=$(get_app_source_root_dir "${appname}")
  if [[ -n "${holohub_app_source}" ]] && [[ -d "${holohub_app_source}/${language}" ]]; then
    holohub_app_source="${holohub_app_source}/${language}"
  fi

  echo -n "${holohub_app_source}"
}

get_app_dockerfile() {
  # Parse the path to a custom application Dockerfile, if provided.
  # Follows the HoloHub convention:
  # 1. Check the metadata.json file for a "dockerfile" entry.
  # 2. Check for a Dockerfile in the application language implementation directory.
  # 3. Check for a Dockerfile in the root application directory.
  # 4. Use the default Dockerfile provided at the top level of the HoloHub repository.
  # Usage: ./run get_app_dockerfile <app_name> <language>
  local appname="$1"
  local language="$2"
  local dockerfile_path=""

  local app_source_root_path=$(get_app_source_root_dir ${appname})
  local app_source_lang_path=$(get_app_source_lang_dir ${appname} ${language})

  local dockerfile_entry=$(cat "${app_source_lang_path}/metadata.json" 2>/dev/null | grep "dockerfile" )
  if [[ -n "${dockerfile_entry}" ]]; then
    pattern="s#<holohub_app_source>#${app_source_lang_path}#g"
    dockerfile_path=$(echo "${dockerfile_entry}" | awk -F'["]' '{ print $4 }' | sed -E ${pattern})
  elif [[ -f "${app_source_lang_path}/Dockerfile" ]]; then
    dockerfile_path="${app_source_lang_path}/Dockerfile"
  elif [[ -f "${app_source_root_path}/Dockerfile" ]]; then
    dockerfile_path="${app_source_root_path}/Dockerfile"
  else
    dockerfile_path="${SCRIPT_DIR}/Dockerfile"
  fi

  echo -n "${dockerfile_path}"
}

get_app_default_language() {
  # If a language was not provided but multiple are available, auto-select
  # the first language matching the app name from the "./run list" command.
  # Yields an empty string if a language folder is not specified.
  # Example `list` string:
  # ultrasound_segmentation (cpp) [Sample Application]
  # Usage: ./run get_app_default_language <app_name>
  appname="$1"

  if [ -z "${language}" ]; then
    language=$(list | grep -m 1 ${appname} | awk -F'[\\(\\)]' '{print $2}')
  fi

  echo -e "${language}"
}


print_error() {
    echo -e "${RED}ERROR${NOCOLOR}:" $*
}

print_warning() {
    echo -e "${YELLOW}WARNING${NOCOLOR}:" $*
}


check_exit_code() {
  local exit_code=$1
  local message=$2

  if [ $exit_code -ne 0 ]; then
      print_error "$message (Exit code: $ret)"
      exit $exit_code
  fi
}

#===========================================================================================
# Helper function to install packages

install_cuda_dependencies_package() {
   package_name=$1
   preferred_version=$2
   optional=false
   if [ -n "$3" ]; then
     optional=true
   fi

   # Checking if compatible packages are already installed
   installed_version=$(apt list --installed ${package_name} 2>/dev/null | grep $package_name)
   if [[ $installed_version != "" ]]; then
      echo "Package $package_name found with version $installed_version"
      return 0
   fi

   available_version=$(apt list -a ${package_name} 2>/dev/null | grep $preferred_version)
   package_version=$(echo $available_version | cut -d' ' -f2)
   package_installed=$(echo $available_version | grep "installed")

   if [[ $package_version == "" ]]; then
    if [[ $optional == "true" ]]; then
      echo "Package $package_name $preferred_version not found. Skipping."
    else
      print_error "$package_name $preferred_version is not installable."
      echo "You might want to try to install a newer version manually and rerun the setup:"
      echo "  sudo apt install $package_name"
      exit 1
    fi
   elif [[ $package_installed == "" ]]; then
    echo "Installing $package_name=$package_version"
    apt install --no-install-recommends -y $package_name=$package_version
   fi
}

# Setup the environment
setup_desc() {
  echo -e "${BOLD}Install the standard set of dependencies for HoloHub applications${NOCOLOR}"
}

setup() {

  # Run apt-get update
  apt-get update

  # Install wget
  wget_version=$(dpkg --status wget | grep -Po '^Version: \K[^-]*')
  if [[ $wget_version == "" ]]; then
    apt-get install -y wget
  fi

  # Install xvfb for running tests/examples headless
  xvfb_version=$(dpkg --status xvfb | grep -Po '^Version: \K[^-]*')
  if [[ $xvfb_version == "" ]]; then
    apt-get install -y xvfb
  fi

  # Check version of CMake otherwise upgrade
  cmake_version=$(dpkg --status cmake | grep -Po '^Version: \K[^-]*')
  ubuntu_codename=$(cat /etc/os-release | grep -Po '^UBUNTU_CODENAME=\K[^ ]*')

  cmake_need_upload=$(compare_version ${cmake_version} "3.24.0")

  # If we should update cmake
  # Install from https://apt.kitware.com/
  if [[ $cmake_version == "" ]] || [[ $cmake_need_upload == 2 ]]; then
    apt install --no-install-recommends -y gpg
    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
    echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ ${ubuntu_codename} main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
    apt-get update
    apt install --no-install-recommends -y cmake cmake-curses-gui
  fi

  # Install python dev
  python3_dev_version=$(dpkg --status python3-dev  2>/dev/null | grep -Po '^Version: \K[^-]*')
  python3_min_version=$(compare_version ${python3_dev_version} "3.9.0")

  # Python 3.8 to 3.11 are supported
  if [[ $python3_dev_version == "" ]] || [[ $python3_min_version == "2" ]]; then
    echo "Installing python"
    apt install --no-install-recommends -y python3 python3-dev
    python3_dev_version=$(dpkg --status python3-dev  2>/dev/null | grep -Po '^Version: \K[^-]*')
    python3_min_version=$(compare_version ${python3_dev_version} "3.9.0")
  fi

  # If python version is too high
  python3_max_version=$(compare_version ${python3_dev_version} "3.12.0")
  if [[ $python3_max_version == "0" ]] || [[ $python3_max_version == "1" ]]; then
    print_error "Python version ${python3_dev_version} is not supported"
    exit 1
  fi

  # Install ffmpeg
  ffmpeg_version=$(dpkg --status ffmpeg 2>/dev/null | grep -Po '^Version: \K[^-]*')
  if [[ $ffmpeg_version == "" ]]; then
    echo "Installing ffmpeg"
    apt install --no-install-recommends -y ffmpeg
  fi

  # Install libv4l-dev
  libv4l_dev_version=$(dpkg --status libv4l-dev | grep -Po '^Version: \K[^-]*')
  if [[ $libv4l_dev_version == "" ]]; then
    echo "Installing libv4l-dev"
    apt-get install --no-install-recommends -y libv4l-dev
  fi

  # git
  git_path=$(which git)
  if [[ $git_path == "" ]]; then
    echo "Installing git"
    apt-get install --no-install-recommends -y git
  fi

  # Install ngc-cli
  ngc_version=$(ngc --version 2>/dev/null | grep -Po '^NGC CLI \K[^-]*')
  if [[ $ngc_version == "" ]]; then
    echo "Installing ngc cli"

    if [ $(uname -m) == "aarch64" ]; then
      wget --content-disposition https://ngc.nvidia.com/downloads/ngccli_arm64.zip && unzip ngccli_arm64.zip && chmod u+x ngc-cli/ngc
    fi
    if [ $(uname -m) == "x86_64" ]; then
      wget --content-disposition https://ngc.nvidia.com/downloads/ngccli_linux.zip && unzip ngccli_linux.zip && chmod u+x ngc-cli/ngc
    fi
    ln -s $(pwd)/ngc-cli/ngc /usr/local/bin/
  fi

  # Install the tensorrt development libs
  # Find the newest cudart version installed
  for version in 12-6 12-5 12-4 12-3 12-2 12-1 12-0 11-8 11-7 11-6 11-4
   do
      cuda_version=$(dpkg --status cuda-cudart-${version} 2>/dev/null | grep -Po '^Version: \K[^-]*')
      # Find the version based on cudart
      if [[ $cuda_version != "" ]]; then
        break
      fi
   done

  short_cuda_version=${cuda_version%.*}
  echo "Cuda version found: $short_cuda_version"

  # Install cudnn9 first
  install_cuda_dependencies_package libcudnn9-cuda-12 cuda$short_cuda_version true
  install_cuda_dependencies_package libcudnn9-dev-cuda-12 cuda$short_cuda_version true

  # Check if cudnn9 is install, if not install cudnn8
  installed_cudnn9_version=$(apt list --installed libcudnn9-cuda-12 2>/dev/null | grep libcudnn9-cuda-12)
  if [[ $installed_cudnn9_version == "" ]]; then
     install_cuda_dependencies_package libcudnn8 cuda$short_cuda_version
     install_cuda_dependencies_package libcudnn8-dev cuda$short_cuda_version
  fi

  # Check the current version of libnvinfer-bin
  installed_libnvinferbin=$(dpkg --status libnvinfer-bin 2>/dev/null | grep -Po '^Version: \K[^-]*')
  installed_libnvinferbin_version=${installed_libnvinferbin%.*}
  install_cuda_dependencies_package libnvinfer-headers-dev $installed_libnvinferbin_version
  install_cuda_dependencies_package libnvinfer-dev $installed_libnvinferbin_version
  install_cuda_dependencies_package libnvinfer-plugin-dev $installed_libnvinferbin_version
  install_cuda_dependencies_package libnvonnxparsers-dev $installed_libnvinferbin_version

  # Install the autocomplete
  echo "Installing autocomplete"
  cp ${SCRIPT_DIR}/utilities/holohub_autocomplete /etc/bash_completion.d/

  echo "Setup for HoloHub is ready. Happy Holocoding!"
}

#===========================================================================================
# Lint

install_lint_deps_desc() {
  echo -e "
  ${BOLD}Install lint dependencies${NOCOLOR}

  This command will install the dependencies required to run the linting tools.

  - Python linting: ruff, isort, black, mypy
  - CMake linting: cmakelint
  - C++ linting: cpplint, clang-format
  - Spelling: codespell
  "
}

install_lint_deps() {
    # We use $(command) || exit_code=1 to run all linting tools, and exit
    # with failure after all commands were executed if any of them failed
    local exit_code=0

    pushd ${SCRIPT_DIR} > /dev/null

    echo "Install Lint Dependencies for Python"
    run_command ${HOLOHUB_PY_EXE} -m \
      pip install -r ${SCRIPT_DIR}/utilities/requirements.lint.txt || exit_code = 1
    run_command apt install --no-install-recommends -y \
      clang-format="1:14.0*" || exit_code=1

    popd > /dev/null

    exit $exit_code
}

lint_desc() {
echo -e "
${BOLD}Lint the repository${NOCOLOR}

Python linting: black, isort, ruff
C++ linting: cpplint
CMake linting: cmakelint
Spelling: codespell

${CYAN}USAGE:${NOCOLOR} ./run lint <directories to lint (default: .)>

${CYAN}Options:${NOCOLOR}
  --fix: to fix all linting errors
"
}

lint() {

    # Parse the arguments
    ARGS=("$@")
    local DIR_TO_RUN="."
    local FIX=false

    for i in "${!ARGS[@]}"; do
      arg="${ARGS[i]}"
      if [ "$arg" = "--fix" ]; then
         FIX=true
      elif [[ $arg = -* ]]; then
        print_error "Unknown option $arg"
        exit 1
      else
        DIR_TO_RUN=$arg
      fi
    done

    local exit_code=0

    # If we call with --fix
    if [ $FIX == true ]; then
      # Fix python
      run_command ruff check --fix --ignore E712 ${DIR_TO_RUN} || exit_code=1
      # Fix python isort issues, run:
      run_command isort ${DIR_TO_RUN} || exit_code=1
      # Fix python black code formatting issues, run:
      run_command black ${DIR_TO_RUN} || exit_code=1
      run_command codespell -w -i 3 ${DIR_TO_RUN} --ignore-words codespell_ignore_words.txt \
                            --skip="*.onnx,*.min.js,*.min.js.map,Contrastive_learning_Notebook.ipynb,./data" || exit_code=1

      # Fix cpplint with clang
      files_to_fix=`set -o pipefail; ${HOLOHUB_PY_EXE} -m cpplint \
            --exclude build \
            --exclude install \
            --exclude build-\* \
            --exclude install-\* \
            --exclude applications/holoviz/template/cookiecutter\* \
            --recursive $DIR_TO_RUN 2>&1 | grep -v 'Ignoring\|Done processing\|Total error' | sed 's/:.*//' | sort | uniq`

      # Fix C++ lint issues by running clang-format:
      for file in ${files_to_fix}; do
        run_command clang-format --style=file --sort-includes=0 --lines=20:10000 -i ${file} || exit_code=1
      done

      exit $exit_code
    fi


    # We use $(command) || exit_code=1 to run all linting tools, and exit
    # with failure after all commands were executed if any of them failed
    pushd ${SCRIPT_DIR} > /dev/null

    echo "Linting Python"
    run_command ruff check $DIR_TO_RUN --ignore E712 || exit_code=1
    run_command ${HOLOHUB_PY_EXE} -m isort -c $DIR_TO_RUN || exit_code=1
    run_command ${HOLOHUB_PY_EXE} -m black --check $DIR_TO_RUN || exit_code=1

    echo "Linting C++"
    # We use `grep -v` to hide verbose output that drowns actual errors
    # Since we care about the success/failure of cpplitn and not of grep, we:
    #  1. use `set -o pipefail` to fail if `cpplint` fails
    #  2. use `grep -v ... || true` to ignore whether grep hid any output
    run_command "set -o pipefail; ${HOLOHUB_PY_EXE} -m cpplint \
            --exclude build \
            --exclude install \
            --exclude build-\* \
            --exclude install-\* \
            --exclude .vscode-server \
            --exclude applications/holoviz/template/cookiecutter\* \
            --recursive $DIR_TO_RUN \
        | { grep -v 'Ignoring\|Done processing' || true; } || exit_code=1"


    echo "Code spelling"
    run_command codespell $DIR_TO_RUN --skip="*.onnx,*.min.js,*.min.js.map,Contrastive_learning_Notebook.ipynb,./data" \
                          --ignore-words codespell_ignore_words.txt \
                          --exclude-file codespell.txt || exit_code=1

    echo "Linting CMake"
    cmakelint --filter=-whitespace/indent,-linelength,-readability/wonkycase,-convention/filename,-package/stdargs \
              $(find $DIR_TO_RUN '(' -name CMakeLists.txt -o -name *.cmake ')' -not -path "*build/*" -not -path "*./.*" -not -path "*install/*" -not -path "*tmp/*") || exit_code=1

    popd > /dev/null

    if [ $exit_code == 0 ]; then
        echo "Everything looks good!"
    fi

    exit $exit_code
}


#===========================================================================================
# Build HoloHub targets

build_desc() {
  echo
  echo -e "${BOLD}Build a Holohub operator, application, or package, in its isolated build directory.${NOCOLOR}"
  echo
  echo -e "${CYAN}USAGE:${NOCOLOR}"
  echo
  echo "  ./run build <target> [options]"
  echo "  ./run build ./path/to/<target> [options]"
  echo
  echo -e "${CYAN}Options:${NOCOLOR}"
  echo
  echo "   --sdk <path_to_holoscan_SDK>         : Provide path to the SDK"
  echo "   --with <list of operators>           : Optional operators that should be built"
  echo "                                          If multiple operators are to be built, list must be"
  echo "                                          in quotes with operators separated by semicolons (;)"
  echo "   --type <debug | release | rel-debug> : Specify the type of build"
  echo "                                          Default: release"
  echo "                                          Associated environment variable: CMAKE_BUILD_TYPE"
  echo "   --buildpath <build_directory>        : Change the build path."
  echo "                                          Default: build"
  echo "                                          Associated environment variable: CMAKE_BUILD_PATH"
  echo "   --benchmark                          : Build for Holoscan Flow Benchmarking. Valid for"
  echo "                                          applications only"
  echo "   --install                            : Install the target and its dependencies to"
  echo "                                          install/<target> if supported."
  echo "   --configure-args <extra_args>        : Additional configuration arguments"
  echo "                                          multiple arguments can be passed between quotes"
  echo "                                          or using several --configure-args in the command line"
  echo "   --parallel <jobs>                    : Build in parallel using the given number of jobs."
  echo "                                          Only needed to override the native build tool's default."
  echo "                                          Example: --parallel $(($(nproc)-1))"
  echo
  echo "Prior to the first build use './run setup' to install the required dependencies."
  echo
}

# See how many directories under the input path matches "install*" pattern
# If we find exactly one, this is probably the SDK install path the user wants to use
# Print out a warning message if we find multiple install paths and ask user to specify exact path using --sdk option
get_install_path() {
    dir=$1
    install_paths=($(find $dir -maxdepth 1 -type d -name "install*" 2>/dev/null))
    if [ ${#install_paths[@]} -eq 1 ]; then
        echo "${install_paths[0]}"
    elif [ ${#install_paths[@]} -gt 1 ]; then
        echo "Warning: Multiple install directories found under $dir, please specify exact path to --sdk option with one of the following directories" >&2
        for path in "${install_paths[@]}"; do
            echo "- $path" >&2
        done
        echo ""
    else
        echo ""
    fi
}

# Allow users to point "--sdk" to root directory of the SDK installation.
# This is to address the differences in directory structure between
# installations between debian image, docker, or when built from source
resolve_sdk_install_path() {
  sdk_path=$1

  install_path=$(get_install_path "${sdk_path}/public")
  if [[ -n $install_path ]]; then
    echo "${install_path}";
    return
  fi

  install_path=$(get_install_path "${sdk_path}/")
  if [[ -n $install_path ]]; then
    echo "${install_path}";
    return
  fi

  echo "${sdk_path}"
}

build() {
  # Holoscan SDK location
  holoscan_sdk=""

  # CMake configuration args
  configure_args=""

  # Parse the arguments
  ARGS=("$@")
  local app
  local i
  local arg
  local skipnext=0
  local build_type="${CMAKE_BUILD_TYPE:-release}"
  local build_path="${CMAKE_BUILD_PATH}"
  local benchmark=false
  local install=false
  local parallel_jobs=""

  for i in "${!ARGS[@]}"; do
      arg="${ARGS[i]}"
      if [[ $skipnext == "1" ]]; then
         skipnext=0
      elif [ "$arg" = "--sdk" ]; then
         sdk_path=$( resolve_sdk_install_path "${ARGS[i+1]}" )
         holoscan_sdk="-Dholoscan_ROOT=${sdk_path}"
         echo "Setting Holoscan SDK install path to ${sdk_path}"
         skipnext=1
      elif [ "$arg" = "--with" ]; then
         operators="${ARGS[i+1]}"
         configure_args="${configure_args} -DHOLOHUB_BUILD_OPERATORS=\"${operators}\""
         echo "Building with operator(s) ${operators}"
         skipnext=1
      elif [ "$arg" = "--benchmark" ]; then
          benchmark=true
          configure_args="${configure_args} -DCMAKE_CXX_FLAGS=-I$PWD/benchmarks/holoscan_flow_benchmarking"
          echo "Building for Holoscan Flow Benchmarking"
      elif [ "$arg" = "--install" ]; then
          install=true
      elif [ "$arg" = "--configure-args" ]; then
         configure_args="${configure_args} ${ARGS[i+1]}"
         echo "Adding configuration arguments: ${ARGS[i+1]}"
         skipnext=1
      elif [ "$arg" = "--type" ]; then
         build_type=$(get_buildtype_str "${ARGS[i+1]}")
         skipnext=1
      elif [ "$arg" = "--buildpath" ]; then
         build_path="${ARGS[i+1]}"
         skipnext=1
      elif [ "$arg" = "--parallel" ]; then
         parallel_jobs="${ARGS[i+1]}"
         echo "Setting the number of parallel build jobs: ${ARGS[i+1]}"
         skipnext=1
      elif [[ $arg = -* ]]; then
        print_error "Unknown option $arg"
        exit 1
      else
        app=$arg
      fi
  done

  # Package, application or operator to build
  if [ "$1" == "" ]; then
    print_error "Please specify which package, application, or operator to build."
    echo "You can run ./run list for a full list of options."
    exit 1
  fi
  project=$(basename "$1")
  opt_prefix=""
  is_app=false
  is_pkg=false
  if list_packages | awk '{print $1}' | grep -Fxq "$project"; then
    opt_prefix="-D PKG_"
    is_pkg=true
  elif list_apps | awk '{print $1}' | grep -Fxq "$project"; then
    opt_prefix="-D APP_"
    is_app=true
  elif list_operators | awk '{print $1}' | grep -Fxq "$project"; then
    opt_prefix="-D OP_"
  else
    print_error "Unknown package, application, or operator: $1"
    echo "You can run ./run list for a full list of options."
    exit 1
  fi
  echo "Building $project"
  opt_flag="$opt_prefix$project:BOOL=1"

  # Assign default build path
  if [ "$build_path" == "" ]; then
    build_path="build/$project"
  fi

  local app_source_root_path
  if [ $benchmark == true ]; then
    if [ $is_app == true ]; then
      app_source_root_path=$(get_app_source_root_dir $project)
      run_command benchmarks/holoscan_flow_benchmarking/patch_application.sh ${app_source_root_path}
    else
      print_warning "Requesting benchmarking with --benchmark is only available for applications, ignoring"
    fi
  fi

  # We define the python path to make sure we grab the right one
  cmake_extra_args="--no-warn-unused-cli -D Python3_EXECUTABLE=${HOLOHUB_PY_EXE} -D Python3_ROOT_DIR=${HOLOHUB_PY_LIB}"

  # We set the data directory to be outside the build directory
  cmake_extra_args="$cmake_extra_args $configure_args -D HOLOHUB_DATA_DIR:PATH=${SCRIPT_DIR}/data"

  # Sets the default path for cuda
  export PATH=$PATH:/usr/local/cuda/bin
  run_command cmake -S . -B ${build_path} ${cmake_extra_args} -D CMAKE_BUILD_TYPE=${build_type} ${holoscan_sdk} ${opt_flag}
  ret=$?
  check_exit_code $ret "Error building $project."
  # Job concurrency determined by the underlying build tool unless a number is specified
  run_command cmake --build ${build_path} -j ${parallel_jobs}
  ret=$?
  check_exit_code $ret "Error building $project."

  if [ $install == true ]; then
    run_command cmake --install ${build_path}
    ret=$?
    check_exit_code $ret "Error installing $project."
  fi

  if [ $benchmark == true ] && [ $is_app == true ]; then
    app_source_root_path=$(get_app_source_root_dir $1)
    run_command benchmarks/holoscan_flow_benchmarking/restore_application.sh ${app_source_root_path}
  fi

  if [ $is_pkg == true ]; then
    for cpack_config in $(find ${build_path}/pkg -name "CPackConfig-*.cmake"); do
      run_command cpack --config "$cpack_config"
    done
  fi

  echo "Build done."
}


# Launch a sample app
launch_desc() {
  echo -e "
  ${BOLD}Run an application.${NOCOLOR}

  ${CYAN}USAGE:${NOCOLOR} ./run launch <application_name> [cpp|python] [options]

  ${CYAN}Options:${NOCOLOR}
     --extra_args <args>       : additional arguments passed to the application command
     --nsys_profile            : profile using Nsight Systems
     --verbose                 : output additional run command information to stdout


  Use './run list' or './run list_apps' to list the available applications.
  Make sure you call './run build <application_name>' to first build the application.
  "
}


launch() {
   local appname=""
   local language=""
   local extra_args=""
   local nsys_profile=false
   local verbose=false
   local holohub_build_dir=""
   local skipnext=0

   for i in "${!ARGS[@]}"; do
     arg="${ARGS[i]}"
     if [[ $skipnext == "1" ]]; then
       skipnext=0
     elif [ "$arg" = "--extra_args" ]; then
       extra_args="${ARGS[i+1]}"
       skipnext=1
     elif [ "$arg" = "--nsys_profile" ]; then
       nsys_profile=true
     elif [ "$arg" = "--buildpath" ]; then
       holohub_build_dir="${ARGS[i+1]}"
       skipnext=1
     elif [ "$arg" = "--verbose" ]; then
       verbose=true
     elif [ -z "${appname}" ]; then
       appname=$arg
     elif [ -z "${language}" ]; then
       language=$arg
     fi
   done

   # Export the python path (by default use the one used to build)
   if [ "$holohub_build_dir" == "" ]; then
      holohub_build_dir="${SCRIPT_DIR}/build/$appname"
   else
     holohub_build_dir=$(readlink --canonicalize $holohub_build_dir)
   fi

   # Do initial sanitary check to make sure build/ is there, to avoid too many other error messages
   if [ ! -d "$holohub_build_dir" ]; then
     print_error "The build directory ${holohub_build_dir} does not exist."
     echo "Did you forget to build your application using './run build $appname' ?"
     build_desc
     exit 1
   fi

   local holoscan_sdk_install=$(grep -Po '^holoscan_DIR:PATH=\K[^ ]*' ${holohub_build_dir}/CMakeCache.txt)
   local holohub_data_dir=$(grep -Po '^HOLOHUB_DATA_DIR:PATH=\K[^ ]*' ${holohub_build_dir}/CMakeCache.txt)

   if [[ -z "${language}" ]]; then
     language=$(get_app_default_language "${appname}")
     language=${language:-cpp}
     if [[ -n "${language}" ]]; then
       echo "Default language for ${appname} selected: ${language}"
     fi
   fi

   holohub_app_source=$(get_app_source_lang_dir "${appname}" "${language}")
   if [[ -z "${holohub_app_source}" ]]; then
     print_error "Could not find app source directory for ${appname} ${language}."
     launch_desc
     exit 1
   fi

   # Check if the metadata file exists
   local metadata_file="${holohub_app_source}/metadata.json"
   if [ ! -f "$metadata_file" ]; then
     print_error "The metadata file for this application does not exist."
     echo "File: ${metadata_file} not found"
     echo "Did you forget to specify the language?"
     exit 1
   fi

   holohub_app_bin="${holohub_build_dir}/${holohub_app_source#"$SCRIPT_DIR/"}"
   # Check if the build directory exists (for C++ apps)
   if [ ! -d "$holohub_app_bin" ] &&  [ "$2" == "cpp" ]; then
     print_error "The build directory for this application does not exist."
     echo "Did you forget to './run build $1' ?"
     build_desc
     exit 1
   fi

   # Use Python to parse json file
   json=$(${HOLOHUB_PY_EXE} -c 'import json,sys
f=open("'${metadata_file}'")
obj=json.load(f)
project_type = "benchmark" if "benchmark" in obj.keys() else "application"
for k, v in obj[project_type]["run"].items():
  print(str(k)+"=\""+str(v)+"\"")
')

   local json_command=$(echo $json | grep -Po 'command="\K[^"]*')

   # replace <holohub_data_dir> by the data dir
   local command=$(echo "${json_command//<holohub_data_dir>/$holohub_data_dir}")

   # replace <holohub_app_bin> by the binary app directory
   command=$(echo "${command//<holohub_app_bin>/$holohub_app_bin}")

   # replace <holohub_app_source> by the source app directory
   command=$(echo "${command//<holohub_app_source>/$holohub_app_source}")

   if [ $nsys_profile == true ]; then
      nsys_command="nsys"
      if ! command -v nsys &>/dev/null; then
        if [ ! -d "/opt/nvidia/nsys-host" ]; then
          print_error "Nsight Systems CLI command 'nsys' not found. No Nsight installation from the host is also mounted."
          echo "Please use --nsys_location with the 'dev_container launch' command to specify the location of the Nsight Systems installation on the host."
          exit 1
        fi
        nsys_command="/opt/nvidia/nsys-host/bin/nsys"
      fi
      local perf_event_level=$(cat /proc/sys/kernel/perf_event_paranoid)
      if (( ${perf_event_level} > 2 )); then
        print_error "For Nsight Systems profiling the Linux operating system's perf_event_paranoid level must be 2 or less."
        echo "See https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html#linux-requirements for more information."
        exit 1
      fi
      command="${nsys_command} profile --trace=cuda,vulkan,nvtx,osrt ${command}"
   fi

   # default workdir is bin
   local workdir="cd ${holohub_app_bin}"
   local json_workdir=$(echo $json | grep -Po 'workdir="\K[^"]*')
   if [[ $json_workdir == "holohub_app_source" ]]; then
      workdir="cd ${holohub_app_source}"
   elif [[ $json_workdir == "holohub_bin" ]]; then
      workdir="cd ${holohub_build_dir}"
   fi

   local environment="export PYTHONPATH=\${PYTHONPATH}:${holoscan_sdk_install}/../../../python/lib:${holohub_build_dir}/python/lib:${SCRIPT_DIR} && export HOLOHUB_DATA_PATH=${holohub_data_dir} && export HOLOSCAN_INPUT_PATH=${HOLOSCAN_INPUT_PATH:=$holohub_data_dir}"
   local reset_environment="export PYTHONPATH=${PYTHONPATH} && export HOLOHUB_DATA_PATH=\"${HOLOHUB_DATA_PATH}\" && export HOLOSCAN_INPUT_PATH=\"${HOLOSCAN_INPUT_PATH}\""

   if [ ${verbose} ]; then
    echo "Run environment: $environment"
    echo "Run workdir: $workdir"
    echo "Run command: $command"
    echo "Run command args: $extra_args"
   fi

   # Run the command
   run_command $environment
   run_command $workdir
   run_command $command $extra_args
   command_status="$?"
   run_command $reset_environment

   return ${command_status}
}

#===========================================================================================
# Other options

list_desc() {
  echo -e "
  ${BOLD}Display the list of Holohub targets.${NOCOLOR}

  Includes applications, operators, packages.
  "
}

list_cmake_dir_options() {
  cmake_function=$1
  cmakelists=$(find ${SCRIPT_DIR} -name "CMakeLists.txt")
  cmake_add_dir=$(grep --no-filename $cmake_function $cmakelists)
  echo "$cmake_add_dir" | sed -n "s|$cmake_function([[:space:]]*\([^[:space:])]*\).*|\1|p" | sort
}

list_metadata_json_dir() {
  json_list=$(find $@ -name 'metadata.json' | sort -d)
  for json in ${json_list}; do
    local json_path=$(dirname "$json")
    local json_dir=$(basename "$json_path")
    if [[ "${json_dir}" =~ \{\{[^}]+\}\} ]]; then
      # skip templates like {{ foo }}
      continue
    elif [[ ${json_dir} == "cpp" ]] || [[ ${json_dir} == "python" ]]; then
      language="(${json_dir})"
      name=$(basename $(dirname "$json_path"))
    else
      language=""
      name="${json_dir}"
    fi
    echo $name $language
  done
}

list_apps() {
  list_metadata_json_dir ${SCRIPT_DIR}/applications ${SCRIPT_DIR}/benchmarks
}

list_operators() {
  list_metadata_json_dir ${SCRIPT_DIR}/operators
}

list_packages() {
  list_cmake_dir_options add_holohub_package
}

list() {
  echo
  echo -e "${BOLD}== APPLICATIONS =================${NOCOLOR}"
  echo

  list_apps
  echo
  echo -e "${BOLD}== OPERATORS ====================${NOCOLOR}"
  echo
  list_operators
  echo
  echo -e "${BOLD}== PACKAGES =====================${NOCOLOR}"
  echo
  list_packages
  echo
  echo -e "${BOLD}=================================${NOCOLOR}"
  echo
}

# Returns the list of words for autocompletion
autocompletion_list() {
  apps=$(find ${SCRIPT_DIR}/applications -name 'metadata.json')
  for d in ${apps}; do
    local appname=$(dirname $d | grep -Po '^'${SCRIPT_DIR}/applications/'\K[^ ]*')
    filename="${appname%/*}"
    # Deal with subdirectories of applications
    echo "${filename##*/}"
  done
  echo "build launch clear_cache cpp python install_lint_deps lint setup"
}

clear_cache() {
  echo "Clearing cache..."
  run_command rm -rf ${SCRIPT_DIR}/build
  run_command rm -rf ${SCRIPT_DIR}/build-*
  run_command rm -rf ${SCRIPT_DIR}/install
}

clear_cache_desc() {
  echo -e "${BOLD}Remove the build folders${NOCOLOR}"
}

parse_args() {
    local OPTIND
    while getopts 'yh' option;
    do
        case "${option}" in
            y)
                ALWAYS_YES="true"
                ;;
            h)
                print_usage
                exit 1
                ;;
            *)
                ;;
        esac
    done
    shift $((OPTIND-1))

    CMD="$1"
    shift

    ARGS=("$@")
    # Check if the command has `--help`, `-h`, and override the CMD
    local i
    local arg
    extra_args=false
    for i in "${!ARGS[@]}"; do
        arg="${ARGS[i]}"
        # if any argument is --extra_args, then a subsequent "-h/--help" is passed to the application
        if [[ "$arg" == "--extra_args" ]]; then
            extra_args=true
            continue
        fi
        if [[ "$extra_args" == false ]]; then
          if [ "$arg" = "--help" ] || [ "$arg" = "-h" ]; then
              ARGS=("$CMD")
              CMD="help"
              break
          fi
        fi
        if [ "$arg" = "--dryrun" ]; then
            unset_pos=$i
            DO_DRY_RUN="true"  # set to true to print commands to screen without running
        fi
    done
    if [ "${unset_pos}" ]; then
        unset 'ARGS[unset_pos]'
    fi
}

print_usage() {
    set +x
    echo
    echo -e "${BOLD}USAGE: $0 [command] [arguments]...${NOCOLOR}"
    echo
    echo -e "${CYAN}Global Arguments${NOCOLOR}"
    echo "  --help, -h                            : Print help messages for [command]"
    echo "  --dryrun                              : Print commands to screen without running"
    echo
    echo -e "${CYAN}Commands${NOCOLOR}"
    echo "  setup                                 : Install HoloHub main required packages"
    echo "  clear_cache                           : Clear cache folders"
    echo "  list                                  : List all the available build targets"
    echo "  build <package|application|operator>  : Build a specific package, application, or operator"
    echo "  launch <application> <language>       : Run the application"
    echo
}

print_cmd_help_messages() {
    local cmd="$1"
    if [ -n "${cmd}" ]; then
        if type ${cmd}_desc > /dev/null 2>&1; then
            ${cmd}_desc
            exit 0
        else
            echo "Command '${cmd}' doesn't exist!"
            exit 1
        fi
    fi
    print_usage
    return 0
}


main() {
    local ret=0
    parse_args "$@"

    case "$CMD" in
        help|"")
            print_cmd_help_messages "${ARGS[@]}"
            exit 0
            ;;
        *)
            if type ${CMD} > /dev/null 2>&1; then
                "$CMD" "${ARGS[@]}"
            else
                print_usage
                exit 1
            fi
            ;;
    esac
}

main "$@"