diff --git a/.github/workflows/sphinx.yaml b/.github/workflows/sphinx.yaml deleted file mode 100644 index eec1775ad..000000000 --- a/.github/workflows/sphinx.yaml +++ /dev/null @@ -1,53 +0,0 @@ -name: sphinx -on: - release: - types: [released] - -jobs: - build: - name: Build Sphinx Documentation - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Sphinx Build - run: python docs/build.py --no-docker - - - uses: actions/upload-artifact@v3 - with: - name: html-docs - path: docs/_build - - publish: - name: Publish to GitHub pages - runs-on: ubuntu-latest - needs: build - - steps: - - uses: actions/download-artifact@v3 - with: - name: html-docs - path: . - - - name: Setup git repository - run: | - git init . - git config --global user.name "github.com/${{ github.actor }}" - git config --global user.email "${{ github.actor }}@${{ github.sha }}" - git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" - git checkout -B gh-pages - - - name: Git commit, force push gh-pages - run: | - git add . - git commit \ - -m "Generated from GitHub "${{ github.workflow }}" Workflow" \ - && git push --force origin gh-pages \ - || echo "Nothing new to commit" diff --git a/.gitignore b/.gitignore index eb2862755..3995ffa13 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ wiki/out LATEST_CHANGELOG.md __pycache__ .git-commit-template +.vscode/ +.venv/ +docs/source/api/ +docs/source/commands/ \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..8b4c8f8e5 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + +sphinx: + builder: html + configuration: docs/source/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile index 87f9e2860..bc5dc404a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,177 +1,28 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -n +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source BUILDDIR = _build -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Rez.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Rez.qhc" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/Rez" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Rez" - @echo "# devhelp" +.PHONY: help clean livehtml Makefile -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." +livehtml: + sphinx-autobuild "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +clean: + rm -rf _build + rm -rf source/api + rm -rf source/commands diff --git a/docs/NOTES.md b/docs/NOTES.md new file mode 100644 index 000000000..357efc0da --- /dev/null +++ b/docs/NOTES.md @@ -0,0 +1,15 @@ +============ +page section +============ + +Section 1 +========= + +Section 1.1 +----------- + +Section 1.1.1 ++++++++++++++ + +Section 1.1.1.1 +*************** diff --git a/docs/api/modules.rst b/docs/api/modules.rst deleted file mode 100644 index 332d86474..000000000 --- a/docs/api/modules.rst +++ /dev/null @@ -1,8 +0,0 @@ -API Documentation -================= - -.. toctree:: - :maxdepth: 4 - - rez - rezplugins diff --git a/docs/api/rez._sys.rst b/docs/api/rez._sys.rst deleted file mode 100644 index 10cd7dd14..000000000 --- a/docs/api/rez._sys.rst +++ /dev/null @@ -1,22 +0,0 @@ -rez._sys package -================ - -Submodules ----------- - -rez._sys._setup module ----------------------- - -.. automodule:: rez._sys._setup - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rez._sys - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rez.bind.rst b/docs/api/rez.bind.rst deleted file mode 100644 index cf1aec9f6..000000000 --- a/docs/api/rez.bind.rst +++ /dev/null @@ -1,62 +0,0 @@ -rez.bind package -================ - -Submodules ----------- - -rez.bind.arch module --------------------- - -.. automodule:: rez.bind.arch - :members: - :undoc-members: - :show-inheritance: - -rez.bind.cmake module ---------------------- - -.. automodule:: rez.bind.cmake - :members: - :undoc-members: - :show-inheritance: - -rez.bind.hello_world module ---------------------------- - -.. automodule:: rez.bind.hello_world - :members: - :undoc-members: - :show-inheritance: - -rez.bind.os module ------------------- - -.. automodule:: rez.bind.os - :members: - :undoc-members: - :show-inheritance: - -rez.bind.platform module ------------------------- - -.. automodule:: rez.bind.platform - :members: - :undoc-members: - :show-inheritance: - -rez.bind.python module ----------------------- - -.. automodule:: rez.bind.python - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rez.bind - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rez.cli.rst b/docs/api/rez.cli.rst deleted file mode 100644 index 486c8878c..000000000 --- a/docs/api/rez.cli.rst +++ /dev/null @@ -1,134 +0,0 @@ -rez.cli package -=============== - -Submodules ----------- - -rez.cli._bez module -------------------- - -.. automodule:: rez.cli._bez - :members: - :undoc-members: - :show-inheritance: - -rez.cli._main module --------------------- - -.. automodule:: rez.cli._main - :members: - :undoc-members: - :show-inheritance: - -rez.cli._util module --------------------- - -.. automodule:: rez.cli._util - :members: - :undoc-members: - :show-inheritance: - -rez.cli.bind module -------------------- - -.. automodule:: rez.cli.bind - :members: - :undoc-members: - :show-inheritance: - -rez.cli.bootstrap module ------------------------- - -.. automodule:: rez.cli.bootstrap - :members: - :undoc-members: - :show-inheritance: - -rez.cli.build module --------------------- - -.. automodule:: rez.cli.build - :members: - :undoc-members: - :show-inheritance: - -rez.cli.context module ----------------------- - -.. automodule:: rez.cli.context - :members: - :undoc-members: - :show-inheritance: - -rez.cli.env module ------------------- - -.. automodule:: rez.cli.env - :members: - :undoc-members: - :show-inheritance: - -rez.cli.forward module ----------------------- - -.. automodule:: rez.cli.forward - :members: - :undoc-members: - :show-inheritance: - -rez.cli.interpret module ------------------------- - -.. automodule:: rez.cli.interpret - :members: - :undoc-members: - :show-inheritance: - -rez.cli.release module ----------------------- - -.. automodule:: rez.cli.release - :members: - :undoc-members: - :show-inheritance: - -rez.cli.settings module ------------------------ - -.. automodule:: rez.cli.settings - :members: - :undoc-members: - :show-inheritance: - -rez.cli.suite module --------------------- - -.. automodule:: rez.cli.suite - :members: - :undoc-members: - :show-inheritance: - -rez.cli.test module -------------------- - -.. automodule:: rez.cli.test - :members: - :undoc-members: - :show-inheritance: - -rez.cli.tools module --------------------- - -.. automodule:: rez.cli.tools - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rez.cli - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rez.rst b/docs/api/rez.rst deleted file mode 100644 index 533c30461..000000000 --- a/docs/api/rez.rst +++ /dev/null @@ -1,256 +0,0 @@ -Rez -=== - -Subpackages ------------ - -.. toctree:: - - rez._sys - rez.bind - rez.cli - rez.tests - -Submodules ----------- - -rez.bind_utils module ---------------------- - -.. automodule:: rez.bind_utils - :members: - :undoc-members: - :show-inheritance: - -rez.bootstrap module --------------------- - -.. automodule:: rez.bootstrap - :members: - :undoc-members: - :show-inheritance: - -rez.build_process module ------------------------- - -.. automodule:: rez.build_process - :members: - :undoc-members: - :show-inheritance: - -rez.build_system module ------------------------ - -.. automodule:: rez.build_system - :members: - :undoc-members: - :show-inheritance: - -rez.build_utils module ----------------------- - -.. automodule:: rez.build_utils - :members: - :undoc-members: - :show-inheritance: - -rez.dot module --------------- - -.. automodule:: rez.dot - :members: - :undoc-members: - :show-inheritance: - -rez.env module --------------- - -.. automodule:: rez.env - :members: - :undoc-members: - :show-inheritance: - -rez.exceptions module ---------------------- - -.. automodule:: rez.exceptions - :members: - :undoc-members: - :show-inheritance: - -rez.formulae_manager module ---------------------------- - -.. automodule:: rez.formulae_manager - :members: - :undoc-members: - :show-inheritance: - -rez.package_maker module ------------------------- - -.. automodule:: rez.package_maker - :members: - :undoc-members: - :show-inheritance: - -rez.package_maker_ module -------------------------- - -.. automodule:: rez.package_maker_ - :members: - :undoc-members: - :show-inheritance: - -rez.packages module -------------------- - -.. automodule:: rez.packages - :members: - :undoc-members: - :show-inheritance: - -rez.platform_ module --------------------- - -.. automodule:: rez.platform_ - :members: - :undoc-members: - :show-inheritance: - -rez.plugin_managers module --------------------------- - -.. automodule:: rez.plugin_managers - :members: - :undoc-members: - :show-inheritance: - -rez.py_dist module ------------------- - -.. automodule:: rez.py_dist - :members: - :undoc-members: - :show-inheritance: - -rez.release_hook module ------------------------ - -.. automodule:: rez.release_hook - :members: - :undoc-members: - :show-inheritance: - -rez.release_vcs module ----------------------- - -.. automodule:: rez.release_vcs - :members: - :undoc-members: - :show-inheritance: - -rez.resolved_context module ---------------------------- - -.. automodule:: rez.resolved_context - :members: - :undoc-members: - :show-inheritance: - -rez.resolver module -------------------- - -.. automodule:: rez.resolver - :members: - :undoc-members: - :show-inheritance: - -rez.resources module --------------------- - -.. automodule:: rez.resources - :members: - :undoc-members: - :show-inheritance: - -rez.rex module --------------- - -.. automodule:: rez.rex - :members: - :undoc-members: - :show-inheritance: - -rez.rex_bindings module ------------------------ - -.. automodule:: rez.rex_bindings - :members: - :undoc-members: - :show-inheritance: - -rez.settings module -------------------- - -.. automodule:: rez.settings - :members: - :undoc-members: - :show-inheritance: - -rez.shells module ------------------ - -.. automodule:: rez.shells - :members: - :undoc-members: - :show-inheritance: - -rez.sigint module ------------------ - -.. automodule:: rez.sigint - :members: - :undoc-members: - :show-inheritance: - -rez.solver module ------------------ - -.. automodule:: rez.solver - :members: - :undoc-members: - :show-inheritance: - -rez.source_retrieval module ---------------------------- - -.. automodule:: rez.source_retrieval - :members: - :undoc-members: - :show-inheritance: - -rez.system module ------------------ - -.. automodule:: rez.system - :members: - :undoc-members: - :show-inheritance: - -rez.util module ---------------- - -.. automodule:: rez.util - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rez - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rez.tests.rst b/docs/api/rez.tests.rst deleted file mode 100644 index c191f9729..000000000 --- a/docs/api/rez.tests.rst +++ /dev/null @@ -1,78 +0,0 @@ -rez.tests package -================= - -Submodules ----------- - -rez.tests.test_build module ---------------------------- - -.. automodule:: rez.tests.test_build - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_commands module ------------------------------- - -.. automodule:: rez.tests.test_commands - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_context module ------------------------------ - -.. automodule:: rez.tests.test_context - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_formatter module -------------------------------- - -.. automodule:: rez.tests.test_formatter - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_rex module -------------------------- - -.. automodule:: rez.tests.test_rex - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_shells module ----------------------------- - -.. automodule:: rez.tests.test_shells - :members: - :undoc-members: - :show-inheritance: - -rez.tests.test_solver module ----------------------------- - -.. automodule:: rez.tests.test_solver - :members: - :undoc-members: - :show-inheritance: - -rez.tests.util module ---------------------- - -.. automodule:: rez.tests.util - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rez.tests - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.build_system.rst b/docs/api/rezplugins.build_system.rst deleted file mode 100644 index 995ab9ee4..000000000 --- a/docs/api/rezplugins.build_system.rst +++ /dev/null @@ -1,38 +0,0 @@ -rezplugins.build_system package -=============================== - -Submodules ----------- - -rezplugins.build_system.bez module ----------------------------------- - -.. automodule:: rezplugins.build_system.bez - :members: - :undoc-members: - :show-inheritance: - -rezplugins.build_system.cmake module ------------------------------------- - -.. automodule:: rezplugins.build_system.cmake - :members: - :undoc-members: - :show-inheritance: - -rezplugins.build_system.make module ------------------------------------ - -.. automodule:: rezplugins.build_system.make - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rezplugins.build_system - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.release_hook.rst b/docs/api/rezplugins.release_hook.rst deleted file mode 100644 index cfb6bae97..000000000 --- a/docs/api/rezplugins.release_hook.rst +++ /dev/null @@ -1,22 +0,0 @@ -rezplugins.release_hook package -=============================== - -Submodules ----------- - -rezplugins.release_hook.emailer module --------------------------------------- - -.. automodule:: rezplugins.release_hook.emailer - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rezplugins.release_hook - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.release_vcs.rst b/docs/api/rezplugins.release_vcs.rst deleted file mode 100644 index 387e0b5f2..000000000 --- a/docs/api/rezplugins.release_vcs.rst +++ /dev/null @@ -1,38 +0,0 @@ -rezplugins.release_vcs package -============================== - -Submodules ----------- - -rezplugins.release_vcs.git module ---------------------------------- - -.. automodule:: rezplugins.release_vcs.git - :members: - :undoc-members: - :show-inheritance: - -rezplugins.release_vcs.hg module --------------------------------- - -.. automodule:: rezplugins.release_vcs.hg - :members: - :undoc-members: - :show-inheritance: - -rezplugins.release_vcs.svn module ---------------------------------- - -.. automodule:: rezplugins.release_vcs.svn - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rezplugins.release_vcs - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.rst b/docs/api/rezplugins.rst deleted file mode 100644 index 9ed107155..000000000 --- a/docs/api/rezplugins.rst +++ /dev/null @@ -1,21 +0,0 @@ -Rez Plugins -=========== - -Subpackages ------------ - -.. toctree:: - - rezplugins.build_system - rezplugins.release_hook - rezplugins.release_vcs - rezplugins.shell - rezplugins.source_retriever - -Module contents ---------------- - -.. automodule:: rezplugins - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.shell.rst b/docs/api/rezplugins.shell.rst deleted file mode 100644 index afc5ecf4e..000000000 --- a/docs/api/rezplugins.shell.rst +++ /dev/null @@ -1,54 +0,0 @@ -rezplugins.shell package -======================== - -Submodules ----------- - -rezplugins.shell.bash module ----------------------------- - -.. automodule:: rezplugins.shell.bash - :members: - :undoc-members: - :show-inheritance: - -rezplugins.shell.csh module ---------------------------- - -.. automodule:: rezplugins.shell.csh - :members: - :undoc-members: - :show-inheritance: - -rezplugins.shell.sh module --------------------------- - -.. automodule:: rezplugins.shell.sh - :members: - :undoc-members: - :show-inheritance: - -rezplugins.shell.tcsh module ----------------------------- - -.. automodule:: rezplugins.shell.tcsh - :members: - :undoc-members: - :show-inheritance: - -rezplugins.shell.windows module -------------------------------- - -.. automodule:: rezplugins.shell.windows - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rezplugins.shell - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/api/rezplugins.source_retriever.rst b/docs/api/rezplugins.source_retriever.rst deleted file mode 100644 index f570aa200..000000000 --- a/docs/api/rezplugins.source_retriever.rst +++ /dev/null @@ -1,38 +0,0 @@ -rezplugins.source_retriever package -=================================== - -Submodules ----------- - -rezplugins.source_retriever.archive module ------------------------------------------- - -.. automodule:: rezplugins.source_retriever.archive - :members: - :undoc-members: - :show-inheritance: - -rezplugins.source_retriever.git module --------------------------------------- - -.. automodule:: rezplugins.source_retriever.git - :members: - :undoc-members: - :show-inheritance: - -rezplugins.source_retriever.hg module -------------------------------------- - -.. automodule:: rezplugins.source_retriever.hg - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: rezplugins.source_retriever - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/build.py b/docs/build.py deleted file mode 100644 index 650384e23..000000000 --- a/docs/build.py +++ /dev/null @@ -1,173 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - - -import argparse -import errno -import os -import re -import subprocess -import tempfile - - -THIS_FILE = os.path.abspath(__file__) -THIS_DIR = os.path.dirname(THIS_FILE) -REZ_SOURCE_DIR = os.getenv("REZ_SOURCE_DIR", os.path.dirname(THIS_DIR)) -REQUIREMENTS = ['sphinx_rtd_theme', REZ_SOURCE_DIR] -DEST_DIR = os.path.join("docs", "_build") -PIP_PATH_REGEX = re.compile(r"'([^']+)' which is not on PATH.") - - -class CliParser(argparse.ArgumentParser): - """Parser flags, using global variables as defaults.""" - INIT_DEFAULTS = { - "prog": "build", - "description": "Build Sphinx Python API docs", - } - - def __init__(self, **kwargs): - """Setup default arguments and parser description/program name. - - If no parser description/program name are given, default ones will - be assigned. - - Args: - kwargs (dict[str]): - Same key word arguments taken by - ``argparse.ArgumentParser.__init__()`` - """ - for key, value in self.INIT_DEFAULTS.items(): - kwargs.setdefault(key, value) - super(CliParser, self).__init__(**kwargs) - - self.add_argument( - "--no-docker", - action="store_false", - dest="docker", - help="Don't run build processes inside Docker container.", - ) - self.add_argument( - "requirement", - nargs="*", - help="Additional packages to pip install.", - ) - - -def construct_docker_run_args(): - """Create subprocess arguments list for running this script inside docker. - - Returns: - list[str]: Arguments list for ``subprocess.call()``. - """ - docker_args = ["docker", "run", "--interactive", "--rm"] - - if os.sys.stdin.isatty() and os.sys.stdout.isatty(): - docker_args.append("--tty") - - if os.name == "posix": - user_group_ids = os.getuid(), os.getgid() - docker_args += ["--user", ":".join(map(str, user_group_ids))] - - docker_args += [ - "--workdir", REZ_SOURCE_DIR, - "--volume", ":".join([REZ_SOURCE_DIR, REZ_SOURCE_DIR]), - "python:{v.major}.{v.minor}".format(v=os.sys.version_info), - "python", THIS_FILE, "--no-docker" - ] - - return docker_args - - -def print_call(cmdline_args, *print_args, **print_kwargs): - """Print command line call for given arguments. - - - Args: - cmdline_args (list): Command line arguments to print for. - print_args (dict): Additional arguments for print function. - print_kwargs (dict): Keyword arguments for print function. - """ - width = os.getenv('COLUMNS', 80) - out_file = print_kwargs.setdefault('file', os.sys.stdout) - message = '{:=^{width}}{nl}{}{nl:=<{width}}'.format( - " Calling ", - subprocess.list2cmdline(cmdline_args), - nl=os.linesep, - width=width - ) - print(message, *print_args, **print_kwargs) - out_file.flush() - - -def path_with_pip_scripts(install_stderr, path_env=None): - """Create new PATH variable with missing pip scripts paths added to it. - - Args: - install_stderr (str): stderr output from pip install command. - path_env (str): Custom PATH env value to start off with. - - Returns: - str: New PATH variable value. - """ - if path_env is None: - path_env = os.getenv('PATH', '') - paths = path_env.split(os.pathsep) - - for match in PIP_PATH_REGEX.finditer(install_stderr): - script_path = match.group(1) - if script_path not in paths: - paths.append(script_path) - - return os.pathsep.join(paths) - - -def _cli(): - """Main routine for when called from command line.""" - args = CliParser().parse_args() - - if args.docker: - docker_args = construct_docker_run_args() + args.requirement - print_call(docker_args) - os.sys.exit(subprocess.call(docker_args)) - else: - docs_env = os.environ.copy() - - # Fake user's $HOME in container to fix permission issues - if os.name == "posix" and os.path.expanduser("~") == "/": - docs_env['HOME'] = tempfile.mkdtemp() - - # Run pip install for required docs building packages - pip_args = ['pip', 'install', '--user'] - pip_args += REQUIREMENTS + args.requirement - with tempfile.TemporaryFile() as stderr_file: - subprocess.check_call(pip_args, env=docs_env, stderr=stderr_file) - stderr_file.seek(0) - stderr = str(stderr_file.read()) - docs_env['PATH'] = path_with_pip_scripts(stderr) - - # Run sphinx-build docs, falling back to use sphinx-build.exe - sphinx_build = 'sphinx-build' - build_args = ['docs', DEST_DIR] - sphinx_build_args = [sphinx_build] + build_args - try: - print_call(sphinx_build_args) - os.sys.exit(subprocess.call(sphinx_build_args, env=docs_env)) - except OSError as error: - if error.errno == errno.ENOENT: - # Windows Py2.7 needs full .exe path, see GitHub workflows run: - # https://github.com/wwfxuk/rez/runs/380399547 - latest_path = docs_env['PATH'].split(os.pathsep)[-1] - sphinx_build = os.path.join(latest_path, sphinx_build + '.exe') - - sphinx_build_args = [sphinx_build] + build_args - print_call(sphinx_build_args) - os.sys.exit(subprocess.call(sphinx_build_args, env=docs_env)) - else: - raise - - -if __name__ == "__main__": - _cli() diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 17a0bb1c4..000000000 --- a/docs/conf.py +++ /dev/null @@ -1,274 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Rez documentation build configuration file, created by -# sphinx-quickstart on Mon Jun 30 11:14:36 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - -sys.path.insert(0, os.path.abspath('../src')) - -import rez - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.3' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon', - 'sphinx.ext.githubpages', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Rez' -copyright = u'2014, Allan Johns' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '.'.join(rez.__version__.split('.')[:2]) -# The full version, including alpha/beta/rc tags. -release = rez.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = None - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'Rezdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'Rez.tex', u'Rez Documentation', - u'Allan Johns', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'rez', u'Rez Documentation', - [u'Allan Johns'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Rez', u'Rez Documentation', - u'Allan Johns', 'Rez', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 0fd843a3f..000000000 --- a/docs/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -REZ API Reference -================= - -.. toctree:: - :maxdepth: 2 - :numbered: - - api/modules - one-liners - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat index 21d8ea40c..826f02ab2 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,53 +1,16 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - +set SOURCEDIR=source +set BUILDDIR=build -%SPHINXBUILD% 2> nul +%SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx @@ -56,187 +19,17 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ + echo.https://www.sphinx-doc.org/ exit /b 1 ) -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Rez.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Rez.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) +if "%1" == "" goto help -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -n +goto end -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end +popd diff --git a/docs/one-liners.rst b/docs/one-liners.rst deleted file mode 100644 index 8f4c8edc4..000000000 --- a/docs/one-liners.rst +++ /dev/null @@ -1,71 +0,0 @@ -One-Liners -========== - -A list of useful one-liners for rez-config and related tools - -Display info about the package foo: - -:: - - rez-info foo - -List the packages that foo depends on: - -:: - - rez-config --print-packages foo - -Jump into an environment containing foo-5(.x.x.x...): - -:: - - rez-env foo-5 - -Run a command inside a configured shell - -:: - - rez-run foo-5 bah-1.2 -- my-command - -Show the resolve dot-graph for a given shell: - -:: - - rez-run foo-5 bah-1.2 fee -- rez-context-image - -Display a dot-graph showing the first failed attempt of the given configuration PKGS: - -:: - - rez-config --max-fails=0 --dot-file=/tmp/dot.jpg PKGS ; firefox /tmp/dot.jpg - -Show a dot-graph of all the packages dependent on foo: - -:: - - rez-depends show-dot foo - -List every package in the system, and the description of each - -:: - - rez-config-list --desc - -Show the resolve dot-graph for a given shell, but just show that part of the graph that -contains packages dependent (directly or indirectly) on fee: - -:: - - rez-run foo-5 bah-1.2 fee -- rez-context-image --package=fee - -Run a command inside a toolchain wrapper: -:: - - rez-run mytoolchain -- sometool -- some-command - -Jump into a toolchain, and then into a wrapper's env: - -:: - - rez-run mytoolchain - sometool ---i diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..470e82a19 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx +furo +sphinx-autobuild diff --git a/docs/rez_sphinxext.py b/docs/rez_sphinxext.py new file mode 100644 index 000000000..f5dce23ec --- /dev/null +++ b/docs/rez_sphinxext.py @@ -0,0 +1,349 @@ +import os +import re +import argparse + +import rez.cli._main +import rez.cli._util +import rez.rezconfig +import docutils.nodes +import sphinx.util.nodes +import sphinx.application +import sphinx.environment +import sphinx.util.logging +import sphinx.util.docutils +import docutils.statemachine + +_LOG = sphinx.util.logging.getLogger(f"ext.{__name__.split('.')[-1]}") + + +def convert_rez_config_to_rst() -> list[str]: + with open(rez.rezconfig.__file__) as fd: + txt = fd.read() + + lines = txt.split('\n') + + start = None + end = None + for i, line in enumerate(lines): + if "__DOC_START__" in line: + start = i + elif "__DOC_END__" in line: + end = i + + lines = lines[start:end + 1] + assign_regex = re.compile("^([a-z0-9_]+) =") + settings = {} + + section_header = re.compile(r"^\#{10,}") + + section_title = None + section_description = None + end_of_section = 0 + + # parse out settings sections, settings and their comment + for i, line in enumerate(lines): + + if section_header.match(line) and i != end_of_section: + section_title = lines[i + 1].split('#', 1)[-1].strip() + section_description = '' + description_linenumber = i + 2 + end_of_section = description_linenumber + + while not section_header.match(lines[description_linenumber]): + section_description += lines[description_linenumber].split('#', 1)[-1].strip() + '\n' + description_linenumber += 1 + end_of_section = description_linenumber + + m = assign_regex.match(line) + if not m: + continue + + start_defn = i + end_defn = i + while lines[end_defn].strip() and not lines[end_defn].startswith('#'): + end_defn += 1 + + value_lines = lines[start_defn:end_defn] + value_lines[0] = value_lines[0].split("=")[-1].strip() + + end_comment = i + while not lines[end_comment].startswith('#'): + end_comment -= 1 + + start_comment = end_comment + while lines[start_comment].startswith('#'): + start_comment -= 1 + start_comment += 1 + + comments = lines[start_comment:end_comment + 1] + comment_lines = [x[2:] for x in comments] # drop leading '# ' + + varname = m.groups()[0] + if section_title in settings: + settings[section_title]['settings'][varname] = (value_lines, comment_lines) + else: + settings[section_title] = { + 'desc': section_description, + 'settings': {varname: (value_lines, comment_lines)} + } + + # generate rst text + # rst = ['.. currentmodule:: config'] + rst = [] + + for section in settings: + rst.append('') + rst.append(section) + rst.append("-" * len(section)) + rst.append('') + + # This seems benign (storing each line individually), but it's actually extremelly + # important. The docutils ViewList class absolutely requires to have only one line per + # entry. If we don't do that, the docutils parser won't parse the sublines, + # and we'll get "garbage" (ie unformatted lines) in the output. + for line in settings[section]['desc'].strip().split('\n'): + rst.append(line) + rst.append('') + + for varname, (value_lines, comment_lines) in sorted(settings[section]['settings'].items()): + rst.append(".. py:data:: {0}".format(varname)) + if len(value_lines) == 1: + rst.append(" :value: {0}".format(value_lines[0])) + else: + rst.append('') + rst.append(' Default:') + rst.append('') + rst.append(' .. code-block:: python') + rst.append('') + for line in value_lines: + rst.append(f' {line}') + + rst.append('') + for line in comment_lines: + rst.append(f' {line}') + rst.append('') + + envvar = f'REZ_{varname.upper()}' + rst.append(f' .. envvar:: {envvar}') + rst.append('') + rst.append(f' The ``{envvar}`` environment variable can also be used to configure this.') + rst.append('') + + return rst + + +# Inspired by https://github.com/pypa/pip/blob/4a79e65cb6aac84505ad92d272a29f0c3c1aedce/docs/pip_sphinxext.py +# https://stackoverflow.com/a/44084890 +class RezConfigDirective(sphinx.util.docutils.SphinxDirective): + """ + Special rez-config directive. This is quite similar to "autodoc" in some ways. + """ + required_arguments = 0 + optional_arguments = 0 + + def run(self) -> list[docutils.nodes.Node]: + # Create the node. + node = docutils.nodes.section() + node.document = self.state.document + + rst = docutils.statemachine.ViewList() + + # Get the configuration settings as reStructuredText text. + configLines = convert_rez_config_to_rst() + + # Add rezconfig as a dependency to the current document. The document + # will be rebuilt if rezconfig changes. + self.env.note_dependency(rez.rezconfig.__file__) + self.env.note_dependency(__file__) + + path, lineNumber = self.get_source_info() + + # Add each line to the view list. + for index, line in enumerate(configLines): + # Note to future people that will look at this. + # "line" has to be a single line! It can't be a line like "this\nthat". + rst.append(line, path, lineNumber+index) + + # Finally, convert the rst into the appropriate docutils/sphinx nodes. + sphinx.util.nodes.nested_parse_with_titles(self.state, rst, node) + + # Return the generated nodes. + return node.children + + +class RezAutoArgparseDirective(sphinx.util.docutils.SphinxDirective): + """ + Special rez-autoargparse directive. This is quite similar to "autosummary" in some ways. + """ + required_arguments = 0 + optional_arguments = 0 + + def run(self) -> list[docutils.nodes.Node]: + # Create the node. + node = docutils.nodes.section() + node.document = self.state.document + + rst = docutils.statemachine.ViewList() + + # Add rezconfig as a dependency to the current document. The document + # will be rebuilt if rezconfig changes. + self.env.note_dependency(rez.cli._util.__file__) + self.env.note_dependency(__file__) + + path, lineNumber = self.get_source_info() + + toc = """.. toctree:: + :maxdepth: 1 + :hidden: + + commands/rez + +""" + listRst = "* :doc:`commands/rez`\n" + + for subcommand, config in rez.cli._util.subcommands.items(): + if config.get('hidden'): + continue + + toc += f" commands/rez-{subcommand}\n" + listRst += f"* :doc:`commands/rez-{subcommand}`\n" + + # Add each line to the view list. + for index, line in enumerate((toc + "\n" + listRst).split("\n")): + # Note to future people that will look at this. + # "line" has to be a single line! It can't be a line like "this\nthat". + rst.append(line, path, lineNumber+index) + + # Finally, convert the rst into the appropriate docutils/sphinx nodes. + sphinx.util.nodes.nested_parse_with_titles(self.state, rst, node) + + # Return the generated nodes. + return node.children + + +# Inspired by autosummary (https://github.com/sphinx-doc/sphinx/blob/fcc38997f1d9b728bb4ffc64fc362c7763a4ee25/sphinx/ext/autosummary/__init__.py#L782) +# and https://github.com/ashb/sphinx-argparse/blob/b2f42564fb03ede94e94c149a425e398764158ca/sphinxarg/parser.py#L49 +def write_cli_documents(app: sphinx.application.Sphinx) -> None: + """ + Write the CLI pages into the "commands" folder. + """ + _LOG.info("[rez-autoargparse] generating command line documents") + + _LOG.info("[rez-autoargparse] seting up the parser") + main_parser = rez.cli._main.setup_parser() + main_parser._setup_all_subparsers() + + parsers = [main_parser] + for action in main_parser._actions: + if isinstance(action, rez.cli._util.LazySubParsersAction): + parsers += action.choices.values() + + for parser in sorted(parsers, key=lambda x: x.prog): + full_cmd = parser.prog.replace(' ', '-') + + # Title + document = [f".. _{full_cmd}:"] + document.append("") + document.append(f"{'='*len(parser.prog)}") + document.append(f"{full_cmd}") + document.append(f"{'='*len(parser.prog)}") + document.append("") + + document.append(f".. program:: {full_cmd}") + document.append("") + document.append("Usage") + document.append("=====") + document.append("") + document.append(".. code-block:: text") + document.append("") + for line in parser.format_usage()[7:].split("\n"): + document.append(f" {line}") + document.append("") + + if parser.description == argparse.SUPPRESS: + continue + + document.append("description") + document.append("===========") + document.extend(parser.description.split("\n")) + + document.append("") + document.append("Options") + document.append("=======") + document.append("") + + for action in parser._action_groups[1]._group_actions: + if isinstance(action, argparse._HelpAction): + continue + + # Quote default values for string/None types + default = action.default + if action.default not in ['', None, True, False] and action.type in [None, str] and isinstance(action.default, str): + default = f'"{default}"' + + # fill in any formatters, like %(default)s + format_dict = dict(vars(action), prog=parser.prog, default=default) + format_dict['default'] = default + help_str = action.help or '' # Ensure we don't print None + try: + help_str = help_str % format_dict + except Exception: + pass + + if help_str == argparse.SUPPRESS: + continue + + # Avoid Sphinx warnings. + help_str = help_str.replace("*", "\\*") + # Replace everything that looks like an argument with an option directive. + help_str = re.sub(r"(?`", help_str) + help_str = help_str.replace("--", "\\--") + + # Options have the option_strings set, positional arguments don't + name = action.option_strings + if name == []: + if action.metavar is None: + name = [action.dest] + else: + name = [action.metavar] + + # Skip lines for subcommands + if name == [argparse.SUPPRESS]: + continue + + metavar = f"<{action.metavar}>" if action.metavar else "" + document.append(f".. option:: {', '.join(name)} {metavar.lower()}") + document.append("") + document.append(f" {help_str}") + if action.choices: + document.append("") + document.append(f" Choices: {', '.join(action.choices)}") + document.append("") + + document = "\n".join(document) + + dest = os.path.join(app.srcdir, "commands", f"{full_cmd}.rst") + os.makedirs(os.path.dirname(dest), exist_ok=True) + + if os.path.exists(dest): + with open(dest, "r") as fd: + if fd.read() == document: + # Documents are the same, skip writing to avoid + # invalidating Sphinx's cache. + continue + + with open(dest, "w") as fd: + fd.write(document) + + +def setup(app: sphinx.application.Sphinx) -> dict[str, bool | str]: + app.setup_extension('sphinx.ext.autodoc') + app.add_directive('rez-config', RezConfigDirective) + + app.connect('builder-inited', write_cli_documents) + app.add_directive('rez-autoargparse', RezAutoArgparseDirective) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css new file mode 100644 index 000000000..f43493b13 --- /dev/null +++ b/docs/source/_static/css/custom.css @@ -0,0 +1,11 @@ +/* Enable dark-mode, if requested. */ +body[data-theme=dark] .rez-diagram { + filter: invert(100%) hue-rotate(180deg); +} + +/* Enable dark mode, unless explicitly told to avoid. */ +@media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) .rez-diagram { + filter: invert(100%) hue-rotate(180deg); + } +} \ No newline at end of file diff --git a/docs/source/_static/other_pkg_mgr.png b/docs/source/_static/other_pkg_mgr.png new file mode 100644 index 000000000..f3891daaf Binary files /dev/null and b/docs/source/_static/other_pkg_mgr.png differ diff --git a/docs/source/_static/pkg_path_anatomy.png b/docs/source/_static/pkg_path_anatomy.png new file mode 100644 index 000000000..d7b660495 Binary files /dev/null and b/docs/source/_static/pkg_path_anatomy.png differ diff --git a/docs/source/_static/rez-horizontal-black.svg b/docs/source/_static/rez-horizontal-black.svg new file mode 100644 index 000000000..c542e7498 --- /dev/null +++ b/docs/source/_static/rez-horizontal-black.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/source/_static/rez-horizontal-white.svg b/docs/source/_static/rez-horizontal-white.svg new file mode 100644 index 000000000..a3c06f3de --- /dev/null +++ b/docs/source/_static/rez-horizontal-white.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/source/_static/rez_deps_simple_eg.png b/docs/source/_static/rez_deps_simple_eg.png new file mode 100644 index 000000000..59e2ba485 Binary files /dev/null and b/docs/source/_static/rez_deps_simple_eg.png differ diff --git a/docs/source/_static/rez_env.png b/docs/source/_static/rez_env.png new file mode 100644 index 000000000..ef6e19c0d Binary files /dev/null and b/docs/source/_static/rez_env.png differ diff --git a/docs/source/_static/rez_pkg_mgr.png b/docs/source/_static/rez_pkg_mgr.png new file mode 100644 index 000000000..87e1b162d Binary files /dev/null and b/docs/source/_static/rez_pkg_mgr.png differ diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst new file mode 100644 index 000000000..7792d1f71 --- /dev/null +++ b/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,18 @@ +{{ fullname }} +{{ underline }} + +.. automodule:: {{ fullname }} + :members: + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 000000000..ad4c76fca --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,47 @@ +========== +Python API +========== + +.. autosummary:: + :toctree: api + :recursive: + + rez.build_process + rez.build_system + rez.bundle_context + rez.command + rez.config + rez.developer_package + rez.exceptions + rez.package_cache + rez.package_copy + rez.package_filter + rez.package_help + rez.package_maker + rez.package_move + rez.package_order + rez.package_py_utils + rez.package_remove + rez.package_repository + rez.package_resources + rez.package_search + rez.package_serialise + rez.package_test + rez.packages + rez.plugin_managers + rez.release_hook + rez.release_vcs + rez.resolved_context + rez.resolver + rez.rex_bindings + rez.rex + rez.serialise + rez.shells + rez.solver + rez.status + rez.suite + rez.system + rez.util + rez.utils + rez.vendor.version + rez.wrapper diff --git a/docs/source/basic_concepts.rst b/docs/source/basic_concepts.rst new file mode 100644 index 000000000..34d2d01fd --- /dev/null +++ b/docs/source/basic_concepts.rst @@ -0,0 +1,435 @@ +============== +Basic concepts +============== + +Rez manages packages. You request a list of packages from rez, and it resolves this request, if +possible. If the resolution is not possible, the system supplies you with the relevant information +to determine why this is so. You typically want to resolve a list of packages because you want to +create an environment in which you can use them in combination, without conflicts occurring. A +conflict occurs when there is a request for two or more different versions of the same package. + +Rez lets you describe the environment you want in a natural way. For example, you can say: +"I want an environment with...": + +* the latest version of houdini +* maya-2009.1 +* the latest rv and the latest maya and houdini-11.something +* rv-3.something or greater +* the latest houdini which works with boost-1.37.0 +* PyQt-2.2 or greater, but less than PyQt-4.5.3 + +In many examples in this documentation we will use the +:ref:`rez-env` command line tool. This tool takes a list of package +requests and creates the resulting configured environment. It places you in a subshell. Simply +exit the shell to return to a non-configured environment. + +.. _versions-concept: + +Versions +======== + +Rez version numbers are alphanumeric (any combination of numbers, letters and +underscores). A version number is a set of *tokens*, separated by either dot or dash. For example, +here is a list of valid package version numbers: + +* ``1`` +* ``1.0.0`` +* ``3.2.build_13`` +* ``4.rc1`` +* ``10a-5`` + +Version number tokens follow a strict ordering schema and are case sensitive. Underscore is the +smallest character, followed by letters (a-z and A-Z), followed by numbers. The ordering rules are +like so: + +* Underscore before everything else; +* Letters alphabetical, and before numbers; +* Lowercase letters before uppercase; +* Zero-padded numbers before equivalent non-padded (or less padded) number (``01`` is smaller than ``1``); +* If a token contains a combination of numbers and letters, it is internally split into groups + containing only numbers or only letters, and the resulting list is compared using the same rules + as above. + +The following table shows some example version token comparisons: + +============= ============ +smaller token larger token +============= ============ +0 1 +a b +a A +a 3 +_5 2 +ham hamster +alpha beta +alpha bob +02 2 +002 02 +13 043 +3 3a +beta3 3beta +============= ============ + +Versions are compared by their token lists. The token delimiter (usually dot, but can also be dash) +is ignored for comparison purposes, thus the versions ``1.0.0`` and ``1-0.0`` are equivalent. If two +versions share the same token list prefix, the longer version is greater, thus ``1.0.0`` is a higher +version than ``1.0``. + +.. note:: + No special importance is given to specific characters or letters in Rez version numbers. + The terms ``alpha`` and ``beta`` for example have no special meaning. Similarly, the number of tokens in + a version number doesn't matter, you can have as many as you like. While you are encouraged to use + semantic versioning (see ``_), it is not enforced. + +.. _packages-concept: + +Packages +======== + +A *package* is a versioned piece of software, that may have dependencies on other packages. Packages +are self-contained. They have a single package definition file (typically ``package.py``), which +describes everything we need to know about the package in order to use it. Rez manages any kind of +package, whether it be a python package, compiled package, or simply build code or configuration +data. + +Here is an example package definition file (see the :doc:`package definition guide ` for further details +of each attribute): + +.. code-block:: python + + name = "foo" + + version = "1.0.0" + + description = "Something that does foo-like things." + + requires = [ + "python-2.6", + "utils-1.1+<2" + ] + + tools = [ + "fooify" + ] + + def commands(): + env.PYTHONPATH.append("{root}/python") + env.PATH.append("{root}/bin") + +The :attr:`requires` section defines the requirements of the package. The :func:`commands` section describes +what happens when this package is added to an environment. Here, the ``bin`` directory in the package +installation is appended to ``PATH``, and similarly the ``python`` subdirectory is appended to +``PYTHONPATH``. + +.. _package-repositories-concept: + +Package Repositories +==================== + +Packages are installed into package repositories. + +.. caution:: + The following is an implementation of the filesystem repository plugin. + +A package repository is a directory on disk, with +packages and their versions laid out in a known structure underneath. Going on with our (foo, bah, +eek) example, here is how the package repository might look: + +.. code-block:: text + + /packages/inhouse/foo/1.1 + /1.2 + /1.3 + /packages/inhouse/bah/2 + /3 + /4 + /packages/inhouse/eek/2.5 + /2.6 + /2.7 + + # more detailed example of foo-1.1 + /packages/inhouse/foo/1.1/package.py + /python/ + /bin/ + +Here we have a package repository under the directory ``/packages/inhouse``. The actual package content +(files, executables etc) is installed into each leaf-node version directory, as shown for ``foo-1.1``. +The package definition file, in this case ``package.py``, is always stored at the root of the package, +right under the version directory for that package. + +Rez only requires that the package's definition file is at the root of the package installation. The +layout of the rest of the package, for example the ``python`` and ``bin`` directories, is completely +up to the package's own build to determine. You should expect to see a package's ``commands`` section +match up with its installation though. For example, notice how the path for foo's python files and +binaries match what its package commands specified from earlier. ``{root}/python`` and ``{root}/bin`` +will expand to these paths respectively. + +.. _package-search-path-concept: + +Package Search Path +=================== + +Rez finds packages using a search path in much the same way that python finds python modules using +``PYTHONPATH``. You can find out what the search path is, using the rez command line tool :ref:`rez-config` +(which you can also use to find any other rez setting): + +.. code-block:: text + + ]$ rez-config packages_path + - /home/ajohns/packages + - /packages/inhouse + - /packages/vendor + +If the same package appears in two or more repositories on the search path, the earlier package is +used in preference. This happens at the version level. For example an earlier package ``foo-1.0.0`` +will hide a later package ``foo-1.0.0``, but not ``foo-1.2.0``. + +The example search path shown is a typical setting. There are some central repositories later in the +search path, where packages are released to so everyone can use them. But there is also a local +package path at the front of the search path. This is where packages go that are being locally +developed by a user. Having this at the start of the search-path allows developers to resolve +environments that pull in test packages in preference to released ones, so they can test a package +before releasing it for general use. + +You can change the packages search path in several ways. A common way is to set the :envvar:`REZ_PACKAGES_PATH` +environment variable. + +.. tip:: + See :doc:`configuring_rez` for more configuration options. + +.. _package-commands-concept: + +Package Commands +================ + +The :func:`commands` section of the package definition determines how the environment is configured in +order to use it. It is a python function, but note that if any imports are used, they must appear +within the body of this function. + +Consider this commands example: + +.. code-block:: python + + def commands(): + env.PYTHONPATH.append("{root}/python") + env.PATH.append("{root}/bin") + +This is a typical example, where a package adds its source path to ``PYTHONPATH``, and its tools to +``PATH``. See :doc:`here ` for details on what can be done within the :func:`commands` section, +as well as details on what order package commands are executed in. + +.. _package-requests-concept: + +Package Requests +================ + +A *package request* is a string with a special syntax which matches a number of possible package +versions. You use package requests in the requires section of a package definition file, and also +when creating your own configured environment directly using tools such as :ref:`rez-env`. + +For example, here is a request (using the :ref:`rez-env` tool) to create an environment containing +*python* version 2.6 or greater, and *my_py_utils* version 5.4 or greater, but less than 6: + +.. code-block:: text + + ]$ rez-env 'python-2.6+' 'my_py_utils-5.4+<6' + +Here are some example package requests: + +=============== =================================== ====================================== +Package request Description Example versions within request +=============== =================================== ====================================== +foo Any version of foo. foo-1, foo-0.4, foo-5.0, foo-2.0.alpha +foo-1 Any version of foo-1[.x.x...x]. foo-1, foo-1.0, foo-1.2.3 +foo-1+ foo-1 or greater. foo-1, foo-1.0, foo-1.2.3, foo-7.0.0 +foo-1.2+<2 foo-1.2 or greater, but less than 2 foo-1.2.0, foo-1.6.4, foo-1.99 +foo<2 Any version of foo less than 2 foo-1, foo-1.0.4 +foo==2.0.0 Only version 2.0.0 exactly foo-2.0.0 +foo-1.3\|5+ OR'd requests foo-1.3.0, foo-6.0.0 +=============== =================================== ====================================== + +.. _conflict-operator-concept: + +The Conflict Operator +--------------------- + +The ``!`` operator is called the *conflict* operator, and is used to define an incompatibility +between packages, or to specify that you do *not* want a package version present. For example, +consider the command: + +.. code-block:: text + + ]$ rez-env maya_utils '!maya-2015.6' + +This specifies that you require any version of ``maya_utils``, but that any version of ``maya`` within +2015.6 (and this includes 2015.6.1 and so on) is not acceptable. + +.. _weak-references-concept: + +Weak References +--------------- + +The ``~`` operator is called the *weak reference* operator. It forces a package version to be within +the specified range if present, but does not actually require the package. For example, consider +the command: + +.. code-block:: text + + ]$ rez-env foo '~nuke-9.rc2' + +This request may or may not pull in the ``nuke`` package, depending on the requirements of ``foo``. +However, if nuke *is* present, it must be within the version ``9.rc2``. + +Weak references are useful in certain cases. For example, applications such as *nuke* and *maya* +sometimes ship with their own version of *python*. Their rez packages don't have a requirement on +*python* (they have their own embedded version already). However often other python libraries are +used both inside and outside of these applications, and those packages *do* have a python +requirement. So, to make sure that they're using a compatible python version when used within the +app, the app may define a *weak package reference* to their relevant python version, like so: + +.. code-block:: python + + # in maya's package.py + requires = [ + "~python-2.7.3" + ] + +This example ensures that any package that uses python, will use the version compatible with maya +when maya is present in the environment. + +.. _implicit-packages-concept: + +Implicit Packages +================= + +The *implicit packages* are a list of package requests that are automatically added to every rez +request (for example, when you use :ref:`rez-env`). They are set by the configuration setting +:data:`implicit_packages`. The default setting looks like so: + +.. todo:: document implicit_packages and make it referenceable + +.. code-block:: python + + implicit_packages = [ + "~platform=={system.platform}", + "~arch=={system.arch}", + "~os=={system.os}", + ] + +Rez models the current system (the platform, architecture and operating systems) as packages +themselves. The default implicits are a set of *weak requirements* on each of ``platform``, ``arch`` and +``os``. This ensures that if any platform-dependent package is requested, the platform, architecture +and/or operating system it depends on, matches the current system. + +The list of implicits that were used in a request are printed by :ref:`rez-env` when you enter the newly +configured subshell, and are also printed by the :ref:`rez-context` tool. + +.. _dependency-resolving: + +Dependency Resolving +==================== + +Rez contains a solving algorithm that takes a *request* (a list of package requests) and produces +a *resolve* (a final list of packages that satisfy the request). The algorithm avoids version +conflicts (two or more different versions of the same package at once). + +When you submit a request to rez, it finds a solution for that request that aims to give you the +latest possible version of each package. If this is not possible, it will give you the next latest +version, and so on. + +Consider the following example (the arrows indicate dependencies): + +.. image:: _static/rez_deps_simple_eg.png + :align: center + :class: rez-diagram + +Here we have three packages, ``foo``, ``bah`` and ``eek``, where both foo and bah have dependencies on +eek. For example, package ``bah-4`` might have a package definition file that looks something like +this (some entries skipped for succinctness): + +.. code-block:: python + + name = "bah" + + version = "4" + + requires = [ + "eek-2.6" + ] + +A request for ``foo-1.3`` is going to result in the resolve (``foo-1.3``, ``eek-2.7``). A request for +``foo`` will give the same result. We are asking for "any version of foo", but rez will prefer the +latest. However, if we request (``foo``, ``bah``), we are not going to get the latest of both because they +depend on different versions of eek, and that would cause a version conflict. Instead, our resolve +is going to be (``foo-1.2``, ``bah-4``, ``eek-2.6``). Rez has given you the latest possible versions of +packages, that do not cause a conflict. + +Sometimes your request is impossible to fulfill. For example, the request (``foo-1.3``, ``bah-4``) is +not possible. In this case, the resolve will fail, and rez will inform you of the conflict. + +Resolving An Environment +======================== + +A user can create a resolved environment using the command line tool :ref:`rez-env` (also via the API - +practically everything in rez can be done in python). When you create the environment, the current +environment is not changed. You are placed into a sub-shell instead. Here is an example of using +rez-env, assuming that the package repository is from our earlier (foo, bah, eek) example: + +.. code-block:: text + + ]$ rez-env foo bah + + You are now in a rez-configured environment. + + resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, + using Rez v2.0.rc1.10 + + requested packages: + foo + bah + + resolved packages: + eek-2.6 /packages/inhouse/eek/2.6 + foo-1.2 /packages/inhouse/foo/1.2 + bah-4 /packages/inhouse/bah/4 + + > ]$ â–ˆ + +The output of rez-env shows the original request, along with the matching resolve. It's the resolve +that tells you what actual package versions are present in the newly resolved environment. Notice +the ``>`` character in the prompt. This is a visual cue telling you that you have been placed +into a rez-resolved environment. + +Putting It All Together +----------------------- + +Let's go through what happens when an environment is resolved, using a new (and slightly more +realistic) example. +Let us assume that the following packages are available: + +* ``maya-2014.sp2``; +* ``nuke-8.0v3``; +* 3 versions of a maya plugin ``mplugin``; +* 2 versions of a nuke plugin ``nplugin``; +* 3 versions of a common base library ``lib``. + +The following diagram shows what happens when the command ``rez-env mplugin-1.3.0`` is run: + +.. image:: _static/rez_env.png + :align: center + :class: rez-diagram + +The diagram shows the following operations occurring: + +* Rez takes the user's request, and runs it through the dependency solver. The solver reads packages + from the package repositories in order to complete the solve; +* This results in a list of resolved packages. These are the packages that are used in the + configured environment; +* The commands from each package are concatenated together; +* This master list of commands is then translated into the target shell language (in this example + that is ``bash``); +* A sub-shell is created and the translated command code is sourced within this environment, + creating the final configured environment. + +The order of package command execution depends on package dependencies, and the order that packages +were requested in. See :ref:`here ` for more details. diff --git a/docs/source/building_packages.rst b/docs/source/building_packages.rst new file mode 100644 index 000000000..c72c76d13 --- /dev/null +++ b/docs/source/building_packages.rst @@ -0,0 +1,293 @@ +================= +Building packages +================= + +Rez packages can be built and locally installed using the :ref:`rez-build` tool. This +tool performs the following actions: + +* Iterates over a package's :doc:`variants ` +* Constructs the build environment +* Runs the build system within this environment + +Each build occurs within a *build path* which is typically either a *build* +subdirectory, or a variant-specific subdirectory under *build*. For example, a +package with two python-based variants might look like this: + +.. code-block:: text + + +- package.py + +- CMakeLists.txt (or other build file) + +-build + +-python-2.6 # build dir for python-2.6 variant + +-python-2.7 # build dir for python-2.6 variant + +The current working directory is set to the *build path* during a build. + +The Build Environment +===================== + +The build environment is a rez resolved environment. Its requirement list is +constructed like so: + +* First, the package's :attr:`requires` list is used; +* Then, the package's :attr:`build_requires` is + appended. This is transitive, meaning that the :attr:`build_requires` of all other packages in the + environment are also used; +* Then, the package's :attr:`private_build_requires` + is appended (unlike :attr:`build_requires`, it is not transitive). +* Finally, if the package has variants, the current variant's requirements are + appended. + +A standard list of environment variables is also set. You can see the full list :ref:`here `. + +The build system is then invoked within this environment, for each variant. + +Build Time Dependencies +======================= + +Sometimes it is desirable for a package to depend on another package only for the purposes +of building its code, or perhaps generating documentation. Let's use documentation as an +example: a C++ project may need to builds its docs using doxygen, but once the docs are +generated, doxygen is no longer needed. + +This is achieved by listing build-time dependencies under a +:attr:`build_requires` or :attr:`private_build_requires` +section in the ``package.py``. The requirements in :attr:`private_build_requires` are only used +from the package being built. Requirements from :attr:`build_requires` however are transitive, build +requirements from all packages in the build environment are included. + +Some example :attr:`private_build_requires` use cases include: + +* Documentation generators such as ``doxygen`` or ``sphinx``; +* Build utilities. For example, you may have a package called ``pyqt_cmake_utils``, which + provides CMake macros for converting ``ui`` files to ``py``; +* Statically linked libraries (since the library is linked at build time, the package + is not needed at runtime). + +An example use case of :attr:`build_requires` is a header-only (hpp) C++ library. If your own +C++ package includes this library in its own headers, other packages will also need this +library at build time (since they may include your headers, which in turn include the +hpp headers). + +Package Communication +===================== + +Let's say I have two C++ packages, ``maya_utils`` and the well-known ``boost`` library. How +does ``maya_utils`` find ``boost``'s header files, or library files? + +The short answer is, that is entirely up to you. Rez is not actually a build system. +It supports various build systems (as the next section describes), and it configures the +build environment, but the details of the build itself are left open for the user. +Having said that, `CMake `_ has been supported by rez for some time, and rez comes with a +decent amount of utility code to manage CMake builds. + +When a rez environment is configured, each required package's +:func:`~commands` section configures the environment for the building +package to use. When a build is occurring, a special variable +:attr:`building` is set to ``True``. Your required packages should use this +variable to communicate build information to the package being built. + +For example, our ``boost`` package's commands might look like so: + +.. code-block:: python + + def commands(): + if building: + # there is a 'FindBoost.cmake' file in this dir.. + env.CMAKE_MODULE_PATH.append("{root}/cmake") + +.. warning:: + Note that :func:`commands` is never executed for the package actually being built. + If you want to run commands in that case, you can use :func:`pre_build_commands` instead. + +A (very simple) ``FindBoost.cmake`` file might look like this: + +.. code-block:: cmake + + set(Boost_INCLUDE_DIRS $ENV{REZ_BOOST_ROOT}/include) + set(Boost_LIBRARY_DIRS $ENV{REZ_BOOST_ROOT}/lib) + set(Boost_LIBRARIES boost-python) + +Then, our ``maya_utils`` package might have a ``CMakeLists.txt`` file (cmake's build script) +containing: + +.. code-block:: cmake + + find_package(Boost) + include_directories(${Boost_INCLUDE_DIRS}) + link_directories(${Boost_LIBRARY_DIRS}) + target_link_libraries(maya_utils ${Boost_LIBRARIES}) + +As it happens, the `find_package `_ +CMake macro searches the paths listed in the `CMAKE_MODULE_PATH `_ environment variable, +and looks for a file called ``FindXXX.cmake``, where ``XXX`` is the name of the package (in this +case, ``Boost``), which it then includes. + +.. hint:: + Modern CMake should be used instead of ``FindXXX.cmake`` files. See the + `cmake packages `_ + documentation for more information. + +The Build System +================ + +Rez supports multiple build systems, and new ones can be added as plugins. When a +build is invoked, the build system is detected automatically. For example, if a +``CMakeLists.txt`` file is found in the package's root directory, the ``cmake`` build +system is used. + +Argument Passing +---------------- + +There are two ways to pass arguments to the build system. + +First, some build system plugins add extra options to the :ref:`rez-build` command directly. +For example, if you are in a CMake-based package, and you run ``rez-build -h``, you will +see cmake-specific options listed, such as ``--build-target``. + +Second, you can pass arguments directly to the build system, either using the +:option:`rez-build --build-args` option or listing the build system arguments after ``--``. + +For example, here we explicitly define a variable in a cmake build: + +.. code-block:: console + + $ rez-build -- -DMYVAR=YES + +Custom Build Commands +--------------------- + +As well as detecting the build system from build files, a package can explicitly +specify its own build command, using the +:attr:`build_command` package attribute. If present, +this takes precedence over other detected build systems. + +For example, consider the following ``package.py`` snippet: + +.. code-block:: python + + name = "nuke_utils" + + version = "1.2.3" + + build_command = "bash {root}/build.sh {install}" + +When :ref:`rez-build` is run on this package, the given ``build.sh`` script will be executed +with ``bash``. The ``{root}`` string expands to the root path of the package (the same +directory containing ``package.py``. The ``{install}`` string expands to ``install`` if +an install is occurring, or the empty string otherwise. This is useful for passing the +install target directly to the command (for example, when using ``make``) rather than +relying on a build script checking the :envvar:`REZ_BUILD_INSTALL` environment variable. + +.. warning:: + The current working directory during a build is set + to the *build path*, **not** to the package root directory. For this reason, you + will typically use the ``{root}`` string to refer to a build script in the package's + root directory. + +.. _custom-build-commands-pass-arguments: + +Passing Arguments ++++++++++++++++++ + +You can add arguments for your build script to the :ref:`rez-build` command directly, by +providing a ``parse_build_args.py`` source file in the package root directory. Here is an example: + +.. code-block:: python + + # in parse_build_args.py + parser.add_argument("--foo", action="store_true", help="do some foo") + +Now if you run ``rez-build -h`` on this package, you will see the option listed: + +.. code-block:: console + + $ rez-build -h + usage: rez build [-h] [-c] [-i] [-p PATH] [--fail-graph] [-s] [--view-pre] + [--process {remote,local}] [--foo] + [--variants INDEX [INDEX ...]] [--ba ARGS] [--cba ARGS] [-v] + + Build a package from source. + + optional arguments: + ... + --foo do some foo + +The added arguments are stored into environment variables so that your build script +can access them. They are prefixed with ``__PARSE_ARG_``; in our example above, the +variable ``__PARSE_ARG_FOO`` will be set. Booleans will be set to 0/1, and lists are +space separated, with quotes where necessary. + +Make Example +++++++++++++ + +Following is a very simple C++ example, showing how to use a custom build command to +build and install via ``make``: + +.. code-block:: python + + # in package.py + build_command = "make -f {root}/Makefile {install}" + +.. code-block:: makefile + + # in Makefile + hai: ${REZ_BUILD_SOURCE_PATH}/lib/main.cpp + g++ -o hai ${REZ_BUILD_SOURCE_PATH}/lib/main.cpp + + .PHONY: install + install: hai + mkdir -p ${REZ_BUILD_INSTALL_PATH}/bin + cp $< ${REZ_BUILD_INSTALL_PATH}/bin/hai + +Local Package Installs +====================== + +After you've made some code changes, you presumably want to test them. You do this +by *locally installing* the package, then resolving an environment with :ref:`rez-env` +to test the package in. The cycle goes like this: + +* Make code changes; +* Run ``rez-build --install`` to install as a local package; +* Run ``rez-env mypackage`` in a separate shell. This will pick up your local package, + and your package requirements; +* Test the package. + +A local install builds and installs the package to the :data:`local package repository `, +which is typically the directory :file:`~/packages`. +This directory is listed at the start of the +:ref:`package search path `, so when you resolve an +environment to test with, the locally installed package will be picked up first. Your +package will typically be installed to :file:`~/packages/{name}/{version}`, for example +:file:`~/packages/maya_utils/1.0.5`. If you have variants, they will be installed into subdirectories +within this install path (see :ref:`variants-disk-structure` for more details). + +.. tip:: + You don't need to run :ref:`rez-env` after every install. If your + package's requirements haven't changed, you can keep using the existing test environment. + +You can make sure you've picked up your local package by checking the output of the +:ref:`rez-env` call: + +.. code-block:: console + + $ rez-env sequence + + You are now in a rez-configured environment. + + resolved by ajohns@turtle, on Thu Mar 09 11:41:06 2017, using Rez v2.7.0 + + requested packages: + sequence + ~platform==linux (implicit) + ~arch==x86_64 (implicit) + ~os==Ubuntu-16.04 (implicit) + + resolved packages: + arch-x86_64 /sw/packages/arch/x86_64 + os-Ubuntu-16.04 /sw/packages/os/Ubuntu-16.04 + platform-linux /sw/packages/platform/linux + python-2.7.12 /sw/packages/python/2.7.12 + sequence-2.1.2 /home/ajohns/packages/sequence/2.1.2 (local) + +Note here that the ``sequence`` package is a local install, denoted by the ``(local)`` label. diff --git a/docs/source/commands_index.rst b/docs/source/commands_index.rst new file mode 100644 index 000000000..b818689a6 --- /dev/null +++ b/docs/source/commands_index.rst @@ -0,0 +1,7 @@ +======== +Commands +======== + +Every rez command is documented in these pages: + +.. rez-autoargparse:: diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..ff4a4623d --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,85 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import os +import sys + +# Add path to rez's source. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'src'))) + +# Add path to the root of the docs folder to get access to the rez_sphinxext extension. +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +import rez.utils._version + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'rez' +copyright = 'Contributors to the rez project' +author = 'Contributors to the rez project' +version = rez.utils._version._rez_version +release = rez.utils._version._rez_version + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + # Rez custom extension + 'rez_sphinxext' +] + +templates_path = ['_templates'] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'furo' +html_static_path = ['_static'] + +html_theme_options = { + 'light_logo': 'rez-horizontal-black.svg', + 'dark_logo': 'rez-horizontal-white.svg', + 'sidebar_hide_name': True, +} + +# These paths are either relative to html_static_path +# or fully qualified paths (eg. https://...) +html_css_files = [ + 'css/custom.css', +] + +# -- Options for intersphinx extension --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#module-sphinx.ext.intersphinx + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), +} + +# -- Options for autodoc extension ------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#module-sphinx.ext.autodoc + +# autoclass_content = 'both' +autodoc_class_signature = 'separated' +autodoc_member_order = 'bysource' + + +# -- Options for extlinks extension ----------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html + +extlinks = { + 'gh-rez': ('https://github.com/AcademySoftwareFoundation/rez/blob/master/%s', '%s'), +} + +# -- Options for todo extension --------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/todo.html + +todo_emit_warnings = True diff --git a/docs/source/configuring_rez.rst b/docs/source/configuring_rez.rst new file mode 100644 index 000000000..3555510ab --- /dev/null +++ b/docs/source/configuring_rez.rst @@ -0,0 +1,142 @@ +=============== +Configuring rez +=============== + +Rez has a good number of configurable settings. The default settings, and +documentation for every setting, can be found :gh-rez:`src/rez/rezconfig.py`. + +Settings are determined in the following way: + +- The setting is first read from the file ``rezconfig.py`` in the rez installation; +- The setting is then overridden if it is present in another settings file pointed at by the + :envvar:`REZ_CONFIG_FILE` environment variable. This can also be a path-like variable, to read from + multiple configuration files; +- The setting is further overriden if it is present in ``$HOME/.rezconfig``; +- The setting is overridden again if the environment variable :envvar:`REZ_XXX` is present, where ``XXX`` is + the uppercase version of the setting key. For example, :data:`.image_viewer` will be overriden by + :envvar:`REZ_IMAGE_VIEWER`. +- This is a special case applied only during a package build or release. In this case, if the + package definition file contains a "config" section, settings in this section will override all + others. See :ref:`configuring-rez-package-overrides`. + +It is fairly typical to provide your site-specific rez settings in a file that the environment +variable :envvar:`REZ_CONFIG_FILE` is then set to for all your users. + +.. tip:: + You do not need to provide a copy of all settings in this file. Just provide those + that are changed from the defaults. + +.. _configuring-rez-settings-merge-rules: + +Settings Merge Rules +==================== + +When multiple configuration sources are present, the settings are merged together - +one config file does not replace the previous one, it overrides it. By default, the +following rules apply: + +* Dicts are recursively merged together; +* Non-dicts override the previous value. + +However, it is also possible to append and/or prepend list-based settings by using the +:class:`ModifyList <.ModifyList>` class. For example, the +following config entry will append to the :data:`.release_hooks` setting value defined by the +previous configuration sources (you can also supply a ``prepend`` argument): + +.. code-block:: python + + release_hooks = ModifyList(append=["custom_release_notify"]) + +.. _configuring-rez-package-overrides: + +Package Overrides +================= + +.. todo:: Properly document the scope function as a function that takes a string, etc and make it referenceable. + +Packages themselves can override configuration settings. To show how this is useful, +consider the following example: + +.. code-block:: python + + # in package.py + with scope("config") as c: + c.release_packages_path = "/svr/packages/internal" + +Here a package is overriding the default release path - perhaps you're releasing +internally- and externally-developed packages to different locations, for example. + +These config overrides are only applicable during building and releasing of the package. +As such, even though any setting can be overridden, it's only useful to do so for +those that have any effect during the build/install process. These include: + +* Settings that determine where packages are found, such as :data:`.packages_path`, + :data:`.local_packages_path` and :data:`.release_packages_path`; +* Settings in the ``build_system``, ``release_hook`` and ``release_vcs`` plugin types; +* :data:`.package_definition_python_path`; +* :data:`.package_filter`. + +.. _configuring-rez-string-expansions: + +String Expansions +================= + +The following string expansions occur on all configuration settings: + +* Any environment variable reference, in the form ``${HOME}``; +* Any property of the ``system`` object, eg ``{system.platform}``. See :class:`rez.system.System` for more details. + +.. _configuring-rez-delay-load: + +Delay Load +========== + +It is possible to store a config setting in a separate file, which will be loaded +only when that setting is referenced. This can be useful if you have a large value +(such as a dict) that you don't want to pollute the main config with. YAML and +JSON formats are supported: + +.. code-block:: python + + # in rezconfig + default_relocatable_per_package = DelayLoad('/svr/configs/rez_relocs.yaml') + +See :Class:`.DelayLoad`. + +.. _configuring-rez-commandline-line: + +Commandline Tool +================ + +You can use the :ref:`rez-config` command line tool to see what the current configured settings are. +Called with no arguments, it prints all settings; if you specify an argument, it prints out just +that setting:: + + ]$ rez-config packages_path + - /home/sclaus/packages + - /home/sclaus/.rez/packages/int + - /home/sclaus/.rez/packages/ext + +Here is an example showing how to override settings using your own configuration file:: + + ]$ echo 'packages_path = ["~/packages", "/packages"]' > myrezconfig.py + ]$ export REZ_CONFIG_FILE=${PWD}/myrezconfig.py + ]$ rez-config packages_path + - /home/sclaus/packages + - /packages + +.. _configuring-rez-configuration-settings: + +Configuration Settings +====================== + +Following is an alphabetical list of rez settings. + +.. note:: + Note that this list has been generated automatically from the :gh-rez:`src/rez/rezconfig.py` + file in the rez source, so you can also refer to that file for the same information. + +.. This is a custom directive. See the rez_sphinxext.py file for more information. +.. TL;DR: It will take care of generating the documentation or all the settings defined +.. in rezconfig.py +.. rez-config:: diff --git a/docs/source/context.rst b/docs/source/context.rst new file mode 100644 index 000000000..a84a5b019 --- /dev/null +++ b/docs/source/context.rst @@ -0,0 +1,111 @@ +======= +Context +======= + +When you use :ref:`rez-env` to create a resolved environment, you are actually +creating something called a *context*. A context is a store of information +including: + +* The initial :ref:`package request ` list; +* The *resolve* (the list of variants that were chosen); +* A graph which shows the resolve visually. + +The context does not store copies of the packages it resolved to; rather, it +stores a kind of handle for each, which gives enough information to know where +to fetch the full package definition and contents from. + +Contexts themselves are quite small, and are stored in JSON format in a file +with the extension ``rxt``. When you use :ref:`rez-env`, it actually creates a temporary +context file on disk, which is removed when the shell is exited: + +.. code-block:: console + + $ rez-env foo bah + + You are now in a rez-configured environment. + + resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, + using Rez v2.0.rc1.10 + + requested packages: + foo + bah + + resolved packages: + eek-2.6 /packages/inhouse/eek/2.6 + foo-1.2 /packages/inhouse/foo/1.2 + bah-4 /packages/inhouse/bah/4 + + > $ echo $REZ_RXT_FILE + /tmp/rez_context_0tMS4U/context.rxt + +.. _context-baking-resolves: + +Baking Resolves +=============== + +You can use the :option:`rez-env --output` flag to write a resolved context directly +to file, rather than invoking a subshell: + +.. code-block:: console + + $ rez-env foo bah --output test.rxt + +Later, you can read the context back again, to reconstruct the same environment: + +.. code-block:: console + + $ rez-env --input test.rxt + + You are now in a rez-configured environment. + + resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, + using Rez v2.0.rc1.10 + + requested packages: + foo + bah + + resolved packages: + eek-2.6 /packages/inhouse/eek/2.6 + foo-1.2 /packages/inhouse/foo/1.2 + bah-4 /packages/inhouse/bah/4 + + > $ â–ˆ + +Contexts do not store a copy of the environment that is configured (that is, the +environment variables exported, for example). A context just stores the resolved +list of packages that need to be applied in order to configure the environment. +When you load a context via :option:`rez-env --input`, each of the packages' :attr:`commands` +sections are interpreted once more. + +You can think of package :attr:`commands` like fragments of a wrapper script which +configures an environment. By creating a context, you are creating a list of +script fragments which, when run in serial, produce the target environment. So, +if your package added a path to ``$PATH`` which included a reference to ``$USER`` +for example, this would work correctly even if Joe created the rxt file, and +Jill read it because the commands are reinterpreted when Jill loads the context. + +The rez-context Tool +==================== + +The :ref:`rez-context` tool inspects context files. When you're within a resolved +subshell, :ref:`rez-context` inspects the current context, unless one is specified +explicitly. For example, we can inspect the context created in the previous +example, without actually being within it: + +.. code-block:: console + + $ rez-context test.rxt + + resolved by ajohns@14jun01.methodstudios.com, on Wed Oct 22 12:44:00 2014, + using Rez v2.0.rc1.10 + + requested packages: + foo + bah + + resolved packages: + eek-2.6 /packages/inhouse/eek/2.6 + foo-1.2 /packages/inhouse/foo/1.2 + bah-4 /packages/inhouse/bah/4 diff --git a/docs/source/context_bundles.rst b/docs/source/context_bundles.rst new file mode 100644 index 000000000..d7846664e --- /dev/null +++ b/docs/source/context_bundles.rst @@ -0,0 +1,86 @@ +=============== +Context bundles +=============== + +A "context bundle" is a directory containing a :doc:`context ` (an rxt file), and a +package repository. All packages in the context are stored in the repository, +making the bundle relocatable and standalone. You can copy a bundle onto a +server for example, or into a container, and there are no external references +to shared package repositories. This is in contrast to a typical context, which +contains absolute references to one or more package repositories that are +typically on shared disk storage. + +To create a bundle via command line: + +.. code-block:: console + + $ rez-env foo -o foo.rxt + $ rez-bundle foo.rxt ./mybundle + + $ # example of running a command from the bundled context + $ rez-env -i ./mybundle/context.rxt -- foo-tool + +To create a bundle via API: + +.. code-block:: python + + from rez.bundle_context import bundle_context + from rez.resolved_context import ResolvedContext + + c = ResolvedContext(["python-3+", "foo-1.2+<2"]) + bundle_context(c, "./mybundle") + +Structure +========= + +A bundle directory looks like this: + +.. code-block:: text + + .../mybundle/ + ./context.rxt + ./packages/ + + +Package references in the rxt file are relative (unlike in a standard context, +where they're absolute), and this makes the bundle relocatable. + +Patching Libraries +================== + +Depending on how compiled libraries and executables within a rez package were +built, it's possible that the dynamic linker will attempt to resolve them to +libraries found outside of the bundle. For example, this is possible in linux +if an `elf `_ (``.so``) +contains an absolute search-path in its rpath/runpath header to a library in another package. + +Rez bundling performs a library patching step that applies various fixes to +solve this issue (use :option:`--no-lib-patch ` if you want to skip this step). This step +is platform-specific and is covered in the following sections. + +.. note:: + Note that in all cases, references to libraries outside of the bundle will remain intact, + if there is no equivalent path found within the bundle (for example, if the reference is + to a system library not provided by a rez package). + +Linux +----- + +On linux, `rpath/runpath `_ headers are altered if paths are found that map to a +subdirectory within another package in the bundle. To illustrate what happens, +consider the following example, where packages from :file:`/sw/packages` have been +bundled into the local directory :file:`./mybundle`: + +.. code-block:: console + + $ # a lib in an original non-bundled package + $ patchelf --print-rpath /sw/packages/foo/1.0.0/bin/foo + /sw/packages/bah/2.1.1/lib + $ + $ # the same lib in our bundle. We assume that package 'bah' is in the bundle + $ # also, since foo links to one of its libs + $ patchelf --print-rpath ./mybundle/packages/foo/1.0.0/bin/foo + $ORIGIN/../../../bah/2.1.1/lib + +Remapped rpaths make use of the special ``$ORIGIN`` variable, which refers to +the directory containing the current file. diff --git a/docs/source/environment.rst b/docs/source/environment.rst new file mode 100644 index 000000000..279db3db8 --- /dev/null +++ b/docs/source/environment.rst @@ -0,0 +1,253 @@ +===================== +Environment variables +===================== + +This chapter lists the environment variables that rez generates in certain +circumstances, as well as environment variables that you can set which affect +the operation of rez. + +.. _context-environment-variables: + +Context Environment Variables +============================= + +These are variables that rez generates within a resolved environment (a "context"). + +.. envvar:: REZ_RXT_FILE + + Filepath of the current context (an rxt file). + + .. seealso:: Documentation on :doc:`contexts `. + +.. envvar:: REZ_USED + + Path to rez installation that was used to resolve this environment. + +.. envvar:: REZ_USED_IMPLICIT_PACKAGES + + The list of implicit packages used in the resolve. + +.. envvar:: REZ_USED_PACKAGES_PATH + + The package search-path used for this resolve. + +.. envvar:: REZ_USED_RESOLVE + + The list of resolved packages, eg ``platform-linux utils-1.2.3``. + +.. envvar:: REZ_USED_EPH_RESOLVE + + The list of resolved ephemerals, eg ``.foo.cli-1 .debugging-0``. + +.. envvar:: REZ_USED_LOCAL_RESOLVE + + The list of resolved local packages, eg ``utils-1.2.3 maya_utils-1.3+``. Packages listed here will always be a subset of the packages in :envvar:`REZ_USED_RESOLVE`. + +.. envvar:: REZ_USED_REQUEST + + The environment request string, eg ``maya-2017 maya_utils-1.3+``. Does not include implicit packages. + +.. envvar:: REZ_USED_REQUESTED_TIMESTAMP + + The epoch time of this resolved environment, explicitly set by the user with (for example) the :option:`rez-env --time` flag; zero otherwise. + +.. envvar:: REZ_USED_TIMESTAMP + + The epoch time when this environment was resolved; OR, the value of :envvar:`REZ_USED_REQUESTED_TIMESTAMP`, if non-zero. + +.. envvar:: REZ_USED_VERSION + + The version of rez used to resolve this environment. + +.. envvar:: REZ_SHELL_INIT_TIMESTAMP + + The epoch time when the current shell was instantiated. + +.. envvar:: REZ_SHELL_INTERACTIVE + + Will be 1 if the shell is interactive, and 0 otherwise + (ie, when a command is specified, like ``rez-env foo -- mycommand``). + +.. envvar:: REZ_CONTEXT_FILE + + Filepath of the current context's shell code that is the result of all the + resolved packages :func:`commands`'s sections. + +Package environment variables +----------------------------- + +Specifically, per-package, the following variables are generated. Note that for a given +package name, ``(PKG)`` in the variables below is the uppercased package name, with any +dots replaced with underscore. + +.. envvar:: REZ_(PKG)_BASE + + The base directory of the package installation, eg ``/packages/utils/1.0.0``. + +.. envvar:: REZ_(PKG)_ROOT + + The root directory of the package installation (actually,the variant), eg ``/packages/utils/1.0.0/python-2.7``. + +.. envvar:: REZ_(PKG)_VERSION + + The version of the package. + +.. envvar:: REZ_(PKG)_MAJOR_VERSION + + The major version of the package, or an empty string. + +.. envvar:: REZ_(PKG)_MINOR_VERSION + + The minor version of the package, or an empty string. + +.. envvar:: REZ_(PKG)_PATCH_VERSION + + The patch version of the package, or an emopty string. + +Ephemeral packages environment variables +---------------------------------------- + +For every ephemeral package request, the following variables are generated. Note +that for a given ephemeral package name, ``(PKG)`` in the variables below is the +uppercased package name, with dots replaced by underscore, and **the leading dot +removed**: + +.. envvar:: REZ_EPH_(PKG)_REQUEST + + The resolved ephemeral package request. + +.. _build-environment-variables: + +Build Environment Variables +=========================== + +These are variables that rez generates within a build environment, in addition +to those listed :ref:`here `. + +.. glossary:: + +.. envvar:: REZ_BUILD_ENV + + Always present in a build, has value 1. + +.. envvar:: REZ_BUILD_INSTALL + + Has a value of 1 if an installation is taking place (either a :option:`rez-build -i` or :ref:`rez-release`), otherwise 0. + +.. envvar:: REZ_BUILD_INSTALL_PATH + + Installation path, if an install is taking place. + +.. envvar:: REZ_BUILD_PATH + + Path where build output goes. + +.. envvar:: REZ_BUILD_PROJECT_DESCRIPTION + + Equal to the *description* attribute of the package being built. + +.. envvar:: REZ_BUILD_PROJECT_FILE + + The filepath of the package being built (typically a ``package.py`` file). + +.. envvar:: REZ_BUILD_PROJECT_NAME + + Name of the package being built. + +.. envvar:: REZ_BUILD_PROJECT_VERSION + + Version of the package being built. + +.. envvar:: REZ_BUILD_REQUIRES + + Space-separated list of requirements for the build - comes from the current package's :attr:`requires`, + :attr:`build_requires` and :attr:`private_build_requires` attributes, including the current variant's requirements. + +.. envvar:: REZ_BUILD_REQUIRES_UNVERSIONED + + Equivalent but unversioned list to :envvar:`REZ_BUILD_REQUIRES`. + +.. envvar:: REZ_BUILD_SOURCE_PATH + + Path containing the package.py file. + +.. envvar:: REZ_BUILD_THREAD_COUNT + :noindex: + + Number of threads being used for the build. + + .. seealso:: The :data:`build_thread_count` setting. + +.. envvar:: REZ_BUILD_TYPE + + One of ``local`` or ``central``. Value is ``central`` if a release is occurring. + +.. envvar:: REZ_BUILD_VARIANT_INDEX + + Zero-based index of the variant currently being built. For non-varianted packages, this is 0. + +.. envvar:: REZ_BUILD_VARIANT_REQUIRES + + Space-separated list of runtime requirements of the current variant. This does not include + the common requirements as found in :envvar:`REZ_BUILD_REQUIRES`. For non-varianted builds, this is an empty string. + +.. envvar:: REZ_BUILD_VARIANT_SUBPATH + + Subdirectory containing the current variant. For non-varianted builds, this is an empty string. + +.. envvar:: __PARSE_ARG_XXX + + .. seealso:: :ref:`custom-build-commands-pass-arguments` + +.. _runtime-environment-variables: + +Runtime Environment Variables +============================= + +These are environment variables that the user can set, which affect the +operation of rez. + +.. envvar:: REZ_CONFIG_FILE + + Path to a rez configuration file. + +.. envvar:: REZ_XXX + + For any given rez config entry (see ``rezconfig.py``), + you can override the setting with an environment variable, for convenience. Here, + ``XXX`` is the uppercased equivalent of the setting name. For example, + a setting commonly overriden this way is :data:`packages_path`, whos equivalent + variable is :envvar:`REZ_PACKAGES_PATH`. + + .. hint:: + Each setting documented in :ref:`configuring-rez-configuration-settings` documents their environment variable. + +.. envvar:: REZ_XXX_JSON + + Same as :envvar:`REZ_XXX`, except that the format + is a JSON string. This means that some more complex settings can be overridden, + that aren't supported in the non-JSON case (:data:`package_filter` is an example). + +.. envvar:: REZ_DISABLE_HOME_CONFIG + + If 1/t/true, the default ``~/.rezconfig.py`` config file is skipped. + +.. envvar:: EDITOR + + On Linux and OSX systems, this will set the default editor to use + if and when rez requires one (an example is on release if the :data:`prompt_release_message` + config setting is true). + +.. envvar:: REZ_KEEP_TMPDIRS + + If set to a non-empty string, this prevents rez from + cleaning up any temporary directories. This is for debugging purposes. + +.. envvar:: REZ_SIGUSR1_ACTION + + If you set this to ``print_stack``, rez will prints its + current stacktrace to stdout if sent a USR1 signal. This is for debugging purposes. + +.. envvar:: REZ_ENV_PROMPT + + See the :data:`set_prompt` and :data:`prefix_prompt` settings. diff --git a/docs/source/ephemerals.rst b/docs/source/ephemerals.rst new file mode 100644 index 000000000..57c6c89f9 --- /dev/null +++ b/docs/source/ephemerals.rst @@ -0,0 +1,182 @@ +========== +Ephemerals +========== + +.. versionadded:: 2.71.0 + +Ephemeral packages (or simply 'ephemerals') are requests for packages that do not +exist. Ephemeral package names always begin with a dot (``.``). Like all package +requests, ephemerals can be requested as part of packages' requires or variants +lists, or directly by the user (via :ref:`rez-env` for eg). + +Example: + +.. code-block:: text + + ]$ rez-env .foo-1 + You are now in a rez-configured environment. + + resolved by ajohns@turtle, on Tue Dec 22 08:17:00 2020, using Rez v2.70.0 + + requested packages: + .foo-1 (ephemeral) + ~platform==linux (implicit) + ~arch==x86_64 (implicit) + ~os==Ubuntu-16.04 (implicit) + + resolved packages: + .foo-1 (ephemeral) + +Ephemerals will act like real packages during a resolve (ie, their request ranges +will intersect, and conflicts can occur) but they never actually correlate to a +real package, nor do they perform any configuration on the runtime (not directly +in any case). + +Example showing range intersection: + +.. code-block:: text + + ]$ rez-env .foo-1 '.foo-1.5+' + + You are now in a rez-configured environment. + + resolved by ajohns@turtle, on Tue Dec 22 08:21:04 2020, using Rez v2.70.0 + + requested packages: + .foo-1 (ephemeral) + .foo-1.5+ (ephemeral) + ~platform==linux (implicit) + ~arch==x86_64 (implicit) + ~os==Ubuntu-16.04 (implicit) + + resolved packages: + .foo-1.5+<1_ (ephemeral) + +Example of conflicting request: + +.. code-block:: text + + ]$ rez-env .foo-1 .foo-2 + The context failed to resolve: + The following package conflicts occurred: (.foo-1 <--!--> .foo-2) + +Environment Variables +===================== + +Ephemerals do not affect the runtime in the way that packages can (via their +:func:`commands` section), however some environment variables are set: + +* :envvar:`REZ_USED_EPH_RESOLVE` +* :envvar:`REZ_EPH_(PKG)_REQUEST` + +The following example illustrates: + +.. code-block:: text + + ]$ rez-env python .foo-1 .bah-2 + ... + ]$ echo $REZ_EPH_FOO_REQUEST + 1 + ]$ echo $REZ_USED_EPH_RESOLVE + .foo-1 .bah-2 + +Introspection +============= + +In order for a package to inspect the ephemerals that are present in a runtime, +there is an :attr:`ephemerals` object provided, similar +to the :attr:`resolve` object. You would typically use the +:func:`intersects` function to inspect it, like so: + +.. code-block:: python + + # in package.py + def commands() + if intersects(ephemerals.get_range('enable_tracking', '0'), '1'): + env.TRACKING_ENABLED = 1 + +In this example, the given package would set the ``TRACKING_ENABLED`` environment +variable if an ephemeral such as ``.enable_tracking-1`` (or ``.enable_tracking-1.2+`` +etc) is present in the resolve. Note that the leading ``.`` is implied and not +included when querying the :attr:`ephemerals` object. + +.. warning:: + Since :attr:`ephemerals` is a dict-like object, so it has + a ``get`` function which will return a full request string if key exists. Hence, + the default value should also be a full request string, not just a version range + string like ``0`` in :func:`ephemerals.get_range`. Or :func:`intersects` may not work as expect. + +Ephemeral Use Cases +=================== + +Why would you want to request packages that don't exist? There are two main use +cases. + +Passing Information to Packages +------------------------------- + +Ephemerals can be used as a kind of 'package option', or a way to pass information +to packages in a resolve. For example, consider the following package definition: + +.. code-block:: python + + name = 'bah' + + def commands(): + if intersects(ephemerals.get_range('bah.cli', '1'), '1'): + env.PATH.append('{root}/bin') + +This package will disable its command line tools if an ephemeral like ``.bah.cli-0`` +is present in the runtime. + +.. note:: + Ephemerals are standard package requests and so can + have any range, such as ``1.2.3``, ``2.5+`` and so on. However, they're often used + as boolean package options, as in the example above. In this case, it is + recommended to use the conventional ranges ``1`` and ``0`` to designate true and + false. + +Since ephemerals can be pretty much anything, you might also decide to use them +as a global package option. Here's another take on our example, but in this case +we introduce a ``.cli`` ephemeral that acts as a global whitelist: + +.. code-block:: python + + name = 'bah' + + def commands(): + if intersects(ephemerals.get_range('cli', ''), 'bah'): + env.PATH.append('{root}/bin') + +Here, all packages' cli will be enabled if ``.cli`` is not specified, but if it is +specified then it acts as a whitelist: + +.. code-block:: text + + # turn on cli for foo and bah only + ]$ rez-env foo-1 bah==2.3.1 eek-2.4 '.cli-foo|bah' + +Abstract Package Representation +------------------------------- + +Sometimes it makes sense for a package to require some form of abstract object or +capability, rather than an actual package. For example, perhaps your package (or +one of its variants) requires a GPU to be present on the host machine. To support +this, you might have something setup that includes a ``.gpu-1`` ephemeral in the +:ref:`implicits ` list on all GPU-enabled hosts. +Then, your package could look like this: + +.. code-block:: python + + name = 'pixxelator' + + variants = [ + ['.gpu-0'], # renders via CPU + ['.gpu-1'] # renders via GPU + ] + +.. warning:: + Be aware that on hosts that do **not** have a gpu + implicit, either variant could be selected. You would want to either guarantee + that every host has the gpu implicit set to 0 or 1, or that the user always + explicitly specifies ``.gpu-0`` or ``.gpu-1`` in their request. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 000000000..da4e133f5 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,77 @@ +.. rez documentation master file, created by + sphinx-quickstart on Sat Aug 5 19:30:28 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to rez's documentation! +=============================== + +.. toctree:: + :maxdepth: 2 + :caption: General + :hidden: + + installation + basic_concepts + building_packages + context + variants + +.. toctree:: + :maxdepth: 2 + :caption: Advanced + :hidden: + + ephemerals + context_bundles + suites + managing_packages + pip + +.. toctree:: + :maxdepth: 2 + :caption: Package definition + :hidden: + + package_definition + package_commands + +.. toctree:: + :maxdepth: 2 + :caption: Reference + :hidden: + + configuring_rez + commands_index + environment + api + +Rez is a cross-platform package manager with a difference. Using Rez you can create +standalone environments configured for a given set of packages. However, unlike many +other package managers, packages are not installed into these standalone environments. +Instead, all package versions are installed into a central repository, and standalone +environments reference these existing packages. This means that configured environments +are lightweight, and very fast to create, often taking just a few seconds to configure +despite containing hundreds of packages. + +Traditional package manager: + +.. figure:: _static/other_pkg_mgr.png + :align: center + :alt: Typical package managers install packages into an environment + :class: rez-diagram + + Typical package managers install packages into an environment + +Rez: + +.. figure:: _static/rez_pkg_mgr.png + :align: center + :alt: Rez installs packages once, and configures environments dynamically + :class: rez-diagram + + Rez installs packages once, and configures environments dynamically + +Rez takes a list of package requests, and constructs the target environment, resolving +all the necessary package dependencies. Any type of software package is supported - +compiled, python, applications and libraries. diff --git a/docs/source/installation.rst b/docs/source/installation.rst new file mode 100644 index 000000000..87c085a2b --- /dev/null +++ b/docs/source/installation.rst @@ -0,0 +1,86 @@ +============ +Installation +============ + +Installation Script +=================== + +To install rez, download the source. Then from the root directory, run:: + + ]$ python ./install.py + +This installs rez to ``/opt/rez``. See ```install.py -h``` for how to install to a +different location. + +Once the installation is complete, a message tells you how to run it:: + + SUCCESS! To activate Rez, add the following path to $PATH: + /opt/rez/bin/rez + + You may also want to source the completion script (for bash): + source /opt/rez/completion/complete.sh + + +.. warning:: + Do **not** move the installation. Re-install to a new location if you want to change + the install path. If you want to install rez for multiple operating systems, + perform separate installs for each of those systems. + + +Installation via pip +==================== + +It is possible to install rez with pip, like so:: + + ]$ pip install rez + +However, this comes with a caveat. Rez command line tools **are not guaranteed +to work correctly** once inside a rez environment (ie after using the :ref:`rez-env` +command). The reasons are given in the next section. + +Pip installation is adequate however, if all you require is the rez API, or you +don't require its command line tools to be available within a resolved environment. + +.. note:: + that running pip-installed rez command line tools will print a warning like so: + + .. code-block:: text + + Pip-based rez installation detected. Please be aware that rez command line tools + are not guaranteed to function correctly in this case. See :ref:`why-not-pip-for-production` + for further details. + +.. _why-not-pip-for-production: + +Why Not Pip For Production? +=========================== + +Rez is not a normal python package. Although it can successfully be installed +using standard mechanisms such as pip, this comes with a number of caveats. +Specifically: + +* When within a rez environment (ie after using the :ref:`rez-env` command), the rez + command line tools are not guaranteed to function correctly; +* When within a rez environment, other packages' tools (that were also installed + with pip) remain visible, but are not guaranteed to work. + +When you enter a rez environment, the rez packages in the resolve configure +that environment as they see fit. For example, it is not uncommon for a python +package to append to :envvar:`PYTHONPATH`. Environment variables such as :envvar:`PYTHONPATH` +affect the behaviour of tools, including rez itself, and this can cause it to +crash or behave abnormally. + +When you use the ``install.py`` script to install rez, some extra steps are taken +to avoid this problem. Specifically: + +* Rez is installed into a virtualenv so that it operates standalone; +* The rez tools are shebanged with ``python -E``, in order to protect them from + environment variables that affect python's behaviour; +* The rez tools are stored in their own directory, so that other unrelated tools + are not visible. + +Due to the way standard wheel-based python installations work, it simply is not +possible to perform these extra steps without using a custom installation script. +Wheels do not give the opportunity to run post-installation code. Neither do +they provide functionality for specifying interpreter arguments to be added for +any given entry point. diff --git a/docs/source/managing_packages.rst b/docs/source/managing_packages.rst new file mode 100644 index 000000000..1e6e1b995 --- /dev/null +++ b/docs/source/managing_packages.rst @@ -0,0 +1,384 @@ +================= +Managing packages +================= + +Ignoring Packages +================= + +Packages can be ignored. When this happens, the package is still present in its +repository, but it will not be visible to the rez API nor to any newly resolved +runtimes. Any runtimes that are currently using an ignored package are unaffected, +since the package's payload has not been removed. + +To ignore a package via commandline: + +.. code-block:: console + + $ # you need to specify the repo, but you'll be shown a list if you don't + $ rez-pkg-ignore foo-1.2.3 + No action taken. Run again, and set PATH to one of: + filesystem@/home/ajohns/packages + + $ rez-pkg-ignore foo-1.2.3 filesystem@/home/ajohns/packages + Package is now ignored and will not be visible to resolves + +Via API: + +.. code-block:: python + + >>> from rez.package_repository import package_repository_manager + >>> + >>> repo_path = "filesystem@/home/ajohns/packages" + >>> repo = package_repository_manager.get_repository(repo_path) + >>> repo.ignore_package("foo", "1.2.3") + 1 # -1: pkg not found; 0: pkg already ignored; 1: pkg ignored + +Both of these options generate a :file:`.ignore{{version}}` file (e.g. +``.ignore3.1.2``) next to the package version directory. + +You can also do the reverse (ie unignore a package). Use the :option:`-u ` option of +:ref:`rez-pkg-ignore`, or the :meth:`~rez.package_repository.PackageRepository.unignore_package` method on the package repository +object. + +Copying Packages +================ + +Packages can be copied from one :ref:`package repository ` +to another, like so: + +Via commandline: + +.. code-block:: console + + $ rez-cp --dest-path /svr/packages2 my_pkg-1.2.3 + +Via API: + +.. code-block:: python + + >>> from rez.package_copy import copy_package + >>> from rez.packages import get_latest_package + >>> + >>> p = get_latest_package("python") + >>> p + Package(FileSystemPackageResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) + >>> + >>> r = copy_package(p, "./repo2") + >>> + >>> print(pprint.pformat(r)) + { + 'copied': [ + ( + Variant(FileSystemVariantResource({'location': '/home/ajohns/packages', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'})), + Variant(FileSystemVariantResource({'location': '/home/ajohns/repo2', 'name': 'python', 'repository_type': 'filesystem', 'index': 0, 'version': '3.7.4'})) + ) + ], + 'skipped': [] + } + +Copying packages is actually done one variant at a time, and you can copy some +variants of a package if you want, rather than the entire package. The API call's +return value shows what variants were copied. The 2-tuple in ``copied`` lists the +source (the variant that was copied from) and destination (the variant that was +created) respectively. + +.. danger:: + Do not simply copy package directories on disk. + You should always use :ref:`rez-cp` or use the API. Copying directly on disk is bypassing rez and + this can cause problems such as a stale resolve cache. Using :ref:`rez-cp` and the API give + you more control anyway. + +.. _enabling-package-copying: + +Enabling Package Copying +------------------------ + +Copying packages is enabled by default, however you're also able to specify which +packages are and are not *relocatable*, for much the same reasons as given +:ref:`here `. + +You can mark a package as non-relocatable by setting :attr:`relocatable` +to ``False`` in its package definition file. There are also config settings that affect relocatability +in the event that relocatable is not defined in a package's definition. For example, +see :data:`default_relocatable`, :data:`default_relocatable_per_package` +and :data:`default_relocatable_per_repository`. + +Attempting to copy a non-relocatable package will raise a :exc:`~rez.exceptions.PackageCopyError`. +However, note that there is a ``force`` option that will override this. Use at +your own risk. + +.. _moving-packages: + +Moving Packages +=============== + +Packages can be moved from one :ref:`package repository ` +to another. Be aware that moving a package does not actually delete the source +package however. Instead, the source package is hidden (ignored). It is up to +you to delete it at some later date. + +To move a package via commandline: + +.. code-block:: console + + $ rez-mv --dest-path /packages2 python-3.7.4 /packages + +Via API: + +.. code-block:: python + + >>> from rez.package_move import move_package + >>> from rez.packages import get_package_from_repository + >>> + >>> p = get_package_from_repository("python", "3.7.4", "/packages") + >>> p + Package(FileSystemPackageResource({'location': '/packages', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) + >>> + >>> new_p = move_package(p, "/packages2") + >>> new_p + Package(FileSystemPackageResource({'location': '/packages2', 'name': 'python', 'repository_type': 'filesystem', 'version': '3.7.4'})) + >>> + >>> p = get_package_from_repository("python", "3.7.4", "/packages") + >>> p + None + +Be aware that a non-relocatable package is also not movable (see +:attr:`here `. Like package +copying, there is a ``force`` option to move it regardless. + +A typical reason you might want to move a package is to archive packages that are +no longer in use. In this scenario, you would move the package to some archival +package repository. In case an old runtime needs to be resurrected, you would add +this archival repository to the packages path before performing the resolve. + +.. note:: + You will probably want to use the :option:`--keep-timestamp ` option when doing this, + otherwise rez will think the package did not exist prior to its archival date. + +.. _removing-packages: + +Removing Packages +================= + +Packages can be removed. This is different from ignoring. The package and its +payload is deleted from storage, whereas ignoring just hides it. It is not +possible to un-remove a package. + +To remove a package via commandline: + +.. code-block:: console + + $ rez-rm --package python-3.7.4 /packages + +Via API: + +.. code-block:: python + + >>> from rez.package_remove import remove_package + >>> + >>> remove_package("python", "3.7.4", "/packages") + +During the removal process, package versions will first be ignored so that +partially-deleted versions are not visible. + +It can be useful to ignore packages that you don't want to use anymore, and +actually remove them at a later date. This gives you a safety buffer in case +current runtimes are using the package. They won't be affected if the package is +ignored, but could break if it is removed. + +To facilitate this workflow, :ref:`rez-rm` lets you remove all packages that have +been ignored for longer than N days (using the timestamp of the +:file:`.ignore{{version}}` file). Here we remove all packages that have been ignored +for 30 days or longer: + +.. code-block:: console + + $ rez-rm --ignored-since=30 -v + 14:47:09 INFO Searching filesystem@/home/ajohns/packages... + 14:47:09 INFO Removed python-3.7.4 from filesystem@/home/ajohns/packages + 1 packages were removed. + +Via API: + +.. code-block:: python + + >>> from rez.package_remove import remove_packages_ignored_since + >>> + >>> remove_packages_ignored_since(days=30) + 1 + +.. _package-caching: + +Package Caching +=============== + +Package caching is a feature that copies package payloads onto local disk in +order to speed up runtime environments. For example, if your released packages +reside on shared storage (which is common), then running say, a Python process, +will fetch all source from the shared storage across your network. The point of +the cache is to copy that content locally instead, and avoid the network cost. + +.. note:: + Please note: Package caching does **NOT** cache package + definitions. Only their payloads (ie, the package root directory). + +.. _enabling-package-caching: + +Enabling Package Caching +======================== + +Package caching is not enabled by default. To enable it, you need to configure +:data:`cache_packages_path` to specify a path to +store the cache in. + +You also have granular control over whether an individual package will or will +not be cached. To make a package cachable, you can set :attr:`cachable` +to False in its package definition file. Reasons you may *not* want to do this include +packages that are large, or that aren't relocatable because other compiled packages are +linked to them in a way that doesn't support library relocation. + +There are also config settings that affect cachability in the event that :attr:`cachable` +is not defined in a package's definition. For example, see +:data:`default_cachable`, :data:`default_cachable_per_package` +and :data:`default_cachable_per_repository`. + +Note that you can also disable package caching on the command line, using +:option:`rez-env --no-pkg-cache`. + +Verifying +--------- + +When you resolve an environment, you can see which variants have been cached by +noting the ``cached`` label in the right-hand column of the :ref:`rez-context` output, +as shown below: + +.. code-block:: console + + $ rez-env Flask + + You are now in a rez-configured environment. + + requested packages: + Flask + ~platform==linux (implicit) + ~arch==x86_64 (implicit) + ~os==Ubuntu-16.04 (implicit) + + resolved packages: + Flask-1.1.2 /home/ajohns/package_cache/Flask/1.1.2/d998/a (cached) + Jinja2-2.11.2 /home/ajohns/package_cache/Jinja2/2.11.2/6087/a (cached) + MarkupSafe-1.1.1 /svr/packages/MarkupSafe/1.1.1/d9e9d80193dcd9578844ec4c2c22c9366ef0b88a + Werkzeug-1.0.1 /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a (cached) + arch-x86_64 /home/ajohns/package_cache/arch/x86_64/6450/a (cached) + click-7.1.2 /home/ajohns/package_cache/click/7.1.2/0da2/a (cached) + itsdangerous-1.1.0 /home/ajohns/package_cache/itsdangerous/1.1.0/b23f/a (cached) + platform-linux /home/ajohns/package_cache/platform/linux/9d4d/a (cached) + python-3.7.4 /home/ajohns/package_cache/python/3.7.4/ce1c/a (cached) + +For reference, cached packages also have their original payload location stored to +an environment variable like so: + +.. code-block:: console + + $ echo $REZ_FLASK_ORIG_ROOT + /svr/packages/Flask/1.1.2/88a70aca30cb79a278872594adf043dc6c40af99 + +How it Works +------------ + +Package caching actually caches :doc:`variants`, not entire packages. When you perform +a resolve, or source an existing context, the variants required are copied to +local disk asynchronously (if they are cachable), in a separate process called +:ref:`rez-pkg-cache`. This means that a resolve will not necessarily use the cached +variants that it should, the first time around. Package caching is intended to have +a cumulative effect, so that more cached variants will be used over time. This is +a tradeoff to avoid blocking resolves while variant payloads are copied across +your network (and that can be a slow process). + +Note that a package cache is **not** a package repository. It is simply a store +of variant payloads, structured in such a way as to be able to store variants from +any package repository, into the one shared cache. + +Variants that are cached are assumed to be immutable. No check is done to see if +a variant's payload has changed, and needs to replace an existing cache entry. So +you should **not** enable caching on package repositories where packages may get +overwritten. It is for this reason that caching is disabled for local packages by +default (see :data:`package_cache_local`). + +Commandline Tool +---------------- + +Inspection +++++++++++ + +Use the :ref:`rez-pkg-cache` tool to view the state of the cache, and to perform +warming and deletion operations. Example output follows: + +.. code-block:: console + + $ rez-pkg-cache + Package cache at /home/ajohns/package_cache: + + status package variant uri cache path + ------ ------- ----------- ---------- + cached Flask-1.1.2 /svr/packages/Flask/1.1.2/package.py[0] /home/ajohns/package_cache/Flask/1.1.2/d998/a + cached Jinja2-2.11.2 /svr/packages/Jinja2/2.11.2/package.py[0] /home/ajohns/package_cache/Jinja2/2.11.2/6087/a + cached Werkzeug-1.0.1 /svr/packages/Werkzeug/1.0.1/package.py[0] /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a + cached arch-x86_64 /svr/packages/arch/x86_64/package.py[] /home/ajohns/package_cache/arch/x86_64/6450/a + cached click-7.1.2 /svr/packages/click/7.1.2/package.py[0] /home/ajohns/package_cache/click/7.1.2/0da2/a + cached itsdangerous-1.1.0 /svr/packages/itsdangerous/1.1.0/package.py[0] /home/ajohns/package_cache/itsdangerous/1.1.0/b23f/a + cached platform-linux /svr/packages/platform/linux/package.py[] /home/ajohns/package_cache/platform/linux/9d4d/a + copying python-3.7.4 /svr/packages/python/3.7.4/package.py[0] /home/ajohns/package_cache/python/3.7.4/ce1c/a + stalled MarkupSafe-1.1.1 /svr/packages/MarkupSafe/1.1.1/package.py[1] /home/ajohns/package_cache/MarkupSafe/1.1.1/724c/a + +Each variant is stored into a directory based on a partial hash of that variant's +unique identifier (its "handle"). The package cache is thread and multiprocess +proof, and uses a file lock to control access where necessary. + +Cached variants have one of the following statuses at any given time: + +* **copying**: The variant is in the process of being copied into the cache, and is not + yet available for use; +* **cached**: The variant has been cached and is ready for use; +* **stalled**: The variant was getting copied, but something went wrong and there is + now a partial copy present (but unused) in the cache. + +Logging ++++++++ + +Caching operations are stored into logfiles within the cache directory. To view: + +.. code-block:: console + + $ rez-pkg-cache --logs + rez-pkg-cache 2020-05-23 16:17:45,194 PID-29827 INFO Started daemon + rez-pkg-cache 2020-05-23 16:17:45,201 PID-29827 INFO Started caching of variant /home/ajohns/packages/Werkzeug/1.0.1/package.py[0]... + rez-pkg-cache 2020-05-23 16:17:45,404 PID-29827 INFO Cached variant to /home/ajohns/package_cache/Werkzeug/1.0.1/fe76/a in 0.202576 seconds + rez-pkg-cache 2020-05-23 16:17:45,404 PID-29827 INFO Started caching of variant /home/ajohns/packages/python/3.7.4/package.py[0]... + rez-pkg-cache 2020-05-23 16:17:46,006 PID-29827 INFO Cached variant to /home/ajohns/package_cache/python/3.7.4/ce1c/a in 0.602037 seconds + +Cleaning The Cache +++++++++++++++++++ + +Cleaning the cache refers to deleting variants that are stalled or no longer in use. +It isn't really possible to know whether a variant is in use, so there is a +configurable :data:`package_cache_max_variant_days` +setting, that will delete variants that have not been used (ie that have not appeared +in a created or sourced context) for more than N days. + +You can also manually remove variants from the cache using :option:`rez-pkg-cache -r`. +Note that when you do this, the variant is no longer available in the cache, +however it is still stored on disk. You must perform a clean (:option:`rez-pkg-cache --clean`) +to purge unused cache files from disk. + +You can use the :data:`package_cache_clean_limit` +setting to asynchronously perform some cleanup every time the cache is updated. If +you do not use this setting, it is recommended that you set up a cron or other form +of execution scheduler, to run :option:`rez-pkg-cache --clean` periodically. Otherwise, +your cache will grow indefinitely. + +Lastly, note that a stalled variant will not attempt to be re-cached until it is +removed by a clean operation. Using :data:`package_cache_clean_limit` will not clean +stalled variants either, as that could result in a problematic variant getting +cached, then stalled, then deleted, then cached again and so on. You must run +:option:`rez-pkg-cache --clean` to delete stalled variants. diff --git a/docs/source/package_commands.rst b/docs/source/package_commands.rst new file mode 100644 index 000000000..24b357d19 --- /dev/null +++ b/docs/source/package_commands.rst @@ -0,0 +1,782 @@ +================ +Package commands +================ + +Package definition files (``package.py``) usually define a :func:`.commands` section. This is a python +function that determines how the environment is configured in order to include the package. + +Consider the simple example: + +.. code-block:: python + + def commands(): + env.PYTHONPATH.append("{root}/python") + env.PATH.append("{root}/bin") + +This is a typical case, where a package adds its source path to ``PYTHONPATH``, and its tools to +``PATH``. The ``{root}`` string expands to the installation directory of the package. + +When a rez environment is configured, every package in the resolve list has its :func:`.commands` section +interpreted and converted into shell code (the language, bash or other, depends on the platform +and is extensible). The resulting shell code is sourced, and this configures the environment. +Within a configured environment, the variable :envvar:`REZ_CONTEXT_FILE` points at this shell code, and the +command :option:`rez-context --interpret` prints it. + +The python API that you use in the :func:`.commands` section is called ``rex`` (**R**\ez **EX**\ecution language). It +is an API for performing shell operations in a shell-agnostic way. Some common operations you would +perform with this API include setting environment variables, and appending/prepending path-like +environment variables. + +.. note:: + By default, environment variables that are not referenced by any package + are left unaltered. There will typically be many system variables that are left unchanged. + +.. warning:: + If you need to import any python modules to use in a :func:`.commands` + section, the import statements **must** be done inside that function. + +.. _package-commands-order-of-execution: + +Order Of Command Execution +========================== + +The order in which package commands are interpreted depends on two factors: the order in which +the packages were requested, and dependencies between packages. This order can be defined as: + +* If package ``A`` was requested before package ``B``, then ``A``'s commands are interpreted before ``B``'s; +* Unless package ``A`` requires (depends on) ``B``, in which case ``B`` will be interpreted before ``A``. + +Consider a package ``maya_anim_tool``. Let us say this is a maya plugin. Naturally it has a dependency +on ``maya``, therefore ``maya``'s commands will be interpreted first. This is because the maya plugin +may depend on certain environment variables that ``maya`` sets. For example, ``maya`` might initialize +the ``MAYA_PLUG_IN_PATH`` environment variable, and ``maya_anim_tool`` may then append to this +variable. + +For example, consider the request: + +.. code-block:: text + + ]$ rez-env maya_anim_tool-1.3+ PyYAML-3.10 maya-2015 + +Assuming that ``PyYAML`` depends on ``python``, and ``maya_anim_tool`` depends on ``maya``, then the +resulting :func:`.commands` execution order would be: + +* maya; +* maya_anim_tool; +* python; +* PyYAML. + +.. _variable-appending-and-prepending: + +Variable Appending And Prepending +================================= + +Path-like environment variables can be appended and prepended like so: + +.. code-block:: python + + env.PATH.append("{root}/bin") + +However, the first append/prepend operation on any given variable actually **overwrites** the +variable, rather than appending. Why does this happen? Consider ``PYTHONPATH``: if an initial +overwrite did not happen, then any modules visible on ``PYTHONPATH`` before the rez environment was +configured would still be there. This would mean you may not have a properly configured +environment. If your system ``PyQt`` were on ``PYTHONPATH`` for example, and you used :ref:`rez-env` to set +a different ``PyQt`` version, an attempt to import it within the configured environment would still, +incorrectly, import the system version. + +.. note:: + ``PATH`` is a special case. It is not simply overwritten, because if that + happened you would lose important system paths and thus utilities like ``ls`` and ``cd``. In this + case the system paths are appended back to ``PATH`` after all commands are interpreted. The system + paths are defined as the default value of ``PATH`` in a non-interactive shell. + +.. todo:: Add custom class for "construction"? + +.. admonition:: Noteasd + + Better control over environment variable initialization is + coming. Specifically, you will be able to specify various modes for variables. For example, one + mode will append the original (pre-rez) value back to the resulting value. + +.. _string-expansion: + +String Expansion +================ + +Object Expansion +---------------- + +Any of the objects available to you in a :func:`commands` section can be referred to in formatted strings +that are passed to rex functions such as :func:`setenv` and so on. For example, consider the code: + +.. code-block:: python + + appendenv("PATH", "{root}/bin") + +Here, ``{root}`` will expand out to the value of :attr:`root`, which is the installation path of the +package (:attr:`this.root` could also have been used). + +You don't *have* to use this feature. It is provided as a convenience. For example, the following +code is equivalent to the previous example, and is just as valid (but more verbose): + +.. code-block:: python + + import os.path + appendenv("PATH", os.path.join(root, "bin")) + +Object string expansion is also supported when setting an environment variable via the :attr:`env` object: + +.. code-block:: python + + env.FOO_LIC = "{this.root}/lic" + +Environment Variable Expansion +------------------------------ + +Environment variable expansion is also supported when passed to rex functions. Both syntax ``$FOO`` +and ``${FOO}`` are supported, regardless of the syntax supported by the target shell. + +Literal Strings +--------------- + +You can use the :func:`literal` function to inhibit object and environment variable string +expansion. For example, the following code will set the environment variable to the literal string: + +.. code-block:: python + + env.TEST = literal("this {root} will not expand") + +There is also an :func:`expandable` function, which matches the default behavior. You wouldn't typically +use this function. However, you can define a string containing literal and expandable parts by +chaining together :func:`literal` and :func:`expandable`: + +.. code-block:: python + + env.DESC = literal("the value of {root} is").expandable("{root}") + +.. _explicit-string-expansion: + +Explicit String Expansion +------------------------- + +Object string expansion usually occurs **only** when a string is passed to a rex function, or to +the :attr:`env` object. For example the simple statement ``var = "{root}/bin"`` would not expand ``{root}`` +into ``var``. However, you can use the :func:`expandvars` function to enable this behavior +explicitly: + +.. code-block:: python + + var = expandvars("{root}/bin") + +The :func:`expandvars` and :func:`expandable` functions are slightly different. :func:`expandable` will generate a +shell variable assignment that will expand out while :func:`expandvars` will expand the value immediately. + +This table illustrates the difference between :func:`literal`, :func:`expandable` and :func:`expandvars`: + +=================================== ======================= +Package command Equivalent bash command +=================================== ======================= +``env.FOO = literal("${USER}")`` ``export FOO='${USER}'`` +``env.FOO = expandable("${USER}")`` ``export FOO="${USER}"`` +``env.FOO = expandvars("${USER}")`` ``export FOO="jbloggs"`` +=================================== ======================= + +.. admonition:: Additional context + :class: admonition note + + In Bash, single quote strings (``'foo'``) will not be expanded. + +Filepaths +========= + +Rez expects POSIX-style filepath syntax in package commands, regardless of the shell or platform. +Thus, even if you're on Windows, you should do this: + +.. code-block:: python + + def commands(): + env.PATH.append("{root}/bin") # note the forward slash + +Where necessary, filepaths will be automatically normalized for you. That is, converted into +the syntax expected by the shell. In order for this to work correctly however, rez needs to know +what environment variables are actually paths. You determine this with the +:data:`pathed_env_vars` config setting. By default, any environment +variable ending in ``PATH`` will be treated as a filepath or list of filepaths, and any +set/append/prepend operation on it will cause those values to be path-normalized automatically. + +.. warning:: + Avoid using :data:`os.pathsep` or hardcoded lists of paths such as + ``{root}/foo:{root}/bah``. Doing so can cause your package to be incompatible with some shells or + platforms. Even the seemingly innocuous :data:`os.pathsep` is an issue, because there are some cases + (eg Git for Windows, aka git-bash) where the shell's path separator does not match the underlying + system's. + +Pre And Post Commands +===================== + +Occasionally, it's useful for a package to run commands either before or after all other packages, +regardless of the command execution order rules. This can be achieved by defining a :func:`pre_commands` +or :func:`post_commands` function. A package can have any, all or none of :func:`pre_commands`, :func:`commands` and +:func:`post_commands` defined, although it is very common for a package to define just :func:`commands`. + +The order of command execution is: + +* All package :func:`pre_commands` are executed, in standard execution order; +* Then, all package :func:`commands` are executed, in standard execution order; +* Then, all package :func:`post_commands` are executed, in standard execution order. + +.. _pre-build-commands: + +Pre Build Commands +================== + +If a package is being built, that package's commands are not run, simply because that package is +not present in its own build environment! However, sometimes there is a need to run commands +specifically for the package being built. For example, you may wish to set some environment +variables to pass information along to the build system. + +The :func:`pre_build_commands` function does just this. It is called prior to the build. Note that info +about the current build (such as the installation path) is available in a +:attr:`build` object (other commands functions do not have this object visible). + +.. _pre-test-commands: + +Pre Test Commands +================= + +Sometimes it's useful to perform some extra configuration in the environment that a package's test +will run in. You can define the :func:`pre_test_commands` function to do this. It will be invoked just +before the test is run. As well as the standard :attr:`this` object, a :attr:`test` object is also +provided to distinguish which test is about to run. + +A Largish Example +================= + +Here is an example of a package definition with a fairly lengthy :func:`commands` section: + +.. code-block:: python + + name = "foo" + + version = "1.0.0" + + requires = [ + "python-2.7", + "~maya-2015" + ] + + def commands(): + import os.path # imports MUST be inline to the function + + # add python module, executables + env.PYTHONPATH.append("{this.root}/python") + env.PATH.append("{this.root}/bin") + + # show include path if a build is occurring + if building: + env.FOO_INCLUDE_PATH = "{this.root}/include" + + # debug support to point at local config + if defined("DEBUG_FOO"): + conf_file = os.path.expanduser("~/.foo/config") + else: + conf_file = "{this.root}/config" + env.FOO_CONFIG_FILE = conf_file + + # if maya is in use then include the maya plugin part of this package + if "maya" in resolve: + env.MAYA_PLUG_IN_PATH.append("{this.root}/maya/plugins") + + if resolve.maya.version.minor == "sp3": + error("known issue with GL renderer in service pack 3, beware") + + # license file per major version + env.FOO_LIC = "/lic/foo_{this.version.major}.lic" + +Objects +======= + +Various objects and functions are available to use in the :func:`commands` function (as well as +:func:`pre_commands` and :func:`post_commands`). + +Following is a list of the objects and functions available. + +.. .. currentmodule:: pkgdefrex + +.. py:function:: alias() + + Create a command alias. + + .. code-block:: python + + alias("nukex", "Nuke -x") + + .. note:: + In ``bash``, aliases are implemented as bash functions. + +.. py:attribute:: base + :type: str + + See :attr:`this.base`. + +.. py:attribute:: build + + This is a dict like object. Each key can also be accessed as attributes. + + This object is only available in the :func:`pre_build_commands` + function. It has the following fields: + + .. code-block:: python + + if build.install: + info("An installation is taking place") + + if build['build_type'] == 'local': + pass + +.. py:attribute:: build.build_type + :type: typing.Literal['local', 'central'] + + One of ``local``, ``central``. The type is ``central`` if a package release is occurring, and ``local`` + otherwise. + +.. py:attribute:: build.install + :type: bool + + True if an installation is taking place, False otherwise. + +.. py:attribute:: build.build_path + :type: str + + Path to the build directory (not the installation path). This will typically reside somewhere + within the ``./build`` subdirectory of the package being built. + +.. py:attribute:: build.install_path + :type: str + + Installation directory. Note that this will be set, even if an installation is **not** taking place. + + .. warning:: + Do not check this variable to detect if an installation is occurring. Use :attr:`build.install` instead. + +.. py:attribute:: building + :type: bool + + This boolean variable is ``True`` if a build is occurring (typically done via the :ref:`rez-build` tool), + and ``False`` otherwise. + + However, the :func:`commands` block is only executed when the package is brought + into a resolved environment, so this is not used when the package itself is building. Typically a + package will use this variable to set environment variables that are only useful during when other + packages are being built. C++ header include paths are a good example. + + .. code-block:: python + + if building: + env.FOO_INCLUDE_PATH = "{root}/include" + +.. py:function:: command(arg: str) + + Run an arbitrary shell command. + + Example: + + .. code-block:: python + + command("rm -rf ~/.foo_plugin") + + .. note:: + Note that you cannot return a value from this function call, because + *the command has not yet run*. All of the packages in a resolve only have their commands executed + after all packages have been interpreted and converted to the target shell language. Therefore any + value returned from the command, or any side effect the command has, is not visible to any package. + + You should prefer to perform simple operations (such as file manipulations and so on) in python + where possible instead. Not only does that take effect immediately, but it's also more cross + platform. For example, instead of running the command above, we could have done this: + + .. code-block:: python + + def commands(): + import shutil + import os.path + path = os.path.expanduser("~/.foo_plugin") + if os.path.exists(path): + shutil.rmtree(path) + +.. py:function:: comment(arg: str) + + Creates a comment line in the converted shell script code. This is only visible if the user views + the current shell's code using the command :option:`rez-context --interpret` or looks at the file + referenced by the environment variable :envvar:`REZ_CONTEXT_FILE`. You would create a comment for debugging + purposes. + + .. code-block:: python + + if "nuke" in resolve: + comment("note: taking over 'nuke' binary!") + alias("nuke", "foo_nuke_replacer") + + +.. py:function:: defined(envvar: str) -> bool + + Use this boolean function to determine whether or not an environment variable is set. + + .. code-block:: python + + if defined("REZ_MAYA_VERSION"): + env.FOO_MAYA = 1 + +.. py:attribute:: env + :type: dict + + The ``env`` object represents the environment dict of the configured environment. Environment variables + can also be accessed as attributes. + + .. note:: + Note that this is different from the standard python :data:`os.environ` dict, which represents the current environment, + not the one being configured. If a prior package's :func:`commands` sets a variable via the ``env`` object, + it will be visible only via ``env``, not :data:`os.environ`. The :data:`os.environ` dict hasn't been updated because the target + configured environment does not yet exist! + + .. code-block:: python + + env.FOO_DEBUG = 1 + env["BAH_LICENSE"] = "/lic/bah.lic" + +.. py:function:: env.append(value: str) + + Appends a value to an environment variable. By default this will use the :data:`os.pathsep` delimiter + between list items, but this can be overridden using the config setting :data:`env_var_separators`. See + :ref:`variable-appending-and-prepending` for further information on the behavior of this function. + + .. code-block:: python + + env.PATH.append("{root}/bin") + +.. py:function:: env.prepend(value: str) + + Like :func:`env.append`, but prepends the environment variable instead. + + .. code-block:: python + + env.PYTHONPATH.prepend("{root}/python") + +.. py:attribute:: ephemerals + + A dict like object representing the list of ephemerals in the resolved environment. Each item is a + string (the full request, eg ``.foo.cli-1``), keyed by the ephemeral package name. Note + that you do **not** include the leading ``.`` when getting items from the ``ephemerals`` + object. + + Example: + + .. code-block:: python + + if "foo.cli" in ephemerals: + info("Foo cli option is being specified!") + +.. py:function:: ephemerals.get_range(name: str, range_: str) -> ~rez.vendor.version.version.VersionRange + + Use ``get_range`` to test with the :func:`intersects` function. + Here, we enable ``foo``'s commandline tools by default, unless explicitly disabled via + a request for ``.foo.cli-0``: + + .. code-block:: python + + if intersects(ephemerals.get_range("foo.cli", "1"), "1"): + info("Enabling foo cli tools") + env.PATH.append("{root}/bin") + +.. py:function:: error(message: str) + + Prints to standard error. + + .. note:: + This function just prints the error, it does not prevent the target + environment from being constructed (use the :func:`stop`) command for that). + + .. code-block:: python + + if "PyQt" in resolve: + error("The floob package has problems running in combo with PyQt") + +.. py:function:: expandable(arg: str) -> ~rez.rex.EscapedString + + See :ref:`explicit-string-expansion`. + +.. py:function:: expandvars(arg: str) + + See :ref:`explicit-string-expansion`. + +.. py:function:: getenv(envvar: str) + + Gets the value of an environment variable. + + .. code-block:: python + + if getenv("REZ_MAYA_VERSION") == "2016.sp1": + pass + + :raises RexUndefinedVariableError: if the environment variable is not set. + +.. py:attribute:: implicits + + A dict like object that is similar to the :attr:`request` object, but it contains only the package request as + defined by the :data:`implicit_packages` configuration setting. + + .. code-block:: python + + if "platform" in implicits: + pass + +.. py:function:: info(message: str) + + Prints to standard out. + + .. code-block:: python + + info("floob version is %s" % resolve.floob.version) + +.. py:function:: intersects(range1: str | ~rez.vendor.version.version.VersionRange | ~rez.rex_bindings.VariantBinding | ~rez.rex_bindings.VersionBinding, range2: str) -> bool + + A boolean function that returns True if the version or version range of the given + object, intersects with the given version range. Valid objects to query include: + + * A resolved package, eg ``resolve.maya``; + * A package request, eg ``request.foo``; + * A version of a resolved package, eg ``resolve.maya.version``; + * A resolved ephemeral, eg ``ephemerals.foo``; + * A version range object, eg ``ephemerals.get_range('foo.cli', '1')`` + + .. warning:: + Do **not** do this: + + .. code-block:: python + + if intersects(ephemerals.get("foo.cli", "0"), "1"): + ... + + .. todo:: document request.get_range + + If ``foo.cli`` is not present, this will unexpectedly compare the unversioned + package named ``0`` against the version range ``1``, which will succeed! Use + :func:`ephemerals.get_range` and ``request.get_range`` functions instead: + + .. code-block:: python + + if intersects(ephemerals.get_range("foo.cli", "0"), "1"): + ... + + Example: + + .. code-block:: python + + if intersects(resolve.maya, "2019+"): + info("Maya 2019 or greater is present") + +.. py:function:: literal(arg: str) -> ~rez.rex.EscapedString + + Inhibits expansion of object and environment variable references. + + .. code-block:: python + + env.FOO = literal("this {root} will not expand") + + You can also chain together ``literal`` and :func:`expandable` functions like so: + + .. code-block:: python + + env.FOO = literal("the value of {root} is").expandable("{root}") + +.. py:function:: optionvars(name: str, default: typing.Any | None = None) -> typing.Any + + A :meth:`dict.get` like function for package accessing arbitrary data from :data:`optionvars` in rez config. + +.. py:attribute:: request + :type: ~rez.rex_bindings.RequirementsBinding + + A dict like object representing the list of package requests. Each item is a request string keyed by the + package name. For example, consider the package request: + + .. code-block:: text + + ]$ rez-env maya-2015 maya_utils-1.2+<2 !corelib-1.4.4 + + This request would yield the following ``request`` object: + + .. code-block:: python + + { + "maya": "maya-2015", + "maya_utils": "maya_utils-1.2+<2", + "corelib": "!corelib-1.4.4" + } + + Use ``get_range`` to test with the :func:`intersects` function: + + if intersects(request.get_range("maya", "0"), "2019"): + info("maya 2019.* was asked for!") + + Example: + + .. code-block:: python + + if "maya" in request: + info("maya was asked for!") + + .. tip:: + If multiple requests are present that refer to the same package, the + request is combined ahead of time. In other words, if requests ``foo-4+`` and ``foo-<6`` were both + present, the single request ``foo-4+<6`` would be present in the ``request`` object. + +.. py:function:: resetenv(envvar: str, value: str, friends=None) -> None + + TODO: Document + +.. py:attribute:: resolve + + A dict like object representing the list of packages in the resolved environment. Each item is a + :ref:`Package ` object, keyed by the package name. + + Packages can be accessed using attributes (ie ``resolve.maya``). + + .. code-block:: python + + if "maya" in resolve: + info("Maya version is %s", resolve.maya.version) + # ..or resolve["maya"].version + +.. py:attribute:: root + :type: str + + See :attr:`this.root`. + +.. py:function:: setenv(envvar: str, value: str) + + This function sets an environment variable to the given value. It is equivalent to setting a + variable via the :attr:`env` object (eg, ``env.FOO = 'BAH'``). + + .. code-block:: python + + setenv("FOO_PLUGIN_PATH", "{root}/plugins") + +.. py:function:: source(path: str) -> None + + Source a shell script. Note that, similarly to :func:`commands`, this function cannot return a value, and + any side effects that the script sourcing has is not visible to any packages. For example, if the + ``init.sh`` script below contained ``export FOO=BAH``, a subsequent test for this variable on the + :attr:`env` object would yield nothing. + + .. code-block:: python + + source("{root}/scripts/init.sh") + +.. py:attribute:: stop(message: str) -> typing.NoReturn + + Raises an exception and stops a resolve from completing. You should use this when an unrecoverable + error is detected and it is not possible to configure a valid environment. + + .. code-block:: python + + stop("The value should be %s", expected_value) + +.. py:attribute:: system + :type: ~rez.system.System + + This object provided system information, such as current platform, arch and os. + + .. code-block:: python + + if system.platform == "windows": + ... + +.. py:attribute:: test + + Dict like object to access test related attributes. Only available in the :func:`pre_test_commands` function. + Keys can be accessed as object attributes. + +.. py:attribute:: test.name + :type: str + + Name of the test about to run. + + .. code-block:: python + + if test.name == "unit": + info("My unit test is about to run yay") + +.. py:attribute:: this + + The ``this`` object represents the current package. The following attributes are most commonly used + in a :func:`commands`) section (though you have access to all package attributes. See :ref:`here `): + + .. py:attribute:: this.base + :type: str + + Similar to :attr:`this.root`, but does not include the variant subpath, if there is one. Different + variants of the same package share the same :attr:`base` directory. See :doc:`here ` for more + information on package structure in relation to variants. + + .. py:attribute:: this.is_package + :type: bool + + .. todo:: Document + + TODO: Document + + .. py:attribute:: this.is_variant + :type: bool + + .. todo:: Document + + TODO: Document + + .. py:attribute:: this.name + :type: str + + The name of the package, eg ``houdini``. + + .. py:attribute:: this.root + :type: str + + The installation directory of the package. If the package contains variants, this path will include + the variant subpath. This is the directory that contains the installed package payload. See + :doc:`here ` for more information on package structure in relation to variants. + + .. py:attribute:: this.version + :type: ~rez.rex_bindings.VersionBinding + + The package version. It can be used as a string, however you can also access specific tokens in the + version (such as major version number and so on), as this code snippet demonstrates: + + .. code-block:: python + + env.FOO_MAJOR = this.version.major # or, this.version[0] + + The available token references are ``this.version.major``, ``this.version.minor`` and + ``this.version.patch``, but you can also use a standard list index to reference any version token. + +.. py:function:: undefined(envvar: str) -> bool + + Use this boolean function to determine whether or not an environment variable is set. This is the + opposite of :func:`defined`. + + .. code-block:: python + + if undefined("REZ_MAYA_VERSION"): + info("maya is not present") + +.. py:function:: unsetenv(envvar: str) -> None + + Unsets an environment variable. This function does nothing if the environment variable was not set. + + .. code-block:: python + + unsetenv("FOO_LIC_SERVER") + +.. py:attribute:: version + :type: ~rez.rex_bindings.VersionBinding + + See :attr:`this.version`. diff --git a/docs/source/package_definition.rst b/docs/source/package_definition.rst new file mode 100644 index 000000000..566ff4278 --- /dev/null +++ b/docs/source/package_definition.rst @@ -0,0 +1,1116 @@ +================== +Package definition +================== + +Packages are defined by a *package definition file*. This is typically a file named ``package.py`` +that is located in the root directory of each package install. For example, given package +repository location :file:`/packages/inhouse`, the package definition file for package "foo-1.0.0" would +be :file:`/packages/inhouse/foo/1.0.0/package.py`. + +Here is an example package definition file: + +.. code-block:: python + + name = 'sequence' + + version = '2.1.2' + + description = 'Sequence detection library.' + + authors = ['ajohns'] + + tools = [ + 'lsq', + 'cpq' + ] + + requires = [ + 'python-2.6+<3', + 'argparse' + ] + + def commands(): + env.PATH.append("{root}/bin") + env.PYTHONPATH.append("{root}/python") + + uuid = '6c43d533-92bb-4f8b-b812-7020bf54d3f1' + +Attributes +========== + +Every variable defined in the package definition file becomes an attribute on the built or +installed package. This includes attributes that are not in the +:ref:`standard-package-attributes`. You can add any custom attribute to a package. + +Some variables are not, however, added as package attributes. Consider the following package +definition snippet: + +.. code-block:: python + + import sys + + description = "This package was built on %s" % sys.platform + +Here we do not want ``sys`` to become a package attribute, because providing a python module as a +package attribute is nonsensical. + +Python variables that do **not** become package attributes include: + +* Python modules; +* Functions, not including :ref:`early ` and :ref:`late ` + binding functions (see next), and not including the :attr:`commands` and related functions; +* Any variable with a leading double underscore; +* Any variable that is a :ref:`build-package-attributes`. + +Function Attributes +------------------- + +Package attributes can be implemented as functions. The return value of the function becomes +the attribute value. There are two types of attribute functions: *early binding* functions, +and *late binding* functions - and these are decorated using ``@early`` and ``@late`` respectively. + +.. warning:: + The :func:`commands` functions are an exception to the rule. They are + late bound, but are not the same as a standard function attribute, and are **never** decorated + with the early or late decorators. + +.. _package-definition-early-binding-functions: + +Early Binding Functions ++++++++++++++++++++++++ + +Early binding functions use the ``@early`` decorator. They are evaluated at *build time*, hence the +'early' in 'early binding'. Any package attribute can be implemented as an early binding function. + +Here is an example of an :attr:`authors` attribute that is automatically set to the contributors of the +package's git project: + +.. code-block:: python + + @early() + def authors(): + import subprocess + p = subprocess.Popen("git shortlog -sn | cut -f2", + shell=True, stdout=subprocess.PIPE) + out, _ = p.communicate() + return out.strip().split('\n') + +.. note:: + You can assume that during evaluation of early binding functions, the + current working directory is the root directory containing your ``package.py``. + +An early bound function can also have access to other package attributes. To do this, use the +implicit :attr:`this` object: + +.. code-block:: python + + @early() + def description(): + # a not very useful description + return "%s version %s" % (this.name, this.version) + +.. warning:: + Do not reference other early bound or late bound attributes in + your early bound function. An error will be raised if you do. + +Early binding functions are a convenience. You can always use an arbitrary function instead, like so: + +.. code-block:: python + + def _description(): + return "%s version %s" % (this.name, this.version) + + description = _description() + +However, using early binding results in a package definition that is cleaner and more explicit. It +is clear that an attribute is intended to be evaluated at build time, and you avoid the need to +define an arbitrary function earlier in the python source. You can always use a combination of the +two as well. An early binding function can call an arbitrary function defined at the bottom of +your definition file. + +Available Objects +***************** + +Following is the list of objects that are available during early evaluation. + +.. todo:: Document these properly with py:attribute? + +* **building**: See :attr:`building`; +* **build_variant_index**: The index of the variant currently being built. This is only relevant if + :attr:`building` is True. +* **build_variant_requires**: The subset of package requirements specific to the variant + currently being built. This is a list of ``PackageRequest`` objects. This is only relevant if + :attr:`building` is True. +* **this**: The current package, as described previously. + +Be aware that early-bound functions are actually evaluated multiple times during a build: once +pre-build, and once per variant, during its build. This is necessary in order for early-bound +functions to change their return value based on variables like ``build_variant_index``. Note that the +*pre-build* evaluated value is the one set into the installed package, and in this case, ``building`` +is False. + +An example of where you'd need to be aware of this is if you wanted the :attr:`requires` field to include +a certain package at runtime only (ie, not present during the package build). In this case, :attr:`requires` +might look like so: + +.. code-block:: python + + @early() + def requires(): + if building: + return ["python-2"] + else: + return ["runtimeonly-1.2", "python-2"] + +.. warning:: + You **must** ensure that your early-bound function returns the value + you want to see in the installed package, when ``building`` is False. + +.. _package-definition-late-binding-functions: + +Late Binding Functions +++++++++++++++++++++++ + +Late binding functions stay as functions in the installed package definition, and are only evaluated +lazily, when the attribute is accessed for the first time (the return value is then cached). + +Not any attribute can be implemented as a late binding function. The allowed attributes are: + +* requires +* build_requires +* private_build_requires +* tools +* help +* any arbitrary attribute + +Here is an example of a late binding :attr:`tools` attribute: + +.. code-block:: python + + @late() + def tools(): + import os + + # get everything in bin dir + binpath = os.path.join(this.root, "bin") + result = os.listdir(binpath) + + # we don't want artists to see the admin tools + if os.getenv("_USER_ROLE") != "superuser": + result = set(result) - set(["delete-all", "mod-things"]) + + return list(result) + +.. warning:: + Late binding function attributes **must** perform any necessary imports + **within** the function, not at the top of the ``package.py`` file. + +Note that, if this function just returned the binaries found in the bin dir, it would have made +more sense to implement this as an :ref:`early binding ` function. +No code evaluation has to happen at runtime then, so it's cheaper. However, here a modification +is made based on the value of the ``_USER_ROLE`` environment variable, which isn't known at build time. + +If some information for an attribute could be calculated once at build time, you can reduce the +runtime cost by storing that part into an early binding arbitrary attribute. For example, we could +reimplement the above example like so: + +.. code-block:: python + + @late() + def tools(): + import os + result = this._tools + + # we don't want artists to see the admin tools + if os.getenv("_USER_ROLE") != "superuser": + result = set(result) - set(["delete-all", "mod-things"]) + + return list(result) + + @early() + def _tools(): + import os + return os.listdir("./bin") + +.. todo:: Make this.root and co terms or something else like data? + +Note how in the ``_tools`` function we're referring to a relative path. Remember that early binding +functions are evaluated at build time. The package hasn't actually been built or installed yet, +so attributes such as :attr:`this.root` don't exist. + +.. _in_context: + +The in_context Function +*********************** + +When late binding functions are evaluated, a boolean function ``in_context`` is present, which +returns ``True`` if the package is part of a resolved context, or ``False`` otherwise. For example, +if you just use the rez API to iterate over packages (as the :ref:`rez-search` tool does), these +packages do not belong to a context. However if you create a :class:`~rez.resolved_context.ResolvedContext` object (as +the :ref:`rez-env` tool does) and iterate over its resolved packages, these belong to a context. + +The in-context or not-in-context distinction is important, because often the package attribute +will need information from the context to give desired behavior. For example, consider the +late binding :attr:`tools` attribute below: + +.. code-block:: python + + @late() + def tools(): + result = ["edit"] + + if in_context() and "maya" in request: + result.append("maya-edit") + + return result + +Here the :attr:`request` object is being checked to see if the ``maya`` package was requested in the +current env; if it was, a maya-specific tool ``maya-edit`` is added to the tool list. + +.. warning:: + Always ensure your late binding function returns a sensible + value regardless of whether :ref:`in_context ` is ``True`` or ``False``. + Otherwise, simply trying to query the package attributes (using :ref:`rez-search` for example) + may cause errors. + +Available Objects +***************** + +Following is the list of objects that are available during late evaluation, if :ref:`in_context ` +is ``True``: + +* **context**: the :class:`~rez.resolved_context.ResolvedContext` instance this package belongs to; +* **system**: see :attr:`system`; +* **building**: see :attr:`building`; +* **request**: see :attr:`request`; +* **implicits**: see :attr:`implicits`. + +The following objects are available in **all** cases: + +* :attr:`this`: the current package/variant (see note below); +* **in_context**: the :ref:`in_context ` function itself. + +.. warning:: + The :attr:`this` object may be either a package or a variant, + depending on the situation. For example, if :ref:`in_context ` is ``True``, + then :attr:`this` is a variant, because variants are the objects present in a resolved context. On the other + hand, if a package is accessed via API (for example, by using the :ref:`rez-search` tool), + then :attr:`this` may be a package. The difference matters, because variants have some + attributes that packages don't, notably, ``root`` and ``index``. Use the properties + :attr:`this.is_package` and :attr:`this.is_variant` to distinguish the case if needed. + +Example - Late Bound build_requires +*********************************** + +Here is an example of a ``package.py`` with a late-bound :attr:`build_requires` field: + +.. code-block:: python + + name = "maya_thing" + + version = "1.0.0" + + variants = [ + ["maya-2017"], + ["maya-2018"] + ] + + @late() + def build_requires(): + if this.is_package: + return [] + elif this.index == 0: + return ["maya_2017_build_utils"] + else: + return ["maya_2018_build_utils"] + +.. todo:: Figure out why I can't link to this.is_package + +Note the check for :attr:`this.is_package`. This is necessary, otherwise the evaluation would +fail in some circumstances. Specifically, if someone ran the following command, the :attr:`this` +field would actually be a :class:`.Package` instance, which doesn't have an ``index`` method: + +.. code-block:: text + + ]$ rez-search maya_thing --type package --format '{build_requires}' + +In this case, :attr:`build_requires` is somewhat nonsensical (there is no common build requirement +for both variants here), but something needs to be returned nonetheless. + +.. _package-definition-sharing-code: + +Sharing Code Across Package Definition Files +============================================ + +It is possible to share common code across package definition function attributes, but the +mechanism that is used is different depending on whether a function is early binding or late +binding. This is to avoid installed packages being dependent on external code that may change +at any time; builds being dependent on external code is not problematic however. + +Sharing Code During A Build +--------------------------- + +Functions in a ``package.py`` file which are evaluated at build time include: + +* The :attr:`preprocess` function; +* Any package attribute implemented as a function using the :ref:`@early ` decorator. + +You expose common code to these functions by using the +:data:`package_definition_build_python_paths` config setting. + +Sharing Code Across Installed Packages +-------------------------------------- + +Functions that are evaluated in installed packages' definition files include: + +.. todo:: Group all commands in one section? + +* The various :doc:`commands ` functions; +* Any package attribute implemented as a function using the :ref:`@late ` decorator. + +You expose common code to these functions by using the ``@include`` decorator, which relies on the +:data:`package_definition_python_path` config setting. +The module source files are actually copied into each package's install payload, so the package +stays self-contained, and will not break or change behavior if the original modules' source +files are changed. The downside though, is that these modules are not imported, and they themselves +cannot import other modules managed in the same way. + +Here is an example of a package's :attr:`commands` using a shared module: + +.. code-block:: python + + # in package.py + @include("utils") + def commands(): + utils.set_common_env_vars(this, env) + +.. _requirements-expansion: + +Requirements Expansion +====================== + +Often a package may be compatible with a broader range of its dependencies at build time than it is +at runtime. For example, a C++ package may build against any version of ``boost-1``, but may +then need to link to the specific minor version that it was built against, say ``boost-1.55``. + +You can describe this in your package's :attr:`requires` attribute (or any of the related attributes, +such as :attr:`build_requires`) by using wildcards as shown here: + +.. code-block:: python + + requires = [ + "boost-1.*" + ] + +If you check the ``package.py`` of the built package, you will see that the boost reference in the +requires list will be expanded to the latest found within the given range (``boost-1.55`` for example). + +There is also a special wilcard available, ``**``. This expands to the full package version. For +example, the requirement ``boost-1.**`` might expand to ``boost-1.55.1``. + +You can also achieve requirements expansion by implementing :attr:`requires` as an early binding +function (and you may want to use some variation of this to generate :attr:`variants` for example), and +using the rez :func:`~rez.package_py_utils.expand_requires` function: + +.. code-block:: python + + @early() + def requires(): + from rez.package_py_utils import expand_requires + return expand_requires(["boost-1.*"]) + +.. _preprocess: + +.. _package-preprocessing: + +Package Preprocessing +===================== + +You can define a :func:`preprocess` function either globally or in a ``package.py``. This can be used to +validate a package, or even change some of its attributes, before it is built. To set a global +preprocessing function, see the :data:`package_preprocess_function` config setting. + +Consider the following preprocessing function, defined in a ``package.py``: + +.. code-block:: python + + def preprocess(this, data): + from rez.package_py_utils import InvalidPackageError + import re + + if not re.match("[a-z]+$", this.name): + raise InvalidPackageError("Invalid name, only lowercase letters allowed") + + if not this.authors: + from preprocess_utils import get_git_committers + data["authors"] = get_git_committers() + +This preprocessor checks the package name against a regex and sets the package authors list to its +git committers, if not already supplied in the ``package.py``. To update package attributes, you have +to update the given ``data`` dict, **not** the package instance (:attr:`this`). + +To halt a build because a package is not valid, you must raise an :exc:`~rez.exceptions.InvalidPackageError` as shown +above. + +.. hint:: + To see the preprocessed contents of a package.py, run the command + ``rez-build --view-pre`` in the source root directory. This will just print the preprocessed + package to standard out, then exit. + +Overriding Config Settings In Preprocessing +------------------------------------------- + +It is not uncommon to override config settings such as the release path in a package, like so: + +.. code-block:: python + + # in package.py + with scope("config") as c: + c.release_packages_path = "/software/packages/external" + +Let's say we have a scenario where we want to install third party packages to a specific install +path, and that we set the arbitrary attribute ``external`` to ``True`` for these packages. We could do +this with a global preprocessing function like this: + +.. code-block:: python + + def preprocess(this, data): + if not data.get("external"): + return + + try: + _ = data["config"]["release_packages_path"] + return # already explicitly specified by package + except KeyError: + pass + + data["config"] = data.get("config", {}) + data["config"]["release_packages_path"] = "/software/packages/external" + +The ``with scope(...)`` statement is just a fancy way of defining a dict, so you can do the same +thing in the preprocess function simply by updating the ``config`` dict within ``data``. + +See :ref:`configuring-rez-package-overrides` for more details on the ``scope`` function. + +Example Package +=============== + +Here is an example package definition, demonstrating several features. This is an example of a +python package which, instead of actually installing python, detects the existing system python +installation instead, and binds that into a rez package. + +.. code-block:: python + + name = "python" + + @early() + def version(): + return this.__version + "-detected" + + authors = [ + "Guido van Rossum" + ] + + description = \ + """ + The Python programming language. + """ + + @early() + def variants(): + from rez.package_py_utils import expand_requires + requires = ["platform-**", "arch-**", "os-**"] + return [expand_requires(*requires)] + + @early() + def tools(): + version_parts = this.__version.split('.') + + return [ + "2to3", + "pydoc", + "python", + "python%s" % (version_parts[0]), + "python%s.%s" % (version_parts[0], version_parts[1]) + ] + + uuid = "recipes.python" + + def commands(): + env.PATH.append("{this._bin_path}") + + if building: + env.CMAKE_MODULE_PATH.append("{root}/cmake") + + # --- internals + + def _exec_python(attr, src): + import subprocess + + p = subprocess.Popen( + ["python", "-c", src], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + + if p.returncode: + from rez.exceptions import InvalidPackageError + raise InvalidPackageError( + "Error determining package attribute '%s':\n%s" % (attr, err)) + + return out.strip() + + @early() + def _bin_path(): + return this._exec_python( + "_bin_path", + "import sys, os.path; print(os.path.dirname(sys.executable))") + + def _version(): + return _exec_python( + "version", + "import sys; print(sys.version.split()[0])") + + __version = _version() + +Note the following: + +.. todo:: Document which attributes supports automatic wildcard expansion? + +* :attr:`variants` is implemented as an early bound attribute, and uses :ref:`requirements-expansion` to + dynamically define the variant requirements. Even though only the :attr:`requires` and related attributes + natively expand wildcards, you can still use the :func:`~rez.package_py_utils.expand_requires` function + yourself, as illustrated here. +* A ``_version`` function has been defined, and its return value stored into the ``__version`` variable. + This is done because two other early binding attributes. :attr:`version` and :attr:`tools` use this value, + and we avoid calling the function twice. Both ``_version`` and ``__version`` are later stripped from + the package, because one is a normal function, and the other has double leading underscores. +* An arbitrary attribute ``_bin_path`` has been defined, and implemented as an early bound attribute. + The :attr:`commands` function then uses this value. In this example, it was far better to take this + approach than the alternative of running the python subprocess in the :attr:`commands` function. Doing that + would have been very costly, since commands are executed every time a new environment is created + (and launching a subprocess is slow). Instead, here we take this cost at build time, and cache the + result into the package attribute. +* Common code was provided in the normal function ``_exec_python``, which will be stripped from the + installed package. + +.. _package-attributes: + +Package Attributes +================== + +.. _standard-package-attributes: + +Standard Package Attributes +--------------------------- + +Following is a list, in alphabetical order, of every standard attribute that a user can define in a +package definition file (you can also define your own arbitrary attributes). Each entry specifies +the data type, and includes a code snippet. + +.. .. currentmodule:: pkgdef + +.. py:attribute:: authors + :type: list[str] + + Package authors. Should be in order, starting with the major contributor. + + .. code-block:: python + + authors = ["jchrist", "sclaus"] + +.. py:attribute:: build_requires + :type: list[str] + + This is the same as :attr:`requires`, except that these dependencies are only included during a build + (typically invoked using the :ref:`rez-build` tool). + + .. code-block:: python + + build_requires = [ + "cmake-2.8", + "doxygen" + ] + +.. py:attribute:: cachable + :type: bool + + Determines whether a package can be cached when :ref:`package-caching` is enabled. + If not provided, this is determined from the global config setting :data:`default_cachable` and related ``default_cachable_*`` settings. + + .. code-block:: python + + cachable = True + +.. py:function:: commands() -> None + + This is a block of python code which tells rez how to update an environment so that this package + can be used. It is executed when the package is brought into a rez environment, either by explicit + request or by another package's requirements. There is a python API provided (see + :doc:`package_commands` for more details) that lets you do things such as: + + * set, unset, prepend and append environment variables; + * create aliases; + * source scripts; + * print messages. + + In this example, the ``foo`` package is appending a path to ``PYTHONPATH``, and appending a path to + ``PATH``. The special string ``{root}`` will expand out to the install location of the package (see :ref:`string-expansion`). + This is a fairly typical example. + + .. code-block:: python + + def commands(): + env.PYTHONPATH.append("{root}/python") + env.PATH.append("{root}/bin") + +.. py:attribute:: config + :type: dict[str, typing.Any] + + Packages are able to override rez configuration settings. This is useful in some cases. For example, + we may want a package to release to a different directory than the default (as this example shows). + See :ref:`here ` for more details. + + .. note:: + ``config`` should not be modified as is. You need to use the ``scope`` function to manipulate it. + + .. code-block:: python + + with scope("config"): + release_packages_path = "/software/packages/apps" + +.. py:attribute:: description + :type: str + + This is a general description of the package. It should not mention details about a particular + version of the package, just about the package in general. + + .. code-block:: python + + description = "Library for communicating with the dead." + +.. py:attribute:: has_plugins + :type: bool + + Indicates that the package is an application that may have plugins. These plugins are often made + available as rez packages also. Used in conjuction with the :ref:`rez-plugins` command. Also, see :attr:`plugin_for`. + + .. code-block:: python + + has_plugins = True + +.. py:attribute:: hashed_variants + :type: bool + + Instructs the package to install variants into a subdirectory based on a hash of the variant's + contents (its requirements in other words). This is useful for variants with a high number of + requirements, or with requirements that do not translate well to directories on the filesystem + (such as conflict requirements). + + .. code-block:: python + + hashed_variants = True + +.. py:attribute:: help + :type: str | list[list[str]] + + URL for package webpage, or, if a string containing spaces, a command to run. You can show the help + for a package using the :ref:`rez-help` command line tool. If this value is a list of list, then this + represents multiple help entries. + + .. code-block:: python + + help = "https://github.com/__GITHUB_REPO__/wiki" + + .. code-block:: + + help = [ + ['Documentation', 'https://example.com/docs'], + ['API docs', 'https://example.com/docs/api'] + ] + +.. py:attribute:: name + :type: str + + **Mandatory** + + This is the name of the package. Alphanumerics and underscores are allowed. Name is case sensitive. + + .. code-block:: python + + name = "maya_utils" + +.. py:attribute:: plugin_for + :type: str + + Provided if this package is a plugin of another package. For example, this might be a maya plugin. + This is useful when using the :ref:`rez-plugins` command. Also, see :attr:`has_plugins`. + + .. code-block:: python + + plugin_for = "maya" + +.. py:function:: post_commands() -> None + + Similar to :func:`pre_commands`, but runs in a final phase rather than the first. See that attribute for + further details. + + .. code-block:: python + + def post_commands(): + env.FOO_PLUGIN_PATH.append("@") + +.. py:function:: pre_commands() -> None + + This is the same as :func:`commands`, except that all packages' ``pre_commands`` are executed in a first + pass; then, all ``commands`` are run in a second; and lastly, ``post_commands`` are all run in a third + phase. It is sometimes useful to ensure that some of a package's commands are run before, or after + all others, and using pre/post_commands is a way of doing that. + + .. code-block:: python + + def pre_commands(): + import os.path + env.FOO_PLUGIN_PATH = os.path.join(this.root, "plugins") + +.. py:function:: pre_test_commands() + + This is similar to :func:`commands`, except that it is run prior to each test defined in + :attr:`tests`. See :ref:`pre-test-commands` for more details. + + .. code-block:: python + + def pre_test_commands(): + if test.name == "unit": + env.IS_UNIT_TEST = 1 + +.. py:attribute:: relocatable + :type: bool + + Determines whether a package can be copied to another package repository (using the :ref:`rez-cp` tool for + example). If not provided, this is determined from the global config setting :data:`default_relocatable` and + related ``default_relocatable_*`` settings. + + .. code-block:: python + + relocatable = True + +.. py:attribute:: requires + :type: list[str] + + This is a list of other packages that this package depends on. A rez package should list all the + packages it needs. Someone should be able to use your package without needing to know about how it + works internally and this includes needing to know its dependencies. + + Rez has a syntax for these package requests. For example, ``python-2.6`` is a package request which + covers the range of all python packages starting with 2.6, for example, ``python-2.6.0``, + ``python-2.6.4`` (it is not simply a prefix. ``python-2.65`` is not within the request). When you + request a package, you are asking rez for any version within this request, although rez will aim to + give you the latest possible version. + + .. hint:: For more details on request syntax, see :ref:`package-requests-concept`. + + .. code-block:: python + + requires = [ + "python-2", + "maya-2016", + "maya_utils-3.4+<4" + ] + +.. py:attribute:: tests + :type: dict[str, str | dict] + + This is a dict of tests that can be run on the package using the :ref:`rez-test` tool. + + If a test entry is a string or list of strings, this is interpreted as the command to run. Command + strings will expand any references to package attributes, such as ``{root}``. + + If you provide a nested dict, you can specify extra fields per test, as follows: + + * ``requires``: Extra package requirements to include in the test's runtime env. + * ``run_on``: When to run this test. Valid values are: + * ``default`` (the default): Run when :ref:`rez-test` is run with test name (ie ``rez-test ``). + * ``pre_install``: Run before an install (ie :option:`rez-build -i`), and abort the install on fail. + * ``pre_release``: Run before a release, and abort the release on fail. + * ``explicit``: Only run if specified when :ref:`rez-test` is run (ie ``rez-test ``). + * ``on_variants``: Which variants the test should be run on. Valid values are: + * ``True``: Run the test on all variants. + * ``False`` (the default): Run the test only on one variant (ie the variant you get by + default when the test env is resolved). This is useful for tests like linting, + where variants may be irrelevant. + * A dict: This is a variant selection mechanism. In the example below, the ``maya_CI`` test will + run only on those variants that directly require ``maya`` (or a package within this range, eg + ``maya-2019``). Note that ``requires`` is the only filter type currently available. + + .. code-block:: python + + tests = { + "unit": "python -m unittest discover -s {root}/python/tests", + "lint": { + "command": "pylint mymodule", + "requires": ["pylint"], + "run_on": ["default", "pre_release"] + }, + "maya_CI": { + "command": "python {root}/ci_tests/maya.py", + "on_variants": { + "type": "requires", + "value": ["maya"] + }, + "run_on": "explicit" + } + } + + As an example, if you want to run the ``maya_CI`` block defined in the example above (named ``maya_utils``), you can run: + + .. code-block:: text + + ]$ rez-test maya_utils lint + + .. note:: + Prior to running the tests, you will need to run :ref:`rez-build`. :ref:`rez-test` can only + run tests on already built packages. + +.. py:attribute:: tools + :type: list[str] + + This is a list of tools that the package provides. This entry is important later on when we talk + about :ref:`suite tools `. + + .. code-block:: python + + tools = [ + "houdini", + "hescape", + "hython" + ] + +.. py:attribute:: uuid + :type: str + + This string should uniquely identify this *package family*. In other words, all the versions of a + particular package, such as ``maya``. It is used to detect the case where two unrelated packages that + happen to have the same name are attempted to be released. If rez detects a uuid mismatch, it will + abort the release. + + You should set the uuid on a new package once, and not change it from then on. The format of the + string doesn't actually matter, but you'd typically use a true UUID, and you can generate one + like so: + + .. code-block:: text + + ]$ python -c 'import uuid; print(uuid.uuid4().hex)' + + Example: + + .. code-block:: python + + uuid = "489ad32867494baab7e5be3e462473c6" + +.. py:attribute:: variants + :type: list[list[str]] + + A package can contain *variants* - think of them as different flavors of the same package version, + but with differing dependencies. See the :doc:`variants` section for further details. + + .. code-block:: python + + variants = [ + ["maya-2015.3"], + ["maya-2016.1"], + ["maya-2016.7"] + ] + +.. py:attribute:: version + :type: str + + This is the version of the package. See :ref:`versions-concept` for further details on valid + package versions. + + .. code-block:: python + + version = "1.0.0" + +.. _build-package-attributes: + +Build Time Package Attributes +----------------------------- + +The following package attributes only appear in packages to be built; they are stripped from the +package once installed because they are only used at build time. + +.. py:attribute:: build_command + :type: str | list[str] | False + + Package build command. If present, this is used as the build command when :ref:`rez-build` is run, + rather than detecting the build system from present build scripts (such as ``CMakeLists.txt``). If + ``False``, this indicates that no build step is necessary (the package definition will still be + installed, and this is enough to define the package). + + The ``{root}`` string expands to the root directory of the package (where the ``package.py`` is + contained). Note that, like all builds, the working directory is set to the *build path*, which + is typically somewhere under a *build* subdirectory, and is where build outputs should go. + + The ``{install}`` string expands to ``install`` if an installation is occurring, or the empty string + otherwise. This is useful for passing the install target directly to the command (for example, when + using ``make``) rather than relying on a build script checking the :envvar:`REZ_BUILD_INSTALL` environment + variable. + + The full set of variables that can be referenced in the build command are: + + * ``root``: (see above); + * ``install``: (see above) + * ``build_path``: The build path (this will also be the current working directory); + * ``install_path``: Full path to install destination; + * ``name``: Name of the package getting built; + * ``variant_index``: Index of the current variant getting built, or an empty + string ('') if no variants are present. + * ``version``: Package version currently getting built. + + .. code-block:: python + + build_command = "bash {root}/build.sh {install}" + +.. py:attribute:: build_system + :type: str + + .. todo:: reference the real --build-system cli flag + + Specify the build system used to build this package. If not set, it is detected automatically when + a build occurs (or the user specifies if using :option:`rez-build --build-system` option). + + .. code-block:: python + + build_system = "cmake" + + +.. py:function:: pre_build_commands() -> None + + This is similar to :func:`commands`, except that it is run *prior to the current package being built*. + See :ref:`pre-build-commands` for more details. + + .. code-block:: python + + def pre_build_commands(): + env.FOO_BUILT_BY_REZ = 1 + +.. py:function:: preprocess(this, data: dict[str, typing.Any]) + + See :ref:`package-preprocessing`. + +.. py:attribute:: private_build_requires + :type: list[str] + + This is the same as :attr:`build_requires`, except that these dependencies are only included if this + package is being built. Contrast this with :attr:`build_requires`, whose dependencies are included if a + build is occurring regardless of whether this package specifically is being built, or whether + this package is a dependency of the package being built. + + .. code-block:: python + + private_build_requires = [ + "cmake-2.8", + "doxygen" + ] + +.. py:attribute:: requires_rez_version + :type: str + + This defines the minimum version of rez needed to build this package. New package features have + been added over time, so older rez versions cannot necessarily build newer packages. + + .. code-block:: python + + requires_rez_version = "2.10" + +.. _release-package-attributes: + +Release Time Package Attributes +------------------------------- + +The following package attributes are created for you by Rez when your package is released via the +:ref:`rez-release` tool. If you look at the released ``package.py`` file you will notice that some or all +of these attributes have been added. + +.. py:attribute:: changelog + :type: str + + Change log containing all commits since the last released package. If the previous release was from + a different branch, the changelog given will go back to the last common commit ancestor. The syntax + of this changelog depends on the version control system. The example here is from a *git*-based + package. + + .. code-block:: python + + changelog = \ + """ + commit 22abe31541ceebced8d4e209e3f6c44d8d0bea1c + Author: allan johns <> + Date: Sun May 15 15:39:10 2016 -0700 + + first commit + """ + +.. py:attribute:: previous_revision + :type: typing.Any + + Revision information of the previously released package, if any (see :attr:`revision` for code example - + the code for this attribute is the same). + +.. py:attribute:: previous_version + :type: str + + The version of the package previously released, if any. + + .. code-block:: python + + previous_version = "1.0.1" + +.. py:attribute:: release_message + :type: str + + .. todo:: Reference --message option directly + + .. todo:: How should we document and link plugin settings? Like TODO_ADD_THIS. + + The package release message. This is supplied either via the :option:`rez-release --message` + option, or was entered in a text editor on release if rez is configured to do this (see the config + setting ``TODO_ADD_THIS``). A package may not have a release message. + + .. code-block:: python + + release_message = "Fixed the flickering thingo" + +.. py:attribute:: revision + :type: typing.Any + + Information about the source control revision containing the source code that was released. The + data type is determined by the version control system plugin that was used. The example code shown + here is the revision dict from a *git*-based package. + + .. code-block:: python + + revision = \ + {'branch': 'master', + 'commit': '22abe31541ceebced8d4e209e3f6c44d8d0bea1c', + 'fetch_url': 'git@github.com:foo/dummy.git', + 'push_url': 'git@github.com:foo/dummy.git', + 'tracking_branch': 'origin/master'} + +.. py:attribute:: timestamp + :type: int + + Epoch time at which the package was released. + + .. code-block:: python + + timestamp = 1463350552 + +.. py:attribute:: vcs + :type: str + + Name of the version control system this package was released from. + + .. code-block:: python + + vcs = "git" diff --git a/docs/source/pip.rst b/docs/source/pip.rst new file mode 100644 index 000000000..6cde13a78 --- /dev/null +++ b/docs/source/pip.rst @@ -0,0 +1,143 @@ +=== +Pip +=== + +Rez is language agnostic. +But since python is so much used in the VFX industry (and outside), +it knows how to convert/translate it into a rez package. +To do so, it provides a :ref:`rez-pip` command. + +.. warning:: + It doesn't know how to translate its own packages into pip packages. + +.. hint:: + We are planning to introduce a new rez-pip implemented as a plugin that will + overcome most of the current rez-pip limitations and annoyances. + +Usage +===== + +.. code-block:: text + + usage: rez pip [-h] [--python-version VERSION] [--pip-version VERSION] [-i] + [-s] [-r] [-v] + PACKAGE + + Install a pip-compatible python package, and its dependencies, as rez + packages. + + positional arguments: + PACKAGE package to install or archive/url to install from + + optional arguments: + -h, --help show this help message and exit + --python-version VERSION + python version (rez package) to use, default is + latest. Note that the pip package(s) will be installed + with a dependency on python-MAJOR.MINOR. + --pip-version VERSION + pip version (rez package) to use, default is latest. + This option is deprecated and will be removed in the + future. + -i, --install install the package + -s, --search search for the package on PyPi + -r, --release install as released package; if not set, package is + installed locally only + -p PATH, --prefix PATH + install to a custom package repository path. + -v, --verbose verbose mode, repeat for more verbosity + + +The :ref:`rez-pip` command allows you to do two main things. + +1. Install or release a pip package as a rez package. +2. Search for a package on PyPI + +.. _which-pip-will-be-used: + +Which pip will be used? +======================= + +:ref:`rez-pip` uses a fallback mechanism to find which pip will be used to run the commands. +The logic is as follow: + +1. Search for pip in the rezified ``python`` package specified with :option:`--python-version `, or + the latest version if not specified; +2. If found, use it; +3. If not found, search for pip in the rezified ``pip`` package specified with :option:`--pip-version `, + or latest version if not specified. + + .. note:: + Note that this is deprecated and will be removed in the future + +4. If rezified ``pip`` is found, use it; +5. If not found, fall back to pip installed in rez own virtualenv. + +.. note:: + In all of these options, we also check if the version of pip is greater or equal + than 19.0. This is a hard requirement of :ref:`rez-pip`. + +Note that rez-pip output should tell you what it tries and which pip it will use. + +It is extremely important to know which pip is used and understand why it is used. Pip packages +define which python version they are compatible with. +When you install a pip package, pip checks which python version it is +currently running under to determine if a package can be downloaded and installed. +Not only this but it also checks which python implementation is used (CPython, PyPy, +IronPython, etc), the architecture python was built with, and other variables. So the thing you +really need to know first is which python you want to use and from there you should know +which pip is used. Knowing the pip version comes in second place. + +At some point, we supported the :option:`--pip-version ` argument, but considering what was just said +above, we decided to deprecate it (but not yet removed) just for backward compatibility reasons. +Pip is too much (read tightly) coupled to the python version/interpreter it is installed with +for us to support having pip as a rez package. We just can't guarantee that pip can be +install once in a central way and work with multiple different python version, and potentially +different implementations. + +.. _how-should-i-install-pip: + +How should I install pip? +========================= + +Following the :ref:`which-pip-will-be-used` section, we recommend to install +pip inside your python packages. For Python 2, this can be done when you compile it with the +``--with-ensurepip`` flag of the ``configure`` script. This will install a version older than 19.0 +though, so you will need to upgrade it. For Python 3, it is already installed by default. +Though your mileage may vary for the version installed, depending on which Python version you +installed. So check the pip version and update it if necessary. We also encourage you +to install ``wheel`` and possibly update ``setuptools``. ``pip``, ``setuptools`` and ``wheel`` +are perfectly fine when installed in the interpreter directly as they are pretty core +packages and all have no dependencies (and that's what ``virtualenv`` does by default too). + +.. tip:: + When installing something in an interpreter, make sure you really install in this interpreter. + That means using something like: + + .. code-block:: console + + $ /path/to/python -E -s -m pip install + + ``-E`` will render any ``PYTHON*`` environment variable to not be used and ``-s`` will + remove your :mod:`user site ` from the equation. + +Install/release +--------------- + +You have two options when you want to convert a pip package to a rez package. You can +install it, or release it. Install means that it will install in your +:data:`local_packages_path`, while +release means it will be installed in your :data:`release_packages_path`. +You can also specify a custom installation location using :option:`--prefix ` (or ``-p``). + +You can (and we recommend) use :option:`--python-version ` to choose for which python +version you want to install a given package. This will make ``rez-pip`` to resolve +the given version of the ``python`` rez package and use it to run the ``pip install``. +See :ref:`which-pip-will-be-used` for more details. +If the pip package is not pure (so contains ``.so`` for example), you will need to +call :ref:`rez-pip` for each python version you want to install the pip package for. + +.. warning:: + :option:`--pip-version ` is deprecated and will be removed in the future. + See :ref:`how-should-i-install-pip` on how we recommend + to install pip from now on. diff --git a/docs/source/suites.rst b/docs/source/suites.rst new file mode 100644 index 000000000..ea1e8cb54 --- /dev/null +++ b/docs/source/suites.rst @@ -0,0 +1,180 @@ +====== +Suites +====== + +Let us say that you wish to provide a number of different tools to your users, +even though these tools may require being run in different environments. For +example, you may want artists at your studio to be able to run ``maya`` and ``nuke`` +from the command line, without needing to know that they execute within different +environments. + +Let's say that in order to do this, you create two :doc:`contexts `: ``maya.rxt`` and +``nuke.rxt`` (see :ref:`here ` for more information). In +order to run maya, you would do this: + +.. code-block:: console + + $ rez-env --input maya.rxt -- maya + +You may then decide to wrap this command within a wrapper script which is also +called ``maya``, and that might look something like this: + +.. code-block:: bash + + #!/bin/bash + rez-env --input maya.rxt -- maya $* + +Now, if you put this somewhere on ``$PATH``, and do the same for ``nuke``, then +voila, your artists can run these applications from the command line, without +needing to know what's happening under the hood. + +This, in a nutshell, is what a *suite* does. A suite is simply a directory +containing a set of contexts, and wrapper scripts which run tools within those +contexts. + +The rez-suite Tool +================== + +Let's go through the same example, this time using the :ref:`rez-suite` tool. + +First, we create the suite. This creates a directory called ``mysuite`` in the +current working directory: + +.. code-block:: console + + $ rez-suite --create mysuite + +Now we need to add contexts to our suite. First we create the contexts: + +.. code-block:: console + + $ rez-env maya-2016.2 --output maya.rxt + $ rez-env nuke --output nuke.rxt + +Then, we add these contexts to the suite (note that the :option:`rez-suite --context` arg just +gives each context a label. You would typically have this match the context +filename as shown). + +.. code-block:: console + + $ rez-suite --add maya.rxt --context maya mysuite + $ rez-suite --add nuke.rxt --context nuke mysuite + +The suite is created! Now all we need to do is to activate it, and that's as +simple as adding its ``bin`` path to ``$PATH``: + +.. code-block:: console + + $ export PATH=$(pwd)/mysuite/bin:$PATH + +You should now see your tools coming from the suite: + +.. code-block:: console + + $ which maya + ./mysuite/bin/maya + + $ ls ./mysuite/bin + maya + nuke + +.. _suite-tools: + +Suite Tools +=========== + +The tools in a context which are exposed by the suite is determined by the +:attr:`tools` package attribute. For example, the +``maya`` package might have a :attr:`tools` definition like so: + +.. code-block:: python + + # in maya package.py + tools = [ + "maya", + "mayapy", + "fcheck" + ] + +All these tools would be made available in the suite (although you can explicitly +hide tools by using :option:`rez-suite --hide` argument). + +.. warning:: + Only packages listed in the context *requests*, + that are not weak or conflict requests, have their tools exposed. Packages + pulled in as dependencies do not. If you need to control the version of a package + not in the request, without adding its command line tools, just add it as a weak + reference to the request list. + +Tool Aliasing +------------- + +Tools can be aliased to different names, either explicitly (on a per-tool basis), +or by applying a common prefix or suffix to all tools in a context. + +Prefixing/suffixing is particularly useful when you want to expose the same +package's tools, but in two or more contexts. For example, you may want to run a +stable version of maya, but also a newer beta version. These would run in +different contexts, and the beta context might prefix all tools with ``_beta``, +hence making available tools such as ``maya_beta``. + +For example, here we create a context with a newer version of maya, add it to +the suite, then add a suffix to all its tools: + +.. code-block:: console + + $ rez-env maya-2017 --output maya2017.rxt + $ rez-suite --add maya2017.rxt --context maya2017 mysuite + $ rez-suite --suffix _beta --context maya2017 mysuite + +Control Arguments +----------------- + +When using suite tools, any arguments passed to the wrappers are passed through +to the underlying tool, as expected. However, there is an exception to the case. +Rez provides a set of *control* arguments, which are prefixed with ``+``/``++`` +rather than the typical ``-``/``--``. These are suite-aware arguments that pass +directly to rez. You can get a listing of them using ``+h``/``++help``, like so: + +.. code-block:: console + + $ maya ++help + usage: maya [+h] [+a] [+i] [+p [PKG [PKG ...]]] [++versions] + [++command COMMAND [ARG ...]] [++stdin] [++strict] [++nl] + [++peek] [++verbose] [++quiet] [++no-rez-args] + + optional arguments: + +h, ++help show this help message and exit + +a, ++about print information about the tool + +i, ++interactive launch an interactive shell within the tool's + configured environment + +p [PKG [PKG ...]], ++patch [PKG [PKG ...]] + run the tool in a patched environment + ++versions list versions of package providing this tool + ++command COMMAND [ARG ...] + read commands from string, rather than executing the + tool + ++stdin read commands from standard input, rather than + executing the tool + ++strict strict patching. Ignored if ++patch is not present + ++nl, ++no-local don't load local packages when patching + ++peek diff against the tool's context and a re-resolved copy + - this shows how 'stale' the context is + ++verbose verbose mode, repeat for more verbosity + ++quiet hide welcome message when entering interactive mode + ++no-rez-args pass all args to the tool, even if they start with '+' + +For example, to see information about the suite wrapper: + +.. code-block:: console + + $ maya ++about + Tool: maya + Path: ./mysuite/bin/maya + Suite: ./mysuite + Context: ./mysuite/contexts/maya2016.rxt ('maya2016') + +.. tip:: + If the target tool also uses ``+`` for some of its + own arguments, you can change the prefix character that rez uses for its + control arguments. See the :option:`rez-suite --prefix-char` option. diff --git a/docs/source/variants.rst b/docs/source/variants.rst new file mode 100644 index 000000000..9b2bc9e45 --- /dev/null +++ b/docs/source/variants.rst @@ -0,0 +1,261 @@ +======== +Variants +======== + +Packages in rez can contain different *variants*. Think of these as different +flavors of the same package version. Each variant has one or more package +dependencies that differ to the other variants in the same package. + +Use of variants is best illustrated with an example. Consider a maya plugin, +``my_maya_plugin``. Let us assume that there are two active versions of maya +currently in use at your studio: ``2016.sp2`` and ``2017``. If your plugin is compiled, +you may need to build it separately for each maya version, even though the source +is no different. + +You would use *variants* to create a version of your plugin that will work with +both maya versions. Consider the following package definition for our plugin: + +.. code-block:: python + + name = "my_maya_plugin" + + version = "1.0.0" + + requires = [ + "openexr-2.2" + ] + + variants = [ + ["maya-2016.sp2"], + ["maya-2017"] + ] + +When you build and install this package, two separate builds will occur: one +using ``maya-2016.sp2``, and the other ``maya-2017``. When an environment is resolved +that includes ``my_maya_plugin``, the correct variant will be selected depending on +the version of maya present. Only one variant of a package is ever used in a +given configured environment. + +Each variant entry is a list of dependencies, no different to the packages listed +in the :attr:`requires` field. These dependencies are appended to the ``requires`` list +for each variant. Thus the first variant requires ``openexr-2.2`` and ``maya-2016.sp1``, +and the second variant requires ``openexr-2.2`` and ``maya-2017``. + +.. _variants-disk-structure: + +Disk Structure +============== + +Package variants are stored within the package, under subdirectories that match +the variant requirements. For example, continuing on with our ``my_maya_plugin`` +package, the installation of that package would look like so: + +.. code-block:: text + + /rez/packages/my_maya_plugin/1.0.0/maya-2016.sp2/ + /maya-2017/ + +The anatomy of a package with variants is illustrated in the following diagram: + +.. image:: _static/pkg_path_anatomy.png + :align: center + :class: rez-diagram + +The ``root`` of a package is the root directory of its current variant (the one +the current environment is configured to use); the ``base`` of a package is the +directory containing its variants. In a package that does not have variants, +``base`` and ``root`` are the same. + +Hashed Variants +=============== + +There are two problems with the variant subpath as illustrated above: + +* The variant install path can become long if there are many requirements; +* If some variant requirements contain characters such as ``!`` and ``<``, they + can cause escaping problems that affect build systems; and, depending on the + platform, may not be a valid filesystem path. + +You can avoid these issues by using :attr:`hashed_variants`. This sets the variant +sub-path to a hash of its requirements, rather than the requirements themselves. +The resulting sub-directory is somewhat unwieldy (example: +``83e0c415db1b602f9d59cee028da6ac785e9bacc``). However, another feature, +:data:`variant shortlinks `, deals with this. A shortlink is a symlink to each variant, +created in a separate subdirectory (default ``_v``). + +Here is an example hashed variant path: + +.. code-block:: text + + /rez/packages/my_maya_plugin/1.0.0/83e0c415db1b602f9d59cee028da6ac785e9bacc + +Here is the matching *shortlink*, which is what will be used in a resolved +environment: + +.. code-block:: text + + /rez/packages/my_maya_plugin/1.0.0/_v/a + +Hashed variants must be enabled explicitly for a package. To do this, simply set +this in your package definition: + +.. code-block:: python + + hashed_variants = True + +Platform As Variant +=================== + +It is not uncommon to see the platform, architecture and/or operating system +packages in package variants (recall that rez represents these as packages). For +example, you might see variants like this: + +.. code-block:: python + + # in package.py + variants = [ + ["platform-linux", "arch-x86_64", "os-Ubuntu-12.04"], + ["platform-linux", "arch-x86_64", "os-Ubuntu-16.04"] + ] + +This indicates that the package has been built for multiple platforms. The correct +variant will be selected for you, because you probably have the relevant +:ref:`implicit packages ` set to limit packages to +the current platform. + +Single Variants +=============== + +You may often see packages with just one variant. There are two reasons for this: + +* *Future proofing*: Let's say you have a compiled package that links against python. + It may currently support ``python-2.7``, however it's conceivable that support for + newer python versions may be added later. **It is not possible to add new variants + to a package that does not have any without changing/bumnig the version**. So by + adding the ``python-2.7`` variant now, you can add variants later without needing + to move to a newer version. +* *Installation path*: People often expect to see platform, architecture and/or + operating system information in the installation path of a piece of software (and + may also expect the same of python version, or other core packages). By putting + these dependencies into a variant, we ensure that they appear in the installation + path of the package. + +Variant Selection +================= + +As mentioned, rez will automatically select the correct variant of a package +depending on the environment being resolved. For example, consider: + +.. code-block:: text + + ]$ rez-env my_maya_plugin maya-2017 -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' + /rez/packages/my_maya_plugin/1.0.0/maya-2017 + ]$ rez-env my_maya_plugin maya-2016 -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' + /rez/packages/my_maya_plugin/1.0.0/maya-2016.sp2 + +You can see how the correct variant (ie the one that does not conflict with other +packages in the request) has been selected. But what if both variants are valid +for the given request? Consider: + +.. code-block:: text + + ]$ rez-env my_maya_plugin -- echo '$REZ_MY_MAYA_PLUGIN_ROOT' + /rez/packages/my_maya_plugin/1.0.0/maya-2017 + +Here ``maya`` was not in the request. Either variant of ``my_maya_plugin`` would have +satisfied the request, since we have not specified which version of maya we actually +want. + +By default, rez will prefer the variant with the higher-versioned packages, which +is why the ``maya-2017`` variant was selected in this example. If there are +multiple packages in the variant, priority is given to those that were in the request +list, if any; following that, priority is given to packages listed earlier in the +variant. For example, consider: + +.. code-block:: python + + name = "foo" + + variants = [ + ["python-2.6", "maya-2017"], + ["python-2.7", "maya-2016"] + ] + +If I run ``rez-env foo``, which variant will I get? The answer is not clear. In this +case it will be the second variant, since ``python`` is given priority (it is the +first listed package in the variant), and the second variant has the higher version +of python. However, if I ran ``rez-env foo maya``, I would get the ``first`` variant +because priority is now given to ``maya``, because it's listed in my request, and the +first variant has the higher version of maya. + +The rez setting :data:`variant_select_mode` affects this selection behavior. +The default mode just described is ``version_priority``, but there is another mode called +``intersection_priority``. In this mode, variants are preferred that have *the most +number of packages present in the request*; version priority is secondary. + +Mutual Exclusivity +------------------ + +In all the examples we've seen so far, a package's variants have been mutually +exclusive. For example, you cannot have both ``python-2.6`` and ``python-2.7`` in the +same environment, so when we request ``foo python-2.6`` we can be sure of which +variant we will get. + +Variants, however, do not need to be mutually exclusive. In fact, you may use +variants in order to provide support for different DCCs for your package. Consider +a package with the following variants: + +.. code-block:: python + + name = "geocache" + + variants = [ + ["maya-2016"], + ["houdini-14"] + ] + +Which variant will I get if I run ``rez-env geocache``? Behavior in this case is +undefined. Rez gives no guarantees as to which variant will be selected. We cannot +meaningfully compare version numbers across packages, so maya will not have preference +simply because 2016 > 14. However, ``version_priority`` mode does give priority to +packages listed in the request. So if we ran ``rez-env geocache maya``, we will get +the first variant... probably. + +Probably? ++++++++++ + +The operative word here is *preference*. Because the variants are not mutually +exclusive, we can't make guarantees. A resolve is still deterministic. You aren't +going to get differing results when requesting the same environment, but predicting +which variant you'll get can be tricky. + +Consider the following request: + +.. code-block:: text + + ]$ rez-env geocache maya animtools-1.4 + +We would expect to get the ``maya-2016`` variant of ``geocache``. However, what if +``animtools`` requires ``maya-2017``? This makes the first ``geocache`` variant impossible +to select, since a conflict would occur, and so the ``houdini`` variant of ``geocache`` +will be selected. Quite possibly not what you expected. + +.. todo:: Document this? +.. > [[media/icons/under_construction.png]] I plan on adding a new package request +.. > syntax, that is able to explicitly select package variants. This will avoid the +.. > ambiguity in cases like the one described here. + +Why Use Variants? +================= + +Variants are a powerful mechanism in rez. They avoid the need to maintain separate +branches of a package in order to support varying dependencies. You may have had +problems in the past where a common library depends on, say, boost, and is used in +various DCCs (maya, nuke etc), and depended on by many other packages in your +pipeline. When a DCC moves to a new version of boost (or python, or OIIO, etc) +you now have to branch this library, which potentially affects many other packages. +The problem gets worse if you have multiple dependencies with varying versions. + +Variants solve that problem. You simply add another boost variant to your library, +and other dependent packages are not affected. Rez will correctly select the +package variant that does not conflict with the resolved environment. diff --git a/src/rez/bundle_context.py b/src/rez/bundle_context.py index c60c827fb..f300c6514 100644 --- a/src/rez/bundle_context.py +++ b/src/rez/bundle_context.py @@ -21,7 +21,7 @@ def bundle_context(context, dest_dir, force=False, skip_non_relocatable=False, This creates a copy of a context with its variants retargeted to a local package repository containing only the variants the context uses. The - generated file structure looks like so: + generated file structure looks like so:: /dest_dir/ /context.rxt diff --git a/src/rez/cli/_main.py b/src/rez/cli/_main.py index dccf6261b..a27de793a 100644 --- a/src/rez/cli/_main.py +++ b/src/rez/cli/_main.py @@ -89,7 +89,7 @@ def setup_parser(): LazyArgumentParser: Argument parser for rez command. """ py = sys.version_info - parser = LazyArgumentParser("rez") + parser = LazyArgumentParser("rez", description="rez CLI") parser.add_argument("-i", "--info", action=InfoAction, help="print information about rez and exit") diff --git a/src/rez/cli/_util.py b/src/rez/cli/_util.py index 45910258e..99646bb6e 100644 --- a/src/rez/cli/_util.py +++ b/src/rez/cli/_util.py @@ -176,12 +176,16 @@ def __init__(self, *args, **kwargs): def format_help(self): """Sets up all sub-parsers when help is requested.""" + self._setup_all_subparsers() + return super(LazyArgumentParser, self).format_help() + + def _setup_all_subparsers(self): + """Sets up all sub-parsers on demand.""" if self._subparsers: for action in self._subparsers._actions: if isinstance(action, LazySubParsersAction): for parser_name, parser in action._name_parser_map.items(): action._setup_subparser(parser_name, parser) - return super(LazyArgumentParser, self).format_help() _handled_int = False diff --git a/src/rez/cli/pkg-cache.py b/src/rez/cli/pkg-cache.py index b8cfe4d6c..26a62e871 100644 --- a/src/rez/cli/pkg-cache.py +++ b/src/rez/cli/pkg-cache.py @@ -50,7 +50,7 @@ def setup_parser(parser, completions=False): parser.add_argument( "-f", "--force", action="store_true", help="Force a package add, even if package is not cachable. Only " - "applicable with --add" + "applicable with --add-variants" ) parser.add_argument( "DIR", nargs='?', diff --git a/src/rez/command.py b/src/rez/command.py index 9b58d4400..3cc0696a0 100644 --- a/src/rez/command.py +++ b/src/rez/command.py @@ -11,15 +11,18 @@ class Command(object): To register plugin and expose subcommand, the plugin module.. * MUST have a module-level docstring (used as the command help) - * MUST provide a `setup_parser()` function - * MUST provide a `command()` function - * MUST provide a `register_plugin()` function - * SHOULD have a module-level attribute `command_behavior` + * MUST provide a ``setup_parser()`` function + * MUST provide a ``command()`` function + * MUST provide a ``register_plugin()`` function + * SHOULD have a module-level attribute ``command_behavior`` - For example, a plugin named 'foo' and this is the `foo.py`: + For example, a plugin named 'foo' and this is the ``foo.py``: - '''The docstring for command help, this is required. - ''' + .. code-block:: python + + \""" + The docstring for command help, this is required. + \""" from rez.command import Command command_behavior = { diff --git a/src/rez/package_cache.py b/src/rez/package_cache.py index d9faecd05..663e61330 100644 --- a/src/rez/package_cache.py +++ b/src/rez/package_cache.py @@ -41,14 +41,13 @@ class PackageCache(object): * A rez-env is performed; * The context is resolved; - * For each variant in the context, we check to see if it's present in the - current package cache; + * For each variant in the context, we check to see if it's present in the current package cache; * If it is, the variant's root is remapped to this location. - A package cache is _not_ a package repository. It just stores copies of + A package cache is **not** a package repository. It just stores copies of variant payloads - no package definitions are stored. - Payloads are stored into the following structure: + Payloads are stored into the following structure:: //foo/1.0.0/af8d/a/ /a.json @@ -62,13 +61,13 @@ class PackageCache(object): matching variant. """ - VARIANT_NOT_FOUND = 0 # Variant was not found - VARIANT_FOUND = 1 # Variant was found - VARIANT_CREATED = 2 # Variant was created - VARIANT_COPYING = 3 # Variant payload is still being copied to this cache - VARIANT_COPY_STALLED = 4 # Variant payload copy has stalled - VARIANT_PENDING = 5 # Variant is pending caching - VARIANT_REMOVED = 6 # Variant was deleted + VARIANT_NOT_FOUND = 0 #: Variant was not found + VARIANT_FOUND = 1 #: Variant was found + VARIANT_CREATED = 2 #: Variant was created + VARIANT_COPYING = 3 #: Variant payload is still being copied to this cache + VARIANT_COPY_STALLED = 4 #: Variant payload copy has stalled + VARIANT_PENDING = 5 #: Variant is pending caching + VARIANT_REMOVED = 6 #: Variant was deleted _FILELOCK_TIMEOUT = 10 _COPYING_TIME_INC = 0.2 @@ -145,18 +144,14 @@ def add_variant(self, variant, force=False): config.package_cache_same_device' is False. Args: - variant (`Variant`): The variant to copy into this cache + variant (Variant): The variant to copy into this cache force (bool): Copy the variant regardless. Use at your own risk (there is no guarantee the resulting variant payload will be functional). Returns: 2-tuple: - str: Path to cached payload - - int: One of: - - VARIANT_FOUND - - VARIANT_CREATED - - VARIANT_COPYING - - VARIANT_COPY_STALLED + - int: One of VARIANT_FOUND, VARIANT_CREATED, VARIANT_COPYING, VARIANT_COPY_STALLED """ from rez.utils.base26 import get_next_base26 from rez.utils.filesystem import safe_makedirs @@ -598,8 +593,8 @@ def clean(self, time_limit=None): This should be run periodically via 'rez-pkg-cache --clean'. This removes: - - Variants that have not been used in more than - 'config.package_cache_max_variant_days' days; + + - Variants that have not been used in more than 'config.package_cache_max_variant_days' days; - Variants that have stalled; - Variants that are already pending deletion (remove_variant() was used). diff --git a/src/rez/packages.py b/src/rez/packages.py index b4c0c914d..2cac8f24a 100644 --- a/src/rez/packages.py +++ b/src/rez/packages.py @@ -190,16 +190,18 @@ def _eval_late_binding(self, sourcecode): class Package(PackageBaseResourceWrapper): """A package. - Note: + Warning: Do not instantiate this class directly, instead use the function - `iter_packages` or `PackageFamily.iter_packages`. + :func:`iter_packages` or :meth:`PackageFamily.iter_packages`. """ keys = schema_keys(package_schema) - # This is to allow for a simple check like 'this.is_package' in late-bound - # funcs, where 'this' may be a package or variant. - # + #: Allows for a simple check like ``this.is_package`` in late-bound + #: funcs, where ``this`` may be a package or variant. is_package = True + + #: Allows for a simple check like ``this.is_variant`` in late-bound + #: funcs, where ``this`` may be a package or variant. is_variant = False def __init__(self, resource, context=None): @@ -326,15 +328,17 @@ def get_variant(self, index=None): class Variant(PackageBaseResourceWrapper): """A package variant. - Note: + Warning: Do not instantiate this class directly, instead use the function - `Package.iter_variants`. + :meth:`Package.iter_variants`. """ keys = schema_keys(variant_schema) keys.update(["index", "root", "subpath"]) - # See comment in `Package` + #: See :attr:`Package.is_package`. is_package = False + + #: See :attr:`Package.is_variant`. is_variant = True def __init__(self, resource, context=None, parent=None): diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index 93e5f7a66..9b11f7fcc 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -48,13 +48,12 @@ import os +### Do not move or delete this comment (__DOC_START__) ############################################################################### # Paths ############################################################################### -### Do not move or delete this comment (__DOC_START__) - # The package search path. Rez uses this to find packages. A package with the # same name and version in an earlier path takes precedence. packages_path = [ @@ -63,64 +62,66 @@ "~/.rez/packages/ext", # external (3rd party) pkgs, such as houdini, boost ] -# The path that Rez will locally install packages to when rez-build is used +# The path that Rez will locally install packages to when :ref:`rez-build` is used local_packages_path = "~/packages" -# The path that Rez will deploy packages to when rez-release is used. For +# The path that Rez will deploy packages to when :ref:`rez-release` is used. For # production use, you will probably want to change this to a site-wide location. release_packages_path = "~/.rez/packages/int" # Where temporary files go. Defaults to appropriate path depending on your -# system - for example, *nix distributions will probably set this to "/tmp". It -# is highly recommended that this be set to local storage, such as /tmp. +# system. For example, \*nix distributions will probably set this to :file:`/tmp`. It +# is highly recommended that this be set to local storage, such as :file:`/tmp`. tmpdir = None # Where temporary files for contexts go. Defaults to appropriate path depending -# on your system - for example, *nix distributions will probably set this to "/tmp". -# This is separate to 'tmpdir' because you sometimes might want to set this to an -# NFS location - for example, perhaps rez is used during a render and you'd like +# on your system. For example, \*nix distributions will probably set this to :file:`/tmp`. +# This is separate to :data:`tmpdir` because you sometimes might want to set this to an +# NFS location. For example, perhaps rez is used during a render and you'd like # to store these tempfiles in the farm queuer's designated tempdir so they're # cleaned up when the render completes. context_tmpdir = None -# These are extra python paths that are added to sys.path **only during a build**. +# These are extra python paths that are added to :data:`sys.path` **only during a build**. # This means that any of the functions in the following list can import modules # from these paths: -# * The *preprocess* function; -# * Any function decorated with @early - these get evaluated at build time. +# +# * The :func:`preprocess` function; +# * Any function decorated with :ref:`@early `. These get evaluated at build time. # # You can use this to provide common code to your package definition files during # a build. To provide common code for packages to use at resolve time instead (for -# example, in a *commands* function) see the following -# *package_definition_python_path* setting. -# +# example, in a :func:`commands` function) see the following +# :data:`package_definition_python_path` setting. package_definition_build_python_paths = [] # This is the directory from which installed packages can import modules. This # is a way for packages to use shared code. # -# This is NOT a standard path added to sys.path. Packages that use modules from +# This is NOT a standard path added to :data:`sys.path`. Packages that use modules from # within this directory need to explicitly name them. Furthermore, modules that -# a package uses are copied into that package's install - this ensures that the +# a package uses are copied into that package's install. This ensures that the # package remains standalone and that changes to the shared code will not break # or alter existing package installs. # # Consider the setting: # -# package_definition_python_path = "/src/rezutils" +# .. code-block:: python +# +# package_definition_python_path = "/src/rezutils" # -# Consider also the following package *commands* function: +# Consider also the following package :func:`commands` function: # -# @include("utils") -# def commands(): -# utils.do_some_common_thing(this) +# .. code-block:: python # -# This package will import the code from */src/rezutils/utils.py* (or more -# specifically, its copy of this sourcefile) and will bind it to the name *utils*. +# @include("utils") +# def commands(): +# utils.do_some_common_thing(this) # -# For further information, see -# [here](Package-Definition-Guide#sharing-code-across-package-definition-files). +# This package will import the code from :file:`/src/rezutils/utils.py` (or more +# specifically, its copy of this sourcefile) and will bind it to the name ``utils``. # +# For further information, see :ref:`package-definition-sharing-code`. package_definition_python_path = None @@ -131,7 +132,7 @@ # Search path for rez plugins. plugin_path = [] -# Search path for bind modules. The *rez-bind* tool uses these modules to create +# Search path for bind modules. The :ref:`rez-bind` tool uses these modules to create # rez packages that reference existing software already installed on the system. bind_module_path = [] @@ -161,8 +162,8 @@ resource_caching_maxsize = -1 # Uris of running memcached server(s) to use as a file and resolve cache. For -# example, the uri "127.0.0.1:11211" points to memcached running on localhost on -# its default port. Must be either null, or a list of strings. +# example, the URI ``127.0.0.1:11211`` points to memcached running on localhost on +# its default port. Must be either None, or a list of strings. memcached_uri = [] # Bytecount beyond which memcached entries are compressed, for cached package @@ -187,37 +188,41 @@ ############################################################################### # Whether a package is relocatable or not, if it does not explicitly state with -# the 'relocatable' attribute in its package definition file. +# the :attr:`relocatable` attribute in its package definition file. default_relocatable = True -# Set relocatable on a per-package basis. This is here for migration purposes - -# it's better for packages themselves to set their relocatable attribute. -# Overrides 'default_relocatable' if a package matches. +# Set relocatable on a per-package basis. This is here for migration purposes. +# It's better for packages themselves to set their relocatable attribute. +# Overrides :data:`default_relocatable` if a package matches. # # Example: # -# default_relocatable_per_package = { -# "nuke": False, -# "maya": True -# } +# .. code-block:: python +# +# default_relocatable_per_package = { +# "nuke": False, +# "maya": True +# } default_relocatable_per_package = None # Set relocatable on a per-package-repository basis. Overrides -# 'default_relocatable_per_package' and 'default_relocatable' for any repos +# :data:`default_relocatable_per_package` and :data:`default_relocatable` for any repos # listed here. # # Example: # -# default_relocatable_per_repostitory = { -# '/svr/packages': False -# } +# .. code-block:: python +# +# default_relocatable_per_repostitory = { +# '/svr/packages': False +# } default_relocatable_per_repository = None ############################################################################### # Package Caching # -# Note: "package caching" refers to copying variant payloads to a path on local +# Package caching refers to copying variant payloads to a path on local # disk, and using those payloads instead. It is a way to avoid fetching files # over shared storage, and is unrelated to memcached-based caching of resolves # and package definitions as seen in the "Caching" config section. @@ -225,30 +230,34 @@ ############################################################################### # Whether a package is cachable or not, if it does not explicitly state with -# the 'cachable' attribute in its package definition file. If None, defaults -# to packages' relocatability (ie cachable will == relocatable). +# the :attr:`cachable` attribute in its package definition file. If None, defaults +# to packages' relocatability (ie cachable == relocatable). default_cachable = False -# Set cachable on a per-package basis. This is here for migration purposes - -# it's better for packages themselves to set their cachable attribute. Overrides -# 'default_cachable' if a package matches. +# Set cachable on a per-package basis. This is here for migration purposes. +# It's better for packages themselves to set their cachable attribute. Overrides +# :data:`default_cachable` if a package matches. # # Example: # -# default_cachable_per_package = { -# "nuke": False, -# "maya": True -# } +# .. code-block:: python +# +# default_cachable_per_package = { +# "nuke": False, +# "maya": True +# } default_cachable_per_package = None # Set cachable on a per-package-repository basis. Overrides -# 'default_cachable_per_package' and 'default_cachable' for any repos listed here. +# :data:`default_cachable_per_package` and :data:`default_cachable` for any repos listed here. # # Example: # -# default_cachable_per_repostitory = { -# '/svr/packages': False -# } +# .. code-block:: python +# +# default_cachable_per_repostitory = { +# '/svr/packages': False +# } default_cachable_per_repository = None # The path where rez locally caches variants. If this is None, then package @@ -262,7 +271,7 @@ # If True, creating or sourcing a context will cause variants to be cached. write_package_cache = True -# Delete variants that haven't been used in N days (see `rez-pkg-cache --clean`). +# Delete variants that haven't been used in N days (see :option:`rez-pkg-cache --clean`). # To disable, set to zero. package_cache_max_variant_days = 30 @@ -280,11 +289,11 @@ # If > 0, spend up to this many seconds cleaning the cache every time the cache # is updated. This is a way to keep the cache size under control without having -# to periodically run 'rez-pkg-cache --clean'. Set to -1 to disable. +# to periodically run :option:`rez-pkg-cache --clean`. Set to -1 to disable. package_cache_clean_limit = 0.5 # Number of days of package cache logs to keep. -# Logs are written to {pkg-cache-root}/.sys/log/*.log +# Logs are written to :file:`{pkg-cache-root}/.sys/log/{filename}.log` package_cache_log_days = 7 @@ -293,7 +302,7 @@ ############################################################################### # Packages that are implicitly added to all package resolves, unless the -# --no-implicit flag is used. +# :option:`rez-env --no-implicit` flag is used. implicit_packages = [ "~platform=={system.platform}", "~arch=={system.arch}", @@ -302,21 +311,25 @@ # Override platform values from Platform.os and arch. # This is useful as Platform.os might show different -# values depending on the availability of lsb-release on the system. -# The map supports regular expression e.g. to keep versions. -# Please note that following examples are not necessarily recommendations. -# -# platform_map = { -# "os": { -# r"Scientific Linux-(.*)": r"Scientific-\1", # Scientific Linux-x.x -> Scientific-x.x -# r"Ubuntu-14.\d": r"Ubuntu-14", # Any Ubuntu-14.x -> Ubuntu-14 -# r'CentOS Linux-(\d+)\.(\d+)(\.(\d+))?': r'CentOS-\1.\2', ' # Centos Linux-X.Y.Z -> CentOS-X.Y -# }, -# "arch": { -# "x86_64": "64bit", # Maps both x86_64 and amd64 -> 64bit -# "amd64": "64bit", -# }, -# } +# values depending on the availability of ``lsb-release`` on the system. +# The map supports regular expression, e.g. to keep versions. +# +# .. note:: +# The following examples are not necessarily recommendations. +# +# .. code-block:: python +# +# platform_map = { +# "os": { +# r"Scientific Linux-(.*)": r"Scientific-\1", # Scientific Linux-x.x -> Scientific-x.x +# r"Ubuntu-14.\d": r"Ubuntu-14", # Any Ubuntu-14.x -> Ubuntu-14 +# r'CentOS Linux-(\d+)\.(\d+)(\.(\d+))?': r'CentOS-\1.\2', ' # Centos Linux-X.Y.Z -> CentOS-X.Y +# }, +# "arch": { +# "x86_64": "64bit", # Maps both x86_64 and amd64 -> 64bit +# "amd64": "64bit", +# }, +# } platform_map = {} # If true, then when a resolve graph is generated during a failed solve, packages @@ -327,146 +340,159 @@ # Variant select mode. This determines which variants in a package are preferred # during a solve. Valid options are: -# - version_priority: Prefer variants that contain higher versions of packages +# +# - **version_priority**: Prefer variants that contain higher versions of packages # present in the request; -# - intersection_priority: Prefer variants that contain the most number of +# - **intersection_priority**: Prefer variants that contain the most number of # packages that are present in the request. # # As an example, suppose you have a package foo which has two variants: # -# variants = [ -# ["bar-3.0", "baz-2.1"], -# ["bar-2.8", "burgle-1.0"] -# ] -# -# if you do: -# -# rez-env foo bar -# -# ...then, in either variant_select_mode, it will prefer the first variant, -# ["bar-3.0", "baz-2.1"], because it has a higher version of the first variant -# requirement (bar). However, if we instead do: +# .. code-block:: python # -# rez-env foo bar burgle +# variants = [ +# ["bar-3.0", "baz-2.1"], +# ["bar-2.8", "burgle-1.0"] +# ] # -# ...we get different behavior. version_priority mode will still return -# ["bar-3.0", "baz-2.1"], because the first requirement's version is higher. +# if you do ``rez-env foo bar`` then, in either variant_select_mode, +# it will prefer the first variant, ``["bar-3.0", "baz-2.1"]``, because it has a +# higher version of the first variant requirement (bar). However, if we instead do +# ``rez-env foo bar burgle`` we get different behavior. version_priority mode will +# still return ``["bar-3.0", "baz-2.1"]``, because the first requirement's version is higher. # -# However, intersection_priority mode will pick the second variant, -# ["bar-2.8", "burgle-1.0"], because it contains more packages that were in the +# However, ``intersection_priority`` mode will pick the second variant, +# ``["bar-2.8", "burgle-1.0"]``, because it contains more packages that were in the # original request (burgle). variant_select_mode = "version_priority" -# Package filter. One or more filters can be listed, each with a list of +# One or more filters can be listed, each with a list of # exclusion and inclusion rules. These filters are applied to each package # during a resolve, and if any filter excludes a package, that package is not # included in the resolve. Here is a simple example: # -# package_filter = { -# 'excludes': 'glob(*.beta)', -# 'includes': 'glob(foo-*)', -# } +# .. code-block:: python +# +# package_filter = { +# 'excludes': 'glob(*.beta)', +# 'includes': 'glob(foo-*)', +# } # # This is an example of a single filter with one exclusion rule and one inclusion -# rule. The filter will ignore all packages with versions ending in '.beta', -# except for package 'foo' (which it will accept all versions of). A filter will +# rule. The filter will ignore all packages with versions ending in ``.beta``, +# except for package ``foo`` (which it will accept all versions of). A filter will # only exclude a package if that package matches at least one exclusion rule, # and does not match any inclusion rule. # # Here is another example, which excludes all beta and dev packages, and all -# packages except 'foo' that are released after a certain date. Note that in +# packages except ``foo`` that are released after a certain date. Note that in # order to use multiple filters, you need to supply a list of dicts, rather # than just a dict: # -# package_filter = [ -# { -# 'excludes': ['glob(*.beta)', 'glob(*.dev)'] -# }, -# { -# 'excludes': ['after(1429830188)'], -# 'includes': ['foo'], # same as range(foo), same as glob(foo-*) -# } -# ] -# -# This example shows why multiple filters are supported - with only one filter, +# .. code-block:: python +# +# package_filter = [ +# { +# 'excludes': ['glob(*.beta)', 'glob(*.dev)'] +# }, +# { +# 'excludes': ['after(1429830188)'], +# 'includes': ['foo'], # same as range(foo), same as glob(foo-*) +# } +# ] +# +# This example shows why multiple filters are supported. With only one filter, # it would not be possible to exclude all beta and dev packages (including foo), # but also exclude all packages after a certain date, except for foo. # # Following are examples of all the possible rules: # -# example | description -# --------------------|---------------------------------------------------- -# glob(*.beta) | Matches packages matching the glob pattern. -# regex(.*-\\.beta) | Matches packages matching re-style regex. -# range(foo-5+) | Matches packages within the given requirement. -# before(1429830188) | Matches packages released before the given date. -# after(1429830188) | Matches packages released after the given date. -# *.beta | Same as glob(*.beta) -# foo-5+ | Same as range(foo-5+) +# ======================== ==================================================== +# Example Description +# ======================== ==================================================== +# ``glob(*.beta)`` Matches packages matching the glob pattern. +# ``regex(.*-\\.beta)`` Matches packages matching re-style regex. +# ``range(foo-5+)`` Matches packages within the given requirement. +# ``before(1429830188)`` Matches packages released before the given date. +# ``after(1429830188)`` Matches packages released after the given date. +# ``*.beta`` Same as glob(\*.beta). +# ``foo-5+`` Same as range(foo-5+). +# ======================== ==================================================== package_filter = None -# Package order. One or more "orderers" can be listed. +# One or more "orderers" can be listed. # This will affect the order of version resolution. # This can be used to ensure that specific version have priority over others. # Higher versions can still be accessed if required. # # A common use case is to ease migration from python-2 to python-3: # -# package_orderers = [ -# { -# "type": "per_family", -# "orderers": [ -# { -# "packages": ["python"], -# "type": "version_split", -# "first_version": "2.7.16" -# } -# ] -# } -# ] +# .. code-block:: python +# +# package_orderers = [ +# { +# "type": "per_family", +# "orderers": [ +# { +# "packages": ["python"], +# "type": "version_split", +# "first_version": "2.7.16" +# } +# ] +# } +# ] # # This will ensure that for the "python" package, versions equals or lower than "2.7.16" will have priority. # Considering the following versions: "2.7.4", "2.7.16", "3.7.4": # -# example | result -# --------------------|---------------------------------------------------- -# rez-env python | python-2.7.16 -# rez-env python-3 | python-3.7.4 +# ==================== ============= +# Example Result +# ==================== ============= +# rez-env python python-2.7.16 +# rez-env python-3 python-3.7.4 +# ==================== ============= # # Here's another example, using another orderer: "soft_timestamp". # This orderer will prefer packages released before a provided timestamp. # The following example will prefer package released before 2019-09-09. # -# package_orderers = [ -# { -# "type": "soft_timestamp", -# "timestamp": 1568001600, # 2019-09-09 -# "rank": 3 -# } -# ] +# .. code-block:: python +# +# package_orderers = [ +# { +# "type": "soft_timestamp", +# "timestamp": 1568001600, # 2019-09-09 +# "rank": 3 +# } +# ] # # A timestamp can be generated with python: # -# $ python -c "import datetime, time; print(int(time.mktime(datetime.date(2019, 9, 9).timetuple())))" -# 1568001600 +# .. code-block:: text +# +# $ python -c "import datetime, time; print(int(time.mktime(datetime.date(2019, 9, 9).timetuple())))" +# 1568001600 # # The rank can be used to allow some versions released after the timestamp to still be considered. # When using semantic versionnng, a value of 3 is the most common. # This will let version with a different patch number to be accepted. # # Considering a package "foo" with the following versions: +# # - "1.0.0" was released at 2019-09-07 # - "2.0.0" was released at 2019-09-08 # - "2.0.1" was released at 2019-09-10 # - "2.1.0" was released at 2019-09-11 # - "3.0.0" was released at 2019-09-12 # -# example | timestamp | rank | result -# --------------------|------------|------|-------------------------------- -# rez-env foo | 2019-09-09 | 0 | foo-2.0.0 -# rez-env foo | 2019-09-09 | 3 | foo-2.0.1 -# rez-env foo | 2019-09-09 | 2 | foo-2.1.0 -# rez-env foo | 2019-09-09 | 1 | foo-3.0.0 +# =========== ========== ==== ========= +# Example Timestamp Rank Result +# =========== ========== ==== ========= +# rez-env foo 2019-09-09 0 foo-2.0.0 +# rez-env foo 2019-09-09 3 foo-2.0.1 +# rez-env foo 2019-09-09 2 foo-2.1.0 +# rez-env foo 2019-09-09 1 foo-3.0.0 +# =========== ========== ==== ========= package_orderers = None # If True, unversioned packages are allowed. Solve times are slightly better if @@ -484,40 +510,44 @@ # python modules from the parent environment would be (incorrectly) accessible # within the Rez environment. # -# "Parent variables" override this behaviour - they are appended/prepended to, -# rather than being overwritten. If you set "all_parent_variables" to true, then -# all variables are considered parent variables, and the value of "parent_variables" -# is ignored. Be aware that if you make variables such as PATH, PYTHONPATH or +# "Parent variables" override this behaviour. They are appended/prepended to, +# rather than being overwritten. If you set :data:`all_parent_variables` to :data:`True`, then +# all variables are considered parent variables, and the value of :data:`parent_variables` +# is ignored. Be aware that if you make variables such as ``PATH``, :envvar:`PYTHONPATH` or # app plugin paths parent variables, you are exposing yourself to potentially # incorrect behaviour within a resolved environment. parent_variables = [] + +# See :data:`parent_variables`. all_parent_variables = False # When two or more packages in a resolve attempt to set the same environment # variable, Rez's default behaviour is to flag this as a conflict and abort the # resolve. You can overcome this in a package's commands section by using the -# Rex command "resetenv" instead of "setenv". However, you can also turn off this -# behaviour globally - for certain variables, by adding them to "resetting_variables", -# and for all variables, by setting "all_resetting_variables" to true. +# Rex command :func:`resetenv` instead of :func:`setenv`. However, you can also turn off this +# behaviour globally for some varibles by adding them to :data:`resetting_variables`, +# and for all variables, by setting :data:`all_resetting_variables` to true. resetting_variables = [] + +# See :data:`resetting_variables`. all_resetting_variables = False # The default shell type to use when creating resolved environments (eg when using -# rez-env, or calling ResolvedContext.execute_shell). If empty or null, the +# :ref:`rez-env`, or calling :meth:`.ResolvedContext.execute_shell`). If empty or None, the # current shell is used (for eg, "bash"). default_shell = "" # The command to use to launch a new Rez environment in a separate terminal (this -# is enabled using rez-env's "detached" option). If None, it is detected. +# is enabled using the :option:`rez-env --detached` option). If None, it is detected. terminal_emulator_command = None -# subprocess.Popen arguments to use in order to execute a shell in a new process -# group (see ResolvedContext.execute_shell, 'start_new_session'). Dict of -# (Popen argument, value). +# :class:`subprocess.Popen` arguments to use in order to execute a shell in a new process +# group (see :meth:`.ResolvedContext.execute_shell` and its ``start_new_session`` argument). +# Dict of (Popen argument, value). new_session_popen_args = None # This setting can be used to override the separator used for environment -# variables that represent a list of items. By default, the value of os.pathsep +# variables that represent a list of items. By default, the value of :data:`os.pathsep` # will be used, unless the environment variable is list here, in which case the # configured separator will be used. env_var_separators = { @@ -527,93 +557,101 @@ # This setting identifies path-like environment variables. This is required # because some shells need to apply path normalization. For example, the command -# `env.PATH.append("{root}/bin")` will be normalized to (eg) `C:\...\bin` in a -# `cmd` shell on Windows. Note that wildcards are supported. If this setting is +# ``env.PATH.append("{root}/bin")`` will be normalized to (eg) ``C:\...\bin`` in a +# ``cmd`` shell on Windows. Note that wildcards are supported. If this setting is # not correctly configured, then your shell may not work correctly. pathed_env_vars = [ "*PATH" ] -# Defines what suites on $PATH stay visible when a new rez environment is resolved. +# Defines what suites on ``$PATH`` stay visible when a new rez environment is resolved. # Possible values are: +# # - "never": Don"t attempt to keep any suites visible in a new env # - "always": Keep suites visible in any new env # - "parent": Keep only the parent suite of a tool visible # - "parent_priority": Keep all suites visible and the parent takes precedence suite_visibility = "always" -# Defines how Rez's command line tools are added back to PATH within a resolved +# Defines how Rez's command line tools are added back to ``$PATH`` within a resolved # environment. Valid values are: +# # - "append": Rez tools are appended to PATH (default); # - "prepend": Rez tools are prepended to PATH; -# - "never": Rez tools are not added back to PATH - rez will not be available +# - "never": Rez tools are not added back to PATH. Rez will not be available # within resolved shells. rez_tools_visibility = "append" # Defines when package commands are sourced during the startup sequence of an # interactive shell. If True, package commands are sourced before startup -# scripts (such as .bashrc). If False, package commands are sourced after. +# scripts (such as ``.bashrc``). If False, package commands are sourced after. package_commands_sourced_first = True -# Defines paths to initially set $PATH to, if a resolve appends/prepends $PATH. +# Defines paths to initially set ``$PATH`` to, if a resolve appends/prepends ``$PATH``. # If this is an empty list, then this initial value is determined automatically -# depending on the shell (for example, *nix shells create a temp clean shell and -# get $PATH from there; Windows inspects its registry). +# depending on the shell (for example, \*nix shells create a temp clean shell and +# get ``$PATH`` from there; Windows inspects its registry). standard_system_paths = [] # If you define this function, by default it will be called as the # *preprocess function* on every package that does not provide its own, # as part of the build process. This behavior can be changed by using the -# [package_preprocess_mode](#package_preprocess_mode) setting so that +# :data:`package_preprocess_mode` setting so that # it gets executed even if a package define its own preprocess function. # -# The setting can be a function defined in your rezconfig.py, or a string. +# The setting can be a function defined in your ``rezconfig.py``, or a string. # # Example of a function to define the setting: # -# # in your rezconfig.py -# def package_preprocess_function(this, data): -# # some custom code... +# .. code-block:: python +# +# # in your rezconfig.py +# def package_preprocess_function(this, data): +# # some custom code... # # In the case where the function is a string, it must be made available by setting -# the value of [package_definition_build_python_paths](#package_definition_build_python_paths) -# appropriately. +# the value of :data:`package_definition_build_python_paths` appropriately. # # For example, consider the settings: # -# package_definition_build_python_paths = ["/src/rezutils"] -# package_preprocess_function = "build.validate" +# .. code-block:: python +# +# package_definition_build_python_paths = ["/src/rezutils"] +# package_preprocess_function = "build.validate" # -# This would use the 'validate' function in the sourcefile /src/rezutils/build.py +# This would use the ``validate`` function in the sourcefile :file:`/src/rezutils/build.py` # to preprocess every package definition file that does not define its own # preprocess function. # # If the preprocess function raises an exception, an error message is printed, # and the preprocessing is not applied to the package. However, if the -# *InvalidPackageError* exception is raised, the build is aborted. +# :exc:`~rez.exceptions.InvalidPackageError` exception is raised, the build is aborted. # # You would typically use this to perform common validation or modification of # packages. For example, your common preprocess function might check that the # package name matches a regex. Here's what that might look like: # -# # in /src/rezutils/build.py -# import re -# from rez.exceptions import InvalidPackageError +# .. code-block:: python +# +# # in /src/rezutils/build.py +# import re +# from rez.exceptions import InvalidPackageError # -# def validate(package, data): -# regex = re.compile("(a-zA-Z_)+$") -# if not regex.match(package.name): -# raise InvalidPackageError("Invalid package name.") +# def validate(package, data): +# regex = re.compile("(a-zA-Z_)+$") +# if not regex.match(package.name): +# raise InvalidPackageError("Invalid package name.") # package_preprocess_function = None -# Defines in which order the [package_preprocess_function](#package_preprocess_function) -# and the `preprocess` function inside a `package.py` are executed. +# Defines in which order the :data:`package_preprocess_function` +# and the :attr:`preprocess` function inside a ``package.py`` are executed. # # Note that "global preprocess" means the preprocess defined by -# [package_preprocess_function](#package_preprocess_function). +# :data:`package_preprocess_function`. # # Possible values are: +# # - "before": Package's preprocess function is executed before the global preprocess; # - "after": Package's preprocess function is executed after the global preprocess; # - "override": Package's preprocess function completely overrides the global preprocess. @@ -624,39 +662,40 @@ # Context Tracking ############################################################################### -# Send data to AMQP whenever a context is created or sourced. +# Send data to AMQP server whenever a context is created or sourced. # The payload is like so: # -# { -# "action": "created", -# "host": "some_fqdn", -# "user": "${USER}", -# "context": { -# ... -# } -# } -# -# Action is one of ("created", "sourced"). Routing key is set to -# '{exchange_routing_key}.{action|upper}', eg 'REZ.CONTEXT.SOURCED'. "Created" +# .. code-block:: python +# +# { +# "action": "created", +# "host": "some_fqdn", +# "user": "${USER}", +# "context": { +# ... +# } +# } +# +# Action is one of (``created``, ``sourced``). Routing key is set to +# ``{exchange_routing_key}.{action|upper}``, eg ``REZ.CONTEXT.SOURCED``. ``created`` # is when a new context is constructed, which will either cause a resolve to -# occur, or fetches a resolve from the cache. "Sourced" is when an existing +# occur, or fetches a resolve from the cache. ``sourced`` is when an existing # context is recreated (eg loading an rxt file). # -# The "context" field contains the context itself (the same as what is present -# in an rxt file), filtered by the fields listed in 'context_tracking_context_fields'. +# The ``context`` field contains the context itself (the same as what is present +# in an rxt file), filtered by the fields listed in :data:`context_tracking_context_fields`. # -# Tracking is enabled if 'context_tracking_host' is non-empty. Set to "stdout" +# Tracking is enabled if :data:`context_tracking_host` is non-empty. Set to ``stdout`` # to just print the message to standard out instead, for testing purposes. -# Otherwise, '{host}[:{port}]' is expected. +# Otherwise, ``{host}[:{port}]`` is expected. # -# If any items are present in 'context_tracking_extra_fields', they are added +# If any items are present in :data:`context_tracking_extra_fields`, they are added # to the payload. If any extra field contains references to unknown env-vars, or # is set to an empty string (possibly due to var expansion), it is removed from # the message payload. -# context_tracking_host = '' -# See [context_tracking_host](#context_tracking_host) +# See :data:`context_tracking_host` context_tracking_amqp = { "userid": '', "password": '', @@ -666,7 +705,7 @@ "message_delivery_mode": 1 } -# See [context_tracking_host](#context_tracking_host) +# See :data:`context_tracking_host` context_tracking_context_fields = [ "status", "timestamp", @@ -678,7 +717,7 @@ "resolved_packages" ] -# See [context_tracking_host](#context_tracking_host) +# See :data:`context_tracking_host` context_tracking_extra_fields = {} @@ -687,8 +726,8 @@ ############################################################################### # If true, print warnings associated with shell startup sequence, when using -# tools such as rez-env. For example, if the target shell type is "sh", and -# the "rcfile" param is used, you would get a warning, because the sh shell +# tools such as :ref:`rez-env`. For example, if the target shell type is ``sh``, and +# the ``rcfile`` param is used, you would get a warning, because the sh shell # does not support rcfile. warn_shell_startup = False @@ -698,7 +737,7 @@ # Turn on all warnings warn_all = False -# Turn off all warnings. This overrides warn_all. +# Turn off all warnings. This overrides :data:`warn_all`. warn_none = False # Print info whenever a file is loaded from disk, or saved to disk. @@ -708,7 +747,7 @@ debug_plugins = False # Print debugging info such as VCS commands during package release. Note that -# rez-pip installations are controlled with this setting also. +# :ref:`rez-pip` installations are controlled with this setting also. debug_package_release = False # Print debugging info in binding modules. Binding modules should print using @@ -726,16 +765,16 @@ # Debug memcache usage. As well as printing debugging info to stdout, it also # sends human-readable strings as memcached keys (that you can read by running -# "memcached -vv" as the server) +# ``memcached -vv`` as the server) debug_memcache = False -# Print debugging info when AMPQ is used in context tracking +# Print debugging info when an AMPQ server is used in context tracking debug_context_tracking = False # Turn on all debugging messages debug_all = False -# Turn off all debugging messages. This overrides debug_all. +# Turn off all debugging messages. This overrides :data:`debug_all`. debug_none = False # When an error is encountered in rex code, rez catches the error and processes @@ -758,19 +797,19 @@ # build files are written). build_directory = "build" -# The number of threads a build system should use, eg the make '-j' option. -# If the string values "logical_cores" or "physical_cores", it is set to the +# The number of threads a build system should use, eg the make ``-j`` option. +# If the string values ``logical_cores`` or ``physical_cores`` are used, it is set to the # detected number of logical / physical cores on the host system. # (Logical cores are the number of cores reported to the OS, physical are the -# number of actual hardware processor cores. They may differ if, ie, the CPUs -# support hyperthreading, in which case logical_cores == 2 * physical_cores). -# This setting is exposed as the environment variable $REZ_BUILD_THREAD_COUNT +# number of actual hardware processor cores. They may differ if, ie, the CPUs +# support hyperthreading, in which case ``logical_cores == 2 * physical_cores``). +# This setting is exposed as the environment variable :envvar:`REZ_BUILD_THREAD_COUNT` # during builds. build_thread_count = "physical_cores" -# The release hooks to run when a release occurs. Release hooks are plugins - if +# The release hooks to run when a release occurs. Release hooks are plugins. If # a plugin listed here is not present, a warning message is printed. Note that a -# release hook plugin being loaded does not mean it will run - it needs to be +# release hook plugin being loaded does not mean it will run. It needs to be # listed here as well. Several built-in release hooks are available, see # rezplugins/release_hook. release_hooks = [] @@ -784,19 +823,16 @@ # existing package (such as releasing a variant into an existing package, or # copying a package) will, if possible, temporarily make a package writable # during these processes. The mode will be set back to original afterwards. -# make_package_temporarily_writable = True # The subdirectory where hashed variant symlinks (known as variant shortlinks) # are created. This is only relevant for packages whose 'hashed_variants' is # set to True. To disable variant shortlinks, set this to None. -# variant_shortlinks_dirname = "_v" # Whether or not to use variant shortlinks when resolving variant root paths. # You might want to disable this for testing purposes, but typically you would # leave this True. -# use_variant_shortlinks = True @@ -804,8 +840,8 @@ # Suites ############################################################################### -# The prefix character used to pass rez-specific commandline arguments to alias -# scripts in a suite. This must be a character other than "-", so that it doesn"t +# The prefix character used to pass rez-specific command line arguments to alias +# scripts in a suite. This must be a character other than ``-``, so that it doesn"t # clash with the wrapped tools" own commandline arguments. suite_alias_prefix_char = "+" @@ -814,43 +850,43 @@ # Appearance ############################################################################### -# Suppress all extraneous output - warnings, debug messages, progress indicators -# and so on. Overrides all warn_xxx and debug_xxx settings. +# Suppress all extraneous output (warnings, debug messages, progress indicators +# and so on). Overrides all ``warn_xxx`` and ``debug_xxx settings``. quiet = False # Show progress bars where applicable show_progress = True # The editor used to get user input in some cases. -# On osx, set this to "open -a " if you want to use a specific app. +# On macOS, set this to ``open -a `` if you want to use a specific app. editor = None -# The program used to view images by tools such as "rez-context -g" -# On osx, set this to "open -a " if you want to use a specific app. +# The program used to view images by tools such as :option:`rez-context -g` +# On macOS, set this to ``open -a `` if you want to use a specific app. image_viewer = None -# The browser used to view documentation; the rez-help tool uses this -# On osx, set this to "open -a " if you want to use a specific app. +# The browser used to view documentation. The :ref:`rez-help` tool uses this +# On macOS, set this to ``open -a `` if you want to use a specific app. browser = None -# The viewer used to view file diffs. On osx, set this to "open -a " +# The viewer used to view file diffs. On macOS, set this to ``open -a `` # if you want to use a specific app. difftool = None # The default image format that dot-graphs are rendered to. dot_image_format = "png" -# If true, tools such as rez-env will update the prompt when moving into a new +# If true, tools such as :ref:`rez-env` will update the prompt when moving into a new # resolved shell. Prompt nerds might do fancy things with their prompt that Rez -# can't deal with (but it can deal with a lot - colors etc - so try it first). -# By setting this to false, Rez will not change the prompt. Instead, you will -# probably want to set it yourself in your startup script (.bashrc etc). You will -# probably want to use the environment variable $REZ_ENV_PROMPT, which contains +# can't deal with (but it can deal with a lot (colors etc) so try it first). +# By setting this to False, Rez will not change the prompt. Instead, you will +# probably want to set it yourself in your startup script (``.bashrc`` etc). You will +# probably want to use the environment variable :envvar:`REZ_ENV_PROMPT`, which contains # the set of characters that are normally prefixed/suffixed to the prompt, ie -# '>', '>>' etc. +# ``>``, ``>>`` etc. set_prompt = True -# If true, prefixes the prompt, suffixes if false. Ignored if 'set_prompt' is +# If true, prefixes the prompt, suffixes if false. Ignored if :data:`set_prompt` is # false. prefix_prompt = True @@ -860,60 +896,63 @@ ############################################################################### # If not zero, truncates all package changelog entries to this maximum length. -# You should set this value - changelogs can theoretically be very large, and +# You should set this value. Changelogs can theoretically be very large, and # this adversely impacts package load times. max_package_changelog_chars = 65536 # If not zero, truncates all package changelogs to only show the last N commits max_package_changelog_revisions = 0 -# Default option on how to create scripts with util.create_executable_script. +# Default option on how to create scripts with :func:`~rez.utils.execution.create_executable_script`. # In order to support both windows and other OS it is recommended to set this -# to 'both'. +# to ``both``. # # Possible modes: +# # - single: # Creates the requested script only. # - py: -# Create .py script that will allow launching scripts on windows, -# if the shell adds .py to PATHEXT. Make sure to use PEP-397 py.exe -# as default application for .py files. +# Create ``.py`` script that will allow launching scripts on windows, +# if the shell adds .py to ``PATHEXT``. Make sure to use :pep:`397` ``py.exe`` +# as default application for ``.py`` files. # - platform_specific: # Will create py script on windows and requested on other platforms # - both: -# Creates the requested file and a .py script so that scripts can be +# Creates the requested file and a ``.py`` script so that scripts can be # launched without extension from windows and other systems. create_executable_script_mode = "single" -# Configurable pip extra arguments passed to the rez-pip install command. -# Since the rez-pip install command already includes some pre-configured -# arguments (target, use-pep517) this setting can potentially override the +# Configurable pip extra arguments passed to the :ref:`rez-pip` install command. +# Since the :ref:`rez-pip` install command already includes some pre-configured +# arguments (``--target``, ``--use-pep517``) this setting can potentially override the # default configuration in a way which can cause package installation issues. # It is recommended to refrain from overriding the default arguments and only # use this setting for additional arguments that might be needed. # https://pip.pypa.io/en/stable/reference/pip_install/#options pip_extra_args = [] -# Substitutions for re.sub when unknown parent paths are encountered in the -# pip package distribution record: *.dist-info/RECORD +# Substitutions for :func:`re.sub` when unknown parent paths are encountered in the +# pip package distribution record: :file:`{name}.dist-info/RECORD` # # Rez reads the distribution record to figure out where pip installed files # to, then copies them to their final sub-path in the rez package. Ie, python -# source files are hard coded to be moved under the "python" sub-folder inside -# the rez package, which then gets added to PYTHONPATH upon rez-env. +# source files are hard coded to be moved under the ``python`` sub-folder inside +# the rez package, which then gets added to :envvar:`PYTHONPATH` upon :ref:`rez-env`. # # When it can't find the file listed in the record AND the path starts -# with a reference to the parent directory "..", the following remaps are +# with a reference to the parent directory ``..``, the following remaps are # used to: +# # 1. Match a path listed in the record to perform the filepath remapping; -# 2. re.sub expression from step 1 to make the relative path of where pip +# 2. :func:`re.sub` expression from step 1 to make the relative path of where pip # actually installed the file to; -# 3. re.sub expression from step 1 to make the destination filepath, relative +# 3. :func:`re.sub` expression from step 1 to make the destination filepath, relative # to the rez variant root. # # Use these tokens to avoid regular expression and OS-specific path issues: -# - "{pardir}" or "{p}" for parent directory: os.pardir, i.e. ".." on Linux/Mac -# - "{sep}" or "{s}" for folder separators: os.sep, i.e. "/" on Linux/Mac +# +# - "{pardir}" or "{p}" for parent directory: :data:`os.pardir`, i.e. ``..`` on Linux/Mac +# - "{sep}" or "{s}" for folder separators: :data:`os.sep`, i.e. ``/`` on Linux/Mac pip_install_remaps = [ # Typical bin copy behaviour # Path in record | pip installed to | copy to rez destination @@ -935,49 +974,58 @@ }, ] -# Optional variables. A dict type config for storing arbitrary data that can be -# accessed by the [optionvars](Package-Commands#optionvars) function in packages -# *commands*. +# A dict type config for storing arbitrary data that can be +# accessed by the :func:`optionvars` function in packages +# :func:`commands`. # # This is like user preferences for packages, which may not easy to define in -# package's definition file directly due to the differences between machines/ -# users/pipeline-roles. +# package's definition file directly due to the differences between machines/users/pipeline-roles. # # Example: # -# # in your rezconfig.py -# optionvars = { -# "userRoles": ["artist"] -# } +# .. code-block:: python +# +# # in your rezconfig.py +# optionvars = { +# "userRoles": ["artist"] +# } # # And to access: # -# # in package.py -# def commands(): -# roles = optionvars("userRoles", default=None) or [] -# if "artist" in roles: -# env.SOMETHING_FRIENDLY = "Yes" +# .. code-block:: python +# +# # in package.py +# def commands(): +# roles = optionvars("userRoles", default=None) or [] +# if "artist" in roles: +# env.SOMETHING_FRIENDLY = "Yes" # # Note that you can refer to values in nested dicts using dot notation: # -# def commands(): -# if optionvars("nuke.lighting_tools_enabled"): -# ... +# .. code-block:: python # +# def commands(): +# if optionvars("nuke.lighting_tools_enabled"): +# ... optionvars = None ############################################################################### # Rez-1 Compatibility +# +# These settings are for rez v1 compatibility purposes. +# ############################################################################### -# If this is true, rxt files are written in yaml format. If false, they are -# written in json, which is a LOT faster. You would only set to true for +# If this is True, rxt files are written in YAML format. If False, they are +# written in JSON, which is a LOT faster. You would only set to true for # backwards compatibility reasons. Note that rez will detect either format on # rxt file load. rxt_as_yaml = False # Warn or disallow when a package is found to contain old rez-1-style commands. warn_old_commands = True + +# See :data:`warn_old_commands`. error_old_commands = False # Print old commands and their converted rex equivalent. Note that this can @@ -990,40 +1038,48 @@ # instead of "commands". Unlike "commands", "commands2" only allows new rex- # style commands. Once you have fully deprecated Rez-1, you should stop using # "commands2". -# TODO DEPRECATE +# TODO: DEPRECATE warn_commands2 = False + +# See :data:`warn_commands2`. error_commands2 = False # If True, Rez will continue to generate the given environment variables in # resolved environments, even though their use has been deprecated in Rez-2. # The variables in question, and their Rez-2 equivalent (if any) are: # -# REZ-1 | REZ-2 -# -------------------|----------------- -# REZ_REQUEST | REZ_USED_REQUEST -# REZ_RESOLVE | REZ_USED_RESOLVE -# REZ_VERSION | REZ_USED_VERSION -# REZ_PATH | REZ_USED -# REZ_RESOLVE_MODE | not set -# REZ_RAW_REQUEST | not set -# REZ_IN_REZ_RELEASE | not set +# ================== ========================== +# REZ-1 REZ-2 +# ================== ========================== +# REZ_REQUEST :envvar:`REZ_USED_REQUEST` +# REZ_RESOLVE :envvar:`REZ_USED_RESOLVE` +# REZ_VERSION :envvar:`REZ_USED_VERSION` +# REZ_PATH :envvar:`REZ_USED` +# REZ_RESOLVE_MODE not set +# REZ_RAW_REQUEST not set +# REZ_IN_REZ_RELEASE not set +# ================== ========================== rez_1_environment_variables = True # If True, Rez will continue to generate the given CMake variables at build and -# release time, even though their use has been deprecated in Rez-2. The +# release time, even though their use has been deprecated in Rez-2. The # variables in question, and their Rez-2 equivalent (if any) are: # -# REZ-1 | REZ-2 -# --------|--------------- -# CENTRAL | REZ_BUILD_TYPE +# ======= ======================== +# REZ-1 REZ-2 +# ======= ======================== +# CENTRAL :envvar:`REZ_BUILD_TYPE` +# ======= ======================== rez_1_cmake_variables = True # If True, override all compatibility-related settings so that Rez-1 support is # deprecated. This means that: +# # * All warn/error settings in this section of the config will be set to # warn=False, error=True; -# * rez_1_environment_variables will be set to False. -# * rez_1_cmake_variables will be set to False. +# * :data:`rez_1_environment_variables` will be set to False. +# * :data:`rez_1_cmake_variables` will be set to False. +# # You should aim to do this - it will mean your packages are more strictly # validated, and you can more easily use future versions of Rez. disable_rez_1_compatibility = False @@ -1034,43 +1090,45 @@ ############################################################################### # Where Rez's own documentation is hosted -documentation_url = "https://github.com/AcademySoftwareFoundation/rez/wiki" +documentation_url = "https://rez.readthedocs.io" ############################################################################### # Colorization -############################################################################### - +# # The following settings provide styling information for output to the console, # and is based on the capabilities of the Colorama module # (https://pypi.python.org/pypi/colorama). # -# *_fore and *_back colors are based on the colors supported by this module and -# the console. One or more styles can be applied using the *_styles +# \*_fore and \*_back colors are based on the colors supported by this module and +# the console. One or more styles can be applied using the \*_styles # configuration. These settings will also affect the logger used by rez. # # At the time of writing, valid values are: # fore/back: black, red, green, yellow, blue, magenta, cyan, white # style: dim, normal, bright +# +############################################################################### # Enables/disables colorization globally. # -# > [[media/icons/warning.png]] Note: Turned off for Windows currently as there seems -# > to be a problem with the Colorama module. +# .. warning:: +# Turned off for Windows currently as there seems to be a problem with the colorama module. # -# May also set to the string "force", which will make rez output color styling +# May also set to the string ``force``, which will make rez output color styling # information, even if the the output streams are not ttys. Useful if you are -# piping the output of rez, but will eventually be printing to a tty later. +# piping the output of rez, but will eventually be printing to a ``tty`` later. # When force is used, will generally be set through an environment variable, eg: # -# echo $(REZ_COLOR_ENABLED=force python -c "from rez.utils.colorize import Printer, local; Printer()('foo', local)") - +# .. code-block:: text +# +# echo $(REZ_COLOR_ENABLED=force python -c "from rez.utils.colorize import Printer, local; Printer()('foo', local)") +# # TODO: Colorama is documented to work on Windows and trivial test case # proves this to be the case, but it doesn't work in Rez (with cmd.exe). -# If the initialise is called in sec/rez/__init__.py then it does work, +# If the initialise is called in src/rez/__init__.py then it does work, # however this is not always desirable. # As it does with with some Windows shells it can be enabled in rezconfig - color_enabled = (os.name == "posix") ### Do not move or delete this comment (__DOC_END__) diff --git a/src/rez/system.py b/src/rez/system.py index 3cea0625f..4a9fed1e3 100644 --- a/src/rez/system.py +++ b/src/rez/system.py @@ -24,21 +24,18 @@ def rez_version(self): @cached_property def platform(self): """Get the current platform. - @returns The current platform. Examples: - linux - windows - osx + Returns: + The current platform (windows, linux, osx, etc). """ return platform_.name @cached_property def arch(self): """Get the current architecture. - @returns The current architecture. Examples: - x86_64 - i386 + Returns: + The current architecture (x86_64, i386, etc). """ r = platform_.arch return self._make_safe_version_string(r) @@ -46,20 +43,18 @@ def arch(self): @cached_property def os(self): """Get the current operating system. - @returns The current operating system. Examples: - Ubuntu-12.04 - CentOS-5.4 - windows-6.1.7600.sp1 - osx-10.6.2 + Returns: + The current operating system (Ubuntu-22.04, CentOS-7.8, windows-6.1.7600.sp1, etc). """ r = platform_.os return self._make_safe_version_string(r) @cached_property def variant(self): - """Returns a list of the form ["platform-X", "arch-X", "os-X"] suitable - for use as a variant in a system-dependent package.""" + """Returns a list of the form ``["platform-X", "arch-X", "os-X"]`` suitable + for use as a variant in a system-dependent package. + """ return ["platform-%s" % self.platform, "arch-%s" % self.arch, "os-%s" % self.os] @@ -68,10 +63,9 @@ def variant(self): @cached_property def shell(self): """Get the current shell. - @returns The current shell this process is running in. Examples: - bash - tcsh + Returns: + The current shell this process is running in (bash, tcsh, pwsh, etc). """ from rez.shells import get_shell_types shells = set(get_shell_types()) @@ -172,7 +166,7 @@ def home(self): @cached_property def fqdn(self): """ - @returns Fully qualified domain name, eg 'somesvr.somestudio.com' + Returns the fully qualified domain name (FQDN) of the current machine, eg ``somesvr.somestudio.com``. """ import socket return socket.getfqdn() @@ -180,7 +174,7 @@ def fqdn(self): @cached_property def hostname(self): """ - @returns The machine hostname, eg 'somesvr' + Returns the machine hostname, eg ``somesvr``. """ import socket return socket.gethostname() @@ -188,7 +182,7 @@ def hostname(self): @cached_property def domain(self): """ - @returns The domain, eg 'somestudio.com' + Returns the domain, eg ``somestudio.com``. """ try: return self.fqdn.split('.', 1)[1] @@ -249,14 +243,11 @@ def is_production_rez_install(self): @property def selftest_is_running(self): - """Return True if tests are running via rez-selftest tool.""" + """Return True if tests are running via ``rez-selftest`` tool.""" return os.getenv("__REZ_SELFTEST_RUNNING") == "1" def get_summary_string(self): """Get a string summarising the state of Rez as a whole. - - Returns: - String. """ from rez.plugin_managers import plugin_manager @@ -274,8 +265,8 @@ def clear_caches(self, hard=False): Args: hard (bool): Perform a 'hard' cache clear. This just means that the - memcached cache is also cleared. Generally this is not needed - - this option is for debugging purposes. + memcached cache is also cleared. Generally this is not needed. + This option is for debugging purposes. """ from rez.package_repository import package_repository_manager from rez.utils.memcached import memcached_client diff --git a/src/rez/utils/data_utils.py b/src/rez/utils/data_utils.py index 333418b5f..c3a8a4ea1 100644 --- a/src/rez/utils/data_utils.py +++ b/src/rez/utils/data_utils.py @@ -7,6 +7,7 @@ """ import os.path import json +import functools from rez.vendor.schema.schema import Schema, Optional from threading import Lock @@ -240,6 +241,8 @@ class cached_property(object): """ def __init__(self, func, name=None): self.func = func + # Make sure that Sphinx autodoc can follow and get the docstring from our wrapped function. + functools.update_wrapper(self, func) self.name = name or func.__name__ def __get__(self, instance, owner=None): @@ -254,6 +257,10 @@ def __get__(self, instance, owner=None): % (self.name, instance)) return result + # This is to silence Sphinx that complains that cached_property is not a callable. + def __call__(self): + raise RuntimeError("@cached_property should not be called.") + @classmethod def uncache(cls, instance, name): if hasattr(instance, name):