From 7be78f90a3c6c8e61dc4b1a6631a4255af08ce89 Mon Sep 17 00:00:00 2001 From: Tyler Hughes Date: Mon, 26 Feb 2024 09:57:44 -0500 Subject: [PATCH 1/8] adds new adjoint notebooks --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index a4cb0db03..42b0f00e2 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit a4cb0db03b0add6b64b9dff145fce06b41024acc +Subproject commit 42b0f00e2ecd2fb83f2b094b0035278b2e3e427a From 1fba4523e065fc6e4afd1489c20daad9a7d721cb Mon Sep 17 00:00:00 2001 From: Tyler Hughes Date: Wed, 28 Feb 2024 11:23:23 -0500 Subject: [PATCH 2/8] updates to adjoint --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index 42b0f00e2..e94ddf828 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit 42b0f00e2ecd2fb83f2b094b0035278b2e3e427a +Subproject commit e94ddf828555b04996c4d2654f94f188b04eb956 From 4c244e04981612be072f376b07fe0a561e5689be Mon Sep 17 00:00:00 2001 From: daquintero Date: Thu, 29 Feb 2024 14:27:27 +0100 Subject: [PATCH 3/8] UPDATE: Notebooks submodule? --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index e94ddf828..a32264b93 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit e94ddf828555b04996c4d2654f94f188b04eb956 +Subproject commit a32264b93609c39ec03333c947e40b70173c47e9 From 954e6d0c5b01dac8b4267848e5eab304f57b00e9 Mon Sep 17 00:00:00 2001 From: daquintero Date: Thu, 29 Feb 2024 19:04:32 +0100 Subject: [PATCH 4/8] FIX: Don't build _faq --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 064396d80..552ebff36 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,7 +74,7 @@ copybutton_prompt_is_regexp = True custom_sitemap_excludes = [r"/notebooks/"] # divparams_enable_postprocessing = True # TODO FIX -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints", "faq/_faqs/*"] extensions = [ "IPython.sphinxext.ipython_directive", "IPython.sphinxext.ipython_console_highlighting", From 948c17e2ca2bd5687022e8b69b39e2c9d35decc1 Mon Sep 17 00:00:00 2001 From: Tyler Hughes Date: Fri, 1 Mar 2024 08:57:26 -0500 Subject: [PATCH 5/8] docs update mar 1 --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index a32264b93..69528468c 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit a32264b93609c39ec03333c947e40b70173c47e9 +Subproject commit 69528468c4adb4fa46f08c829ee8304579d54cdb From 3525fc56906a3d5bfc8c048e0caac0690397bd93 Mon Sep 17 00:00:00 2001 From: daquintero Date: Tue, 5 Mar 2024 14:47:30 +0100 Subject: [PATCH 6/8] FEAT: Update notebooks to fix missing links :hammer: --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index 69528468c..66221f745 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit 69528468c4adb4fa46f08c829ee8304579d54cdb +Subproject commit 66221f745c7a44e3444c7bcc2916e534408ede02 From 16a6f2b7f161d5237238bf4db11f570315340cf2 Mon Sep 17 00:00:00 2001 From: daquintero Date: Tue, 5 Mar 2024 15:01:10 +0100 Subject: [PATCH 7/8] FEAT: Update submodule states --- docs/notebooks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notebooks b/docs/notebooks index 66221f745..8fc320014 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit 66221f745c7a44e3444c7bcc2916e534408ede02 +Subproject commit 8fc320014572333320397026d2d2b0fad1e924f7 From a8caa86e9aafde65e1a93f2bcfbccf5e276d337e Mon Sep 17 00:00:00 2001 From: daquintero Date: Tue, 2 Apr 2024 13:31:16 +0200 Subject: [PATCH 8/8] :book: DOCS: Release documentation improvement flow --- .github/workflows/release.yml | 24 ++++++++-- docs/development/installation.rst | 19 +++++--- docs/development/project_structure.rst | 33 +++++++++++++ docs/development/recommendations.rst | 55 ++++++++++++++++++++-- docs/development/release/documentation.rst | 26 ++++++++++ docs/development/release/flow.rst | 15 +++--- docs/development/release/index.rst | 7 +-- docs/development/release/requirements.rst | 15 ------ docs/development/release/version.rst | 41 +++------------- docs/development/usage.rst | 2 +- docs/notebooks | 2 +- 11 files changed, 165 insertions(+), 74 deletions(-) create mode 100644 docs/development/release/documentation.rst delete mode 100644 docs/development/release/requirements.rst diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ccc7852d7..faa796b46 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,9 @@ jobs: github-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} - name: Exit if any RC release if: contains(${{ github.ref }}, 'rc') == false uses: everlytic/branch-merge@1.1.2 @@ -20,7 +22,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} source_ref: ${{ github.ref }} target_branch: "latest" - commit_message_template: 'RELEASE: :shipit: :boom: :tada: Merged {source_ref} into target {target_branch}' + commit_message_template: ':tada: RELEASE: Merged {source_ref} into target {target_branch}' - name: Release uses: softprops/action-gh-release@v1 with: @@ -30,7 +32,9 @@ jobs: pypi-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} - uses: actions/setup-python@v2 - name: Install dependencies run: | @@ -43,3 +47,17 @@ jobs: run: | python -m build python -m twine upload --repository pypi dist/* + sync_to_develop: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: "latest" + - name: Exit if any RC release + if: contains(${{ github.ref }}, 'rc') == false + uses: everlytic/branch-merge@1.1.2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + source_ref: "latest" + target_branch: "develop" + commit_message_template: ':tada: RELEASE: Synced latest into develop' diff --git a/docs/development/installation.rst b/docs/development/installation.rst index 0458c9e1b..d485b788e 100644 --- a/docs/development/installation.rst +++ b/docs/development/installation.rst @@ -1,17 +1,22 @@ Installation ============== +Beginners Guide +^^^^^^^^^^^^^^^ + + + The Fast Lane ^^^^^^^^^^^^^ Maybe you already have ``tidy3d`` installed in some form. After installing version ``tidy3d>=2.6``, you can use a few terminal commands to set you up on the correct environment and perform common development tasks. Just run in your terminal, :code:`tidy3d develop` to get the latest list of commands. -It does not matter how you have installed ``tidy3d`` before as long as you have any form of `tidy3d>=2.6`` in your environment. This can help you transition from a standard user installation to a development environment installation. +It does not matter how you have installed ``tidy3d`` before as long as you have any form of ``tidy3d>=2.6`` in your environment. This can help you transition from a standard user installation to a development environment installation. Quick Start '''''''''''' -Instructions for anyone who wants to test the new development flow before it gets included as part of the pre-release: +Instructions for anyone who wants to migrate to the development flow from a version before 2.6: For ubuntu: @@ -30,8 +35,8 @@ For ubuntu: Now you can run the following ``tidy3d`` cli commands to test them. -Automatic Environment Installation (Beta) -''''''''''''''''''''''''''''''''''''''''''''' +Automatic Environment Installation *Beta* +'''''''''''''''''''''''''''''''''''''''''' If you are transitioning from the old development flow, to this new one, there are a list of commands you can run to make your life easier and set you up well: @@ -40,7 +45,7 @@ If you are transitioning from the old development flow, to this new one, there a # Automatically check and install requirements like pipx, poetry, pandoc tidy3d develop install-dev-environment -Note that this is just a automatic script implementation of the `The Detailed Lane`_ instructions. It is intended to help you and raise warnings with suggestions of how to fix an environment setup issue. You do not have to use this helper function and can just follow the instructions in `The Detailed Lane`_. All commands are echo-ed in the terminal so you will be able to observe and reproduce what is failing if you desire. +Note that this is just a automatic script implementation of the :ref:`The Detailed Lane` instructions. It is intended to help you and raise warnings with suggestions of how to fix an environment setup issue. You do not have to use this helper function and can just follow the instructions in :ref:`The Detailed Lane`. All commands are echo-ed in the terminal so you will be able to observe and reproduce what is failing if you desire. The way this command works is dependent on the operating system you are running. There are some prerequisites for each platform, but the command line tool will help you identify and install the tools it requires. You should rerun the command after you have installed any prerequisite as it will just progress with the rest of the tools installation. If you already have the tool installed, it will verify that it conforms to the supported versions. @@ -49,14 +54,14 @@ This command will first check if you already have installed the development requ Environment Verification '''''''''''''''''''''''' -If you rather install ``poetry``, ``pipx`` and ``pandoc`` yourself, you can run the following command to verify that your environment conforms to the reproducible development environment which would be equivalent to the one installed automatically above and described in `The Detailed Lane`_. +If you rather install ``poetry``, ``pipx`` and ``pandoc`` yourself, you can run the following command to verify that your environment conforms to the reproducible development environment which would be equivalent to the one installed automatically above and described in :ref:`The Detailed Lane`. .. code:: tidy3d develop verify-dev-environment -.. _detailed_lane: +.. _The Detailed Lane:: The Detailed Lane ^^^^^^^^^^^^^^^^^ diff --git a/docs/development/project_structure.rst b/docs/development/project_structure.rst index 9abbbde51..736ef5512 100644 --- a/docs/development/project_structure.rst +++ b/docs/development/project_structure.rst @@ -11,6 +11,9 @@ As of ``tidy3d>=2.6``, the frontend has been restructured to improve the develop notebooks/ # Git submodule repository # Checks out github.com/flexcompute/tidy3d-notebooks + faq/ + # Git submodule repository + # Checks out github.com/flexcompute/tidy3d-faq tests/ # pytest source and docs # pytest notebooks @@ -27,6 +30,36 @@ It is important to note the new tools we are using to manage our development env - ``poetry`` - ``pipx`` +Important Branches +------------------ + +We currently have *three* main branches that have to be kept track of when creating a release, each with different functionality. + +.. list-table:: Project Branches + :header-rows: 1 + :widths: 10 45 45 + + * - Name + - Description + - Caveats + * - ``latest`` + - Contains the latest version of the docs. Version release tags are created from this branch. + - Feature PRs should not be made to this branch as will cause divergence. Only in important documentation patches. + * - ``develop`` + - Contains the "staging" version of the project. Patch versions and development occurs from these branches. + - Docs PRs that are non-crucial for the current version should be made to this branch. + * - ``pre/^*`` + - Contains the next version of the project. + - Documentation and source code that will only go live in the next version should be updated here. + +Sometimes, hopefully infrequently, the `latest` and `develop` branches might diverge. +It is important to bring them back together. However, what happens if we rebase develop into latest? + +It could be argued that all the commits in the `latest` branch should have constructed within the `develop` branch. +Then, there is the question if we want to maintain the commit history accordingly. If we just want to maintain the content, +then rebasing and fixing up all the branches works fine. The problem with a merge commit is that it inserts the commits at the historical period in which they were made, rather than the commit period in which we desire to add them. +Hence, it makes sense to merge the `develop` and `latest` branches in order to maintain the same history, assuming the commits should in theory have been in both branches. + diff --git a/docs/development/recommendations.rst b/docs/development/recommendations.rst index b4d47b273..e2964df19 100644 --- a/docs/development/recommendations.rst +++ b/docs/development/recommendations.rst @@ -1,8 +1,56 @@ Recommendations ================= +Standardised Commit Messages +---------------------------- + +Now, realistically, this is a `matter of preference `_. +However, it could be argued there is something nice in having standard commit messages that can be easily searched through +to understand the role of each change, and also render nicely in the git history. Also, having a commit standard maybe makes people +looking through our code feel that we take pride in our work and also like to make it nice. It is debatable whether this is a way to do this, however, we can update these recommendations depending on how we consider best. + +However, if we do decide to commit with emojis, I believe it would be worth having a standard, so that it does not get polluted with different emojis (as I have been guilty of before) and also as can be seen in other open-source projects. + +.. list-table:: Commit Standard + :header-rows: 1 + :widths: 25 15 15 45 + + * - Purpose + - Emoji + - Types + - Example + * - |:sparkles:| New Feature + - ``:sparkles:`` + - ``FEAT:`` + - ``:sparkles: FEAT: `` + * - |:wrench:| Fix Broken Code + - ``:wrench:`` + - ``FIX:`` + - ``:wrench: FIX: `` + * - |:package:| Packaging-related + - ``:package:`` + - ``BUILD:`` + - ``:package: BUILD: `` + * - |:book:| Documentation-related + - ``:book:`` + - ``DOCS:`` + - ``:book: DOCS: `` + * - |:rocket:| Refactor code + - ``:rocket:`` + - ``REFC:`` + - ``:rocket: REFC: `` + * - |:test_tube:| Testing related + - ``:test_tube:`` + - ``TEST:`` + - ``:test_tube: TEST: `` + * - |:tada:| Release commit + - ``:tada:`` + - ``RELEASE:`` + - ``:tada: RELEASE: `` + + Package Speedup Best Practices -'''''''''''''''''''''''''''''' +---------------------------- ``tidy3d`` is a pretty big project already, and will get bigger. We want to optimise the performance of the codebase throughout the multiple operations that we perform. @@ -29,7 +77,8 @@ This is because the latter will import the entire package, which is not necessar Managing Optional Dependencies On-The-Fly -'''''''''''''''''''''''''''''''''''''''''' +---------------------------- + If you look within ``pyproject.toml``, it is possible to see that we have different packages relating to different functionalities that are optional. @@ -37,7 +86,7 @@ Some examples from these are ``[vtk, jax, trimesh, gdstk, gdspy]`` etc. What we Benchmarking Package Import -'''''''''''''''''''''''''''' +---------------------------- We want to make the tidy3d package be as light as possible for a given set of operations. As such, it is important to understand exactly where a given set of operations is expending computational power. diff --git a/docs/development/release/documentation.rst b/docs/development/release/documentation.rst new file mode 100644 index 000000000..8b8050ce4 --- /dev/null +++ b/docs/development/release/documentation.rst @@ -0,0 +1,26 @@ +Documentation Release +--------------------- + +The `tidy3d-docs` repository automatically mirrors the `tidy3d` repository. Specifically, these branches are automatically synced. + +- main +- latest +- develop +- 'pre/*' +- 'v*' + +These branches are synced to the tidy3d-docs repo through the sync-readthedocs-repo Github action. +You can read the latest versions synced in the action file. +However, you need to configure how they appear in the documentation build in the readthedocs admin page. +Only latest is the public version, others are private. + +The `latest` branch holds the state of the docs that we want to host in `latest` version on the website. These are the latest docs (including new notebooks, typo fixes, etc.) related to the last official release (not pre-release). + +The `stable` version of the docs on our website is built based on the last version tag which is not a pre-release tag (no `rc` ending). + +Hot Fix & Submodule Updates +''''''''''''''''''''''''''' + +To make a “hot fix” (eg fix a typo, add a notebook, update the release FAQ), just update the ``latest`` branch in ``tidy3d`` repo. This should automatically sync to `tidy3d-docs`, and trigger a docs rebuild. **However, we should avoid this as this will cause the ``develop`` and ``latest branches`` to diverge.** Ideally, these hot fixes could wait until the next pre/post-release to be propagated through. + +NOTE: To avoid conflicts, ideally we should only update ``latest`` by merging ``develop`` in it, or at the very least we should make sure changes are propagated to both branches. \ No newline at end of file diff --git a/docs/development/release/flow.rst b/docs/development/release/flow.rst index c1b3a0bf2..accac7adf 100644 --- a/docs/development/release/flow.rst +++ b/docs/development/release/flow.rst @@ -1,12 +1,16 @@ Feature Development Workflow ------------------------------ +Currently most of our release development flow is made under the latest ``pre/*`` branch under the main frontend +tidy3d repository. You want to fork from this latest branch to develop your feature in order for it to be included +under that release. + We are using a variation of the `gitflow workflow `__ - so this is the first thing to familiarize yourselves with. The splitting of branches into ``main``, ``develop`` and separate feature branches is as explained there. Most importantly, **all contributions -should happen through a PR from a feature branch into the ``develop`` +should happen through a PR from a feature branch into the develop branch.** The extra step that we have in our workflow is to @@ -16,11 +20,8 @@ history clean, but it does require a little more work. As an extra advantage, once you get the hang of rebasing it also becomes a very useful tool to prune your commits and write more meaningful commit messages when you’re done with the work. The main purpose of this page -is to give an example of the workflow. - - for more information on the difference between rebasing vs merging, - see this - `article `__. +is to give an example of the workflow. For more information on the difference between rebasing vs merging, +see this `article `__. The first thing to do when starting a new batch of work is to start from a clean branch on your machine. @@ -39,7 +40,7 @@ Create your feature rebase Before rebasing, you should make sure you have the latest version of ``develop``, in case other work has been merged meanwhile. -:: +.. code-block:: bash git checkout develop git pull origin develop diff --git a/docs/development/release/index.rst b/docs/development/release/index.rst index 9abe36d03..d2958eaa7 100644 --- a/docs/development/release/index.rst +++ b/docs/development/release/index.rst @@ -1,7 +1,7 @@ Release Flow ============== -Currently most of our release development flow is made under the latest ``pre/*`` branch under the main frontend tidy3d repository. You want to fork from this latest branch to develop your feature in order for it to be included under that release. +This page contains all the relevant information relating to each version release process for ``tidy3d``. .. You just need to make sure that the ```` branches of both ``tidy3d/`` and ``tidy3d-notebooks/`` repositories within the ``./`` and ``./docs/notebooks/`` directories are updated. The ``readthedocs`` documentation will be automatically updated through the ``sync-readthedocs-repo`` Github action. @@ -11,10 +11,11 @@ Currently most of our release development flow is made under the latest ``pre/*` flow version - requirements + documentation notebooks + .. include:: /development/release/flow.rst .. include:: /development/release/version.rst -.. include:: /development/release/requirements.rst +.. include:: /development/release/documentation.rst .. include:: /development/release/notebooks.rst \ No newline at end of file diff --git a/docs/development/release/requirements.rst b/docs/development/release/requirements.rst deleted file mode 100644 index a41c20257..000000000 --- a/docs/development/release/requirements.rst +++ /dev/null @@ -1,15 +0,0 @@ -Deployment Requirements ------------------------ - -**Documentation** - -When a new release is created, it is necessary to create a "mirror" branch under the `tidy3d-docs` repository in order for it to build. If this is not there an error such as the following might appear under the `sync-readthedocs-repo` Github action. You will also need to do this for full releases. - -.. code-block:: - - /usr/bin/git worktree remove github-pages-deploy-action-temp-deployment-folder --force - Error: The deploy step encountered an error: There was an error creating the worktree: The process '/usr/bin/git' failed with exit code 128 ❌ ❌ - Notice: Deployment failed! ❌ - -**Coverage** - diff --git a/docs/development/release/version.rst b/docs/development/release/version.rst index e482e6409..d13ffa44f 100644 --- a/docs/development/release/version.rst +++ b/docs/development/release/version.rst @@ -1,39 +1,12 @@ -Version Release ----------------- +Releasing a new ``tidy3d`` version +---------------------------------- -There's a really nice tool to manage the releases which is called ``bump-my-version``, which has already been configured. -It's really easy to use. +This document contains the relevant information to create and publish a new tidy3d version. -Docs of the tool available here https://callowayproject.github.io/bump-my-version/#create-a-default-configuration +Version Information Management +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You need to have installed the development installation of ``tidy3d``: +The ``pyproject.toml`` is declarative (ie static) and provides information to the packaging tools like PyPi on what version is ``tidy3d``. However, we also have a ``version.py`` file so that we can dynamically query ``tidy3d.__version__`` within our python version. These two files need to be kept with the same version. This is achieved by using the ``bump-my-version`` utility as described in the following section. **These files should not be manually updated.** -.. code-block:: bash - - poetry install -E dev - -Now, make sure the git index is clean, and you're ready to release. Test that the release would update the correct files: - -.. code-block:: bash - - poetry run bump-my-version show-bump - -When you want to bump the version, you only have to do: - -.. code-block:: bash - - poetry run bump-my-version bump - -An example of the ``.bump-my-version.toml`` is as below, and just configures the files to be updated: - -.. code-block:: bash - - 2024-03-20 16:58:02 ⌚ dxps in ~/flexcompute/tidy3d - ± |dario/2.6.2/fix_versioning S:1 U:3 ?:2 ✗| → poetry run bump-my-version show-bump - Specified version (2.6.1) does not match last tagged version (2.6.0) - 2.6.1 ── bump ─┬─ major ─ 3.0.0 - ├─ minor ─ 2.7.0 - ├─ patch ─ 2.6.2 - ├─ pre_l ─ 2.6.1-rc0 - ╰─ pre_n ─ 2.6.1-dev1 +The configuration of the way the version bumping occurs is described in the ``pyproject.toml``. diff --git a/docs/development/usage.rst b/docs/development/usage.rst index 33db05944..1bacada72 100644 --- a/docs/development/usage.rst +++ b/docs/development/usage.rst @@ -4,7 +4,7 @@ Using the Development Flow Developing ``tidy3d`` with ``poetry`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``poetry`` is an incredibly powerful tool for reproducible package development environments and dependency management. TODO add link. +`poetry `_ is an incredibly powerful tool for reproducible package development environments and dependency management. If you are developing ``tidy3d``, we recommend you work within the configured ``poetry`` environment defined by ``poetry.lock``. The way to install this environment is simple: diff --git a/docs/notebooks b/docs/notebooks index f43ed15d7..0aa01ae48 160000 --- a/docs/notebooks +++ b/docs/notebooks @@ -1 +1 @@ -Subproject commit f43ed15d7f13db8aecc5f4020254e0876c37b338 +Subproject commit 0aa01ae4844ba6db496577601c369de143cbdd82