From 5f81f7bc8c0ee8031e339ec4873fb81ed5e9bbf6 Mon Sep 17 00:00:00 2001 From: Gabriel Stefanini Vicente Date: Wed, 1 Nov 2023 21:50:59 -0400 Subject: [PATCH] Clean up content (#7) --- .github/FUNDING.yml | 12 -- .github/dependabot.yml | 6 - .github/init.sh | 68 ------ .github/release_message.sh | 3 - .github/rename_project.sh | 36 ---- .github/workflows/main.yml | 92 -------- .github/workflows/release.yml | 50 ----- .github/workflows/rename_project.yml | 42 ---- ABOUT_THIS_TEMPLATE.md | 198 ------------------ CONTRIBUTING.md | 114 ++-------- Containerfile | 5 - HISTORY.md | 13 -- MANIFEST.in | 5 - Makefile | 122 ----------- README.md | 88 ++++---- ntl_usa.png => docs/ntl_usa.png | Bin raster_example.png => docs/raster_example.png | Bin trends_example.png => docs/trends_example.png | Bin mkdocs.yml | 2 - requirements-test.txt | 10 - requirements.txt | 22 -- 21 files changed, 67 insertions(+), 821 deletions(-) delete mode 100644 .github/FUNDING.yml delete mode 100644 .github/dependabot.yml delete mode 100755 .github/init.sh delete mode 100755 .github/release_message.sh delete mode 100755 .github/rename_project.sh delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/rename_project.yml delete mode 100644 ABOUT_THIS_TEMPLATE.md delete mode 100644 Containerfile delete mode 100644 HISTORY.md delete mode 100644 MANIFEST.in delete mode 100644 Makefile rename ntl_usa.png => docs/ntl_usa.png (100%) rename raster_example.png => docs/raster_example.png (100%) rename trends_example.png => docs/trends_example.png (100%) delete mode 100644 mkdocs.yml delete mode 100644 requirements-test.txt delete mode 100644 requirements.txt diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 1e9d2a1..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: [rochacbruno] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 120c689..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" \ No newline at end of file diff --git a/.github/init.sh b/.github/init.sh deleted file mode 100755 index 8a32ee3..0000000 --- a/.github/init.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -overwrite_template_dir=0 - -while getopts t:o flag -do - case "${flag}" in - t) template=${OPTARG};; - o) overwrite_template_dir=1;; - esac -done - -if [ -z "${template}" ]; then - echo "Available templates: flask" - read -p "Enter template name: " template -fi - -repo_urlname=$(basename -s .git `git config --get remote.origin.url`) -repo_name=$(basename -s .git `git config --get remote.origin.url` | tr '-' '_' | tr '[:upper:]' '[:lower:]') -repo_owner=$(git config --get remote.origin.url | awk -F ':' '{print $2}' | awk -F '/' '{print $1}') -echo "Repo name: ${repo_name}" -echo "Repo owner: ${repo_owner}" -echo "Repo urlname: ${repo_urlname}" - -if [ -f ".github/workflows/rename_project.yml" ]; then - .github/rename_project.sh -a "${repo_owner}" -n "${repo_name}" -u "${repo_urlname}" -d "Awesome ${repo_name} created by ${repo_owner}" -fi - -function download_template { - rm -rf "${template_dir}" - mkdir -p .github/templates - git clone "${template_url}" "${template_dir}" -} - -echo "Using template:${template}" -template_url="https://github.com/rochacbruno/${template}-project-template" -template_dir=".github/templates/${template}" -if [ -d "${template_dir}" ]; then - # Template directory already exists - if [ "${overwrite_template_dir}" -eq 1 ]; then - # user passed -o flag, delete and re-download - echo "Overwriting ${template_dir}" - download_template - else - # Ask user if they want to overwrite - echo "Directory ${template_dir} already exists." - read -p "Do you want to overwrite it? [y/N] " -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]]; then - echo "Overwriting ${template_dir}" - download_template - else - # User decided not to overwrite - echo "Using existing ${template_dir}" - fi - fi -else - # Template directory does not exist, download it - echo "Downloading ${template_url}" - download_template -fi - -echo "Applying ${template} template to this project"} -./.github/templates/${template}/apply.sh -a "${repo_owner}" -n "${repo_name}" -u "${repo_urlname}" -d "Awesome ${repo_name} created by ${repo_owner}" - -# echo "Removing temporary template files" -# rm -rf .github/templates/${template} - -echo "Done! review, commit and push the changes" diff --git a/.github/release_message.sh b/.github/release_message.sh deleted file mode 100755 index f5a9062..0000000 --- a/.github/release_message.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -previous_tag=$(git tag --sort=-creatordate | sed -n 2p) -git shortlog "${previous_tag}.." | sed 's/^./ &/' diff --git a/.github/rename_project.sh b/.github/rename_project.sh deleted file mode 100755 index 3a71f82..0000000 --- a/.github/rename_project.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -while getopts a:n:u:d: flag -do - case "${flag}" in - a) author=${OPTARG};; - n) name=${OPTARG};; - u) urlname=${OPTARG};; - d) description=${OPTARG};; - esac -done - -echo "Author: $author"; -echo "Project Name: $name"; -echo "Project URL name: $urlname"; -echo "Description: $description"; - -echo "Renaming project..." - -original_author="ramarty" -original_name="blackmarblepy" -original_urlname="blackmarblepy" -original_description="Awesome blackmarblepy created by ramarty" -# for filename in $(find . -name "*.*") -for filename in $(git ls-files) -do - sed -i "s/$original_author/$author/g" $filename - sed -i "s/$original_name/$name/g" $filename - sed -i "s/$original_urlname/$urlname/g" $filename - sed -i "s/$original_description/$description/g" $filename - echo "Renamed $filename" -done - -mv blackmarblepy $name - -# This command runs only once on GHA! -rm -rf .github/template.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5e82e08..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,92 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: [ main ] - pull_request: - branches: [ main ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - linter: - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install project - run: make install - - name: Run linter - run: make lint - - tests_linux: - needs: linter - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install project - run: make install - - name: Run tests - run: make test - - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 - # with: - # fail_ci_if_error: true - - tests_mac: - needs: linter - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install project - run: make install - - name: Run tests - run: make test - - tests_win: - needs: linter - strategy: - fail-fast: false - matrix: - python-version: [3.9] - os: [windows-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install Pip - run: pip install --user --upgrade pip - - name: Install project - run: pip install -e .[test] - - name: run tests - run: pytest -s -vvvv -l --tb=long tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c42c994..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Upload Python Package - -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - '*' # Push events to matching v*, i.e. v1.0, v20.15.10 - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - release: - name: Create Release - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v3 - with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 - - name: Generate Changelog - run: .github/release_message.sh > release_message.md - - name: Release - uses: softprops/action-gh-release@v1 - with: - body_path: release_message.md - - deploy: - needs: release - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/rename_project.yml b/.github/workflows/rename_project.yml deleted file mode 100644 index ae6d2eb..0000000 --- a/.github/workflows/rename_project.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Rename the project from template - -on: [push] - -permissions: write-all - -jobs: - rename-project: - if: ${{ !contains (github.repository, '/python-project-template') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 - ref: ${{ github.head_ref }} - - - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}' | tr '-' '_' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_URLNAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_OWNER=$(echo '${{ github.repository }}' | awk -F '/' '{print $1}')" >> $GITHUB_ENV - shell: bash - - - name: Is this still a template - id: is_template - run: echo "::set-output name=is_template::$(ls .github/template.yml &> /dev/null && echo true || echo false)" - - - name: Rename the project - if: steps.is_template.outputs.is_template == 'true' - run: | - echo "Renaming the project with -a(author) ${{ env.REPOSITORY_OWNER }} -n(name) ${{ env.REPOSITORY_NAME }} -u(urlname) ${{ env.REPOSITORY_URLNAME }}" - .github/rename_project.sh -a ${{ env.REPOSITORY_OWNER }} -n ${{ env.REPOSITORY_NAME }} -u ${{ env.REPOSITORY_URLNAME }} -d "Awesome ${{ env.REPOSITORY_NAME }} created by ${{ env.REPOSITORY_OWNER }}" - - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "✅ Ready to clone and code." - # commit_options: '--amend --no-edit' - push_options: --force diff --git a/ABOUT_THIS_TEMPLATE.md b/ABOUT_THIS_TEMPLATE.md deleted file mode 100644 index 43328c5..0000000 --- a/ABOUT_THIS_TEMPLATE.md +++ /dev/null @@ -1,198 +0,0 @@ -# About this template - -Hi, I created this template to help you get started with a new project. - -I have created and maintained a number of python libraries, applications and -frameworks and during those years I have learned a lot about how to create a -project structure and how to structure a project to be as modular and simple -as possible. - -Some decisions I have made while creating this template are: - - - Create a project structure that is as modular as possible. - - Keep it simple and easy to maintain. - - Allow for a lot of flexibility and customizability. - - Low dependency (this template doesn't add dependencies) - -## Structure - -Lets take a look at the structure of this template: - -```text -├── Containerfile # The file to build a container using buildah or docker -├── CONTRIBUTING.md # Onboarding instructions for new contributors -├── docs # Documentation site (add more .md files here) -│   └── index.md # The index page for the docs site -├── .github # Github metadata for repository -│   ├── release_message.sh # A script to generate a release message -│   └── workflows # The CI pipeline for Github Actions -├── .gitignore # A list of files to ignore when pushing to Github -├── HISTORY.md # Auto generated list of changes to the project -├── LICENSE # The license for the project -├── Makefile # A collection of utilities to manage the project -├── MANIFEST.in # A list of files to include in a package -├── mkdocs.yml # Configuration for documentation site -├── blackmarblepy # The main python package for the project -│   ├── base.py # The base module for the project -│   ├── __init__.py # This tells Python that this is a package -│   ├── __main__.py # The entry point for the project -│   └── VERSION # The version for the project is kept in a static file -├── README.md # The main readme for the project -├── setup.py # The setup.py file for installing and packaging the project -├── requirements.txt # An empty file to hold the requirements for the project -├── requirements-test.txt # List of requirements for testing and devlopment -├── setup.py # The setup.py file for installing and packaging the project -└── tests # Unit tests for the project (add mote tests files here) - ├── conftest.py # Configuration, hooks and fixtures for pytest - ├── __init__.py # This tells Python that this is a test package - └── test_base.py # The base test case for the project -``` - -## FAQ - -Frequent asked questions. - -### Why this template is not using [Poetry](https://python-poetry.org/) ? - -I really like Poetry and I think it is a great tool to manage your python projects, -if you want to switch to poetry, you can run `make switch-to-poetry`. - -But for this template I wanted to keep it simple. - -Setuptools is the most simple and well supported way of packaging a Python project, -it doesn't require extra dependencies and is the easiest way to install the project. - -Also, poetry doesn't have a good support for installing projects in development mode yet. - -### Why the `requirements.txt` is empty ? - -This template is a low dependency project, so it doesn't have any extra dependencies. -You can add new dependencies as you will or you can use the `make init` command to -generate a `requirements.txt` file based on the template you choose `flask, fastapi, click etc`. - -### Why there is a `requirements-test.txt` file ? - -This file lists all the requirements for testing and development, -I think the development environment and testing environment should be as similar as possible. - -Except those tools that are up to the developer choice (like ipython, ipdb etc). - -### Why the template doesn't have a `pyproject.toml` file ? - -It is possible to run `pip install https://github.com/name/repo/tarball/main` and -have pip to download the package direcly from Git repo. - -For that to work you need to have a `setup.py` file, and `pyproject.toml` is not -supported for that kind of installation. - -I think it is easier for example you want to install specific branch or tag you can -do `pip install https://github.com/name/repo/tarball/{TAG|REVISON|COMMIT}` - -People automating CI for your project will be grateful for having a setup.py file - -### Why isn't this template made as a cookiecutter template? - -I really like [cookiecutter](https://github.com/cookiecutter/cookiecutter) and it is a great way to create new projects, -but for this template I wanted to use the Github `Use this template` button, -to use this template doesn't require to install extra tooling such as cookiecutter. - -Just click on [Use this template](https://github.com/rochacbruno/python-project-template/generate) and you are good to go. - -The substituions are done using github actions and a simple sed script. - -### Why `VERSION` is kept in a static plain text file? - -I used to have my version inside my main module in a `__version__` variable, then -I had to do some tricks to read that version variable inside the setuptools -`setup.py` file because that would be available only after the installation. - -I decided to keep the version in a static file because it is easier to read from -wherever I want without the need to install the package. - -e.g: `cat blackmarblepy/VERSION` will get the project version without harming -with module imports or anything else, it is useful for CI, logs and debugging. - -### Why to include `tests`, `history` and `Containerfile` as part of the release? - -The `MANIFEST.in` file is used to include the files in the release, once the -project is released to PyPI all the files listed on MANIFEST.in will be included -even if the files are static or not related to Python. - -Some build systems such as RPM, DEB, AUR for some Linux distributions, and also -internal repackaging systems tends to run the tests before the packaging is performed. - -The Containerfile can be useful to provide a safer execution environment for -the project when running on a testing environment. - -I added those files to make it easier for packaging in different formats. - -### Why conftest includes a go_to_tmpdir fixture? - -When your project deals with file system operations, it is a good idea to use -a fixture to create a temporary directory and then remove it after the test. - -Before executing each test pytest will create a temporary directory and will -change the working directory to that path and run the test. - -So the test can create temporary artifacts isolated from other tests. - -After the execution Pytest will remove the temporary directory. - -### Why this template is not using [pre-commit](https://pre-commit.com/) ? - -pre-commit is an excellent tool to automate checks and formatting on your code. - -However I figured out that pre-commit adds extra dependency and it an entry barrier -for new contributors. - -Having the linting, checks and formatting as simple commands on the [Makefile](Makefile) -makes it easier to undestand and change. - -Once the project is bigger and complex, having pre-commit as a dependency can be a good idea. - -### Why the CLI is not using click? - -I wanted to provide a simple template for a CLI application on the project main entry point -click and typer are great alternatives but are external dependencies and this template -doesn't add dependencies besides those used for development. - -### Why this doesn't provide a full example of application using Flask or Django? - -as I said before, I want it to be simple and multipurpose, so I decided to not include -external dependencies and programming design decisions. - -It is up to you to decide if you want to use Flask or Django and to create your application -the way you think is best. - -This template provides utilities in the Makefile to make it easier to you can run: - -```bash -$ make init -Which template do you want to apply? [flask, fastapi, click, typer]? > flask -Generating a new project with Flask ... -``` - -Then the above will download the Flask template and apply it to the project. - -## The Makefile - -All the utilities for the template and project are on the Makefile - -```bash -❯ make -Usage: make - -Targets: -help: ## Show the help. -install: ## Install the project in dev mode. -fmt: ## Format code using black & isort. -lint: ## Run pep8, black, mypy linters. -test: lint ## Run tests and generate coverage report. -watch: ## Run tests on every change. -clean: ## Clean unused files. -virtualenv: ## Create a virtual environment. -release: ## Create a new tag for release. -docs: ## Build the documentation. -switch-to-poetry: ## Switch to poetry package manager. -init: ## Initialize the project based on an application template. -``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c4d3cfd..a9448b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,113 +1,45 @@ -# How to develop on this project +# CONTRIBUTING -blackmarblepy welcomes contributions from the community. +Thank you for considering contributing! We appreciate your interest in helping us improve our project. By contributing, you agree to abide by our [Code of Conduct](docs/CODE_OF_CONDUCT.md). -**You need PYTHON3!** +Please take a moment to review this document for important information on how to contribute effectively. -This instructions are for linux base systems. (Linux, MacOS, BSD, etc.) -## Setting up your own fork of this repo. +## How Can I Contribute? -- On github interface click on `Fork` button. -- Clone your fork of this repo. `git clone git@github.com:YOUR_GIT_USERNAME/blackmarblepy.git` -- Enter the directory `cd blackmarblepy` -- Add upstream repo `git remote add upstream https://github.com/ramarty/blackmarblepy` +There are several ways you can contribute to this project: -## Setting up your own virtual environment +- **Bug Reports:** If you encounter a bug or unexpected behavior, please open an issue on our GitHub issue tracker. Be sure to include as much detail as possible to help us identify and fix the problem. -Run `make virtualenv` to create a virtual environment. -then activate it with `source .venv/bin/activate`. +- **Feature Requests**: If you have an idea for a new feature or enhancement, please open an issue on our GitHub issue tracker and label it as a "feature request." Describe the feature and its use case in detail. -## Install the project in develop mode +- **Pull Requests:** If you'd like to contribute code or documentation changes, we encourage you to submit a pull request (PR). Please follow the guidelines outlined in the [Contributing Code](CONTRIBUTING.md) section below. -Run `make install` to install the project in develop mode. +- **Documentation:** If you find any errors or have suggestions for improving our documentation, you can submit changes directly through a pull request. -## Run the tests to ensure everything is working +- **Community Engagement:** Help answer questions and engage with other users and contributors on our GitHub Discussions (if applicable). -Run `make test` to run the tests. +## Contributing Code -## Create a new branch to work on your contribution +If you're contributing code, please follow these guidelines: -Run `git checkout -b my_contribution` +1. **Fork the Repository**: Click the "Fork" button on the top-right corner of this repository on GitHub. This will create a copy of the project in your GitHub account. -## Make your changes +2. **Create a Branch:** Create a new branch for your feature or bug fix. Use a clear and descriptive name for your branch, like `feature/my-new-feature` or `bugfix/issue-123`. -Edit the files using your preferred editor. (we recommend VIM or VSCode) +3. **Make Changes:** Make your code changes and ensure they adhere to our coding standards. -## Format the code +4. **Test:** Ensure that your changes do not break existing functionality and add tests for new features or bug fixes. -Run `make fmt` to format the code. +5. **Commit and Push:** Commit your changes with a clear and concise commit message. Reference any related issues or pull requests in your commit message. Push your branch to your forked repository on GitHub. -## Run the linter +6. **Create a Pull Request:** Open a pull request against the main branch of this repository. Provide a clear description of your changes and reference any relevant issues. Your PR will be reviewed by maintainers. -Run `make lint` to run the linter. +7. **Review and Iterate:** Expect feedback and be prepared to make additional changes if necessary. We may request changes, and once everything looks good, your PR will be merged. -## Test your changes +## Code of Conduct -Run `make test` to run the tests. +Please note that we have a [Code of Conduct](docs/CODE_OF_CONDUCT) in place. We expect all contributors to adhere to it, both in interactions within this project and in interactions with other project members. -Ensure code coverage report shows `100%` coverage, add tests to your PR. +## Licensing -## Build the docs locally - -Run `make docs` to build the docs. - -Ensure your new changes are documented. - -## Commit your changes - -This project uses [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/). - -Example: `fix(package): update setup.py arguments 🎉` (emojis are fine too) - -## Push your changes to your fork - -Run `git push origin my_contribution` - -## Submit a pull request - -On github interface, click on `Pull Request` button. - -Wait CI to run and one of the developers will review your PR. -## Makefile utilities - -This project comes with a `Makefile` that contains a number of useful utility. - -```bash -❯ make -Usage: make - -Targets: -help: ## Show the help. -install: ## Install the project in dev mode. -fmt: ## Format code using black & isort. -lint: ## Run pep8, black, mypy linters. -test: lint ## Run tests and generate coverage report. -watch: ## Run tests on every change. -clean: ## Clean unused files. -virtualenv: ## Create a virtual environment. -release: ## Create a new tag for release. -docs: ## Build the documentation. -switch-to-poetry: ## Switch to poetry package manager. -init: ## Initialize the project based on an application template. -``` - -## Making a new release - -This project uses [semantic versioning](https://semver.org/) and tags releases with `X.Y.Z` -Every time a new tag is created and pushed to the remote repo, github actions will -automatically create a new release on github and trigger a release on PyPI. - -For this to work you need to setup a secret called `PIPY_API_TOKEN` on the project settings>secrets, -this token can be generated on [pypi.org](https://pypi.org/account/). - -To trigger a new release all you need to do is. - -1. If you have changes to add to the repo - * Make your changes following the steps described above. - * Commit your changes following the [conventional git commit messages](https://www.conventionalcommits.org/en/v1.0.0/). -2. Run the tests to ensure everything is working. -4. Run `make release` to create a new tag and push it to the remote repo. - -the `make release` will ask you the version number to create the tag, ex: type `0.1.1` when you are asked. - -> **CAUTION**: The make release will change local changelog files and commit all the unstaged changes you have. +By contributing to this project, you agree that your contributions will be licensed under the project's [LICENSE](LICENSE). diff --git a/Containerfile b/Containerfile deleted file mode 100644 index 5db7ee2..0000000 --- a/Containerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.7-slim -COPY . /app -WORKDIR /app -RUN pip install . -CMD ["blackmarblepy"] diff --git a/HISTORY.md b/HISTORY.md deleted file mode 100644 index 9bf6ef0..0000000 --- a/HISTORY.md +++ /dev/null @@ -1,13 +0,0 @@ -Changelog -========= - - -0.1.2 (2021-08-14) ------------------- -- Fix release, README and windows CI. [Bruno Rocha] -- Release: version 0.1.0. [Bruno Rocha] - - -0.1.0 (2021-08-14) ------------------- -- Add release command. [Bruno Rocha] diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 26cf123..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include LICENSE -include HISTORY.md -include Containerfile -graft tests -graft blackmarblepy diff --git a/Makefile b/Makefile deleted file mode 100644 index d7f5637..0000000 --- a/Makefile +++ /dev/null @@ -1,122 +0,0 @@ -.ONESHELL: -ENV_PREFIX=$(shell python -c "if __import__('pathlib').Path('.venv/bin/pip').exists(): print('.venv/bin/')") -USING_POETRY=$(shell grep "tool.poetry" pyproject.toml && echo "yes") - -.PHONY: help -help: ## Show the help. - @echo "Usage: make " - @echo "" - @echo "Targets:" - @fgrep "##" Makefile | fgrep -v fgrep - - -.PHONY: show -show: ## Show the current environment. - @echo "Current environment:" - @if [ "$(USING_POETRY)" ]; then poetry env info && exit; fi - @echo "Running using $(ENV_PREFIX)" - @$(ENV_PREFIX)python -V - @$(ENV_PREFIX)python -m site - -.PHONY: install -install: ## Install the project in dev mode. - @if [ "$(USING_POETRY)" ]; then poetry install && exit; fi - @echo "Don't forget to run 'make virtualenv' if you got errors." - $(ENV_PREFIX)pip install -e .[test] - -.PHONY: fmt -fmt: ## Format code using black & isort. - $(ENV_PREFIX)isort blackmarblepy/ - $(ENV_PREFIX)black -l 79 blackmarblepy/ - $(ENV_PREFIX)black -l 79 tests/ - -.PHONY: lint -lint: ## Run pep8, black, mypy linters. - $(ENV_PREFIX)flake8 blackmarblepy/ - $(ENV_PREFIX)black -l 79 --check blackmarblepy/ - $(ENV_PREFIX)black -l 79 --check tests/ - $(ENV_PREFIX)mypy --ignore-missing-imports blackmarblepy/ - -.PHONY: test -test: lint ## Run tests and generate coverage report. - $(ENV_PREFIX)pytest -v --cov-config .coveragerc --cov=blackmarblepy -l --tb=short --maxfail=1 tests/ - $(ENV_PREFIX)coverage xml - $(ENV_PREFIX)coverage html - -.PHONY: watch -watch: ## Run tests on every change. - ls **/**.py | entr $(ENV_PREFIX)pytest -s -vvv -l --tb=long --maxfail=1 tests/ - -.PHONY: clean -clean: ## Clean unused files. - @find ./ -name '*.pyc' -exec rm -f {} \; - @find ./ -name '__pycache__' -exec rm -rf {} \; - @find ./ -name 'Thumbs.db' -exec rm -f {} \; - @find ./ -name '*~' -exec rm -f {} \; - @rm -rf .cache - @rm -rf .pytest_cache - @rm -rf .mypy_cache - @rm -rf build - @rm -rf dist - @rm -rf *.egg-info - @rm -rf htmlcov - @rm -rf .tox/ - @rm -rf docs/_build - -.PHONY: virtualenv -virtualenv: ## Create a virtual environment. - @if [ "$(USING_POETRY)" ]; then poetry install && exit; fi - @echo "creating virtualenv ..." - @rm -rf .venv - @python3 -m venv .venv - @./.venv/bin/pip install -U pip - @./.venv/bin/pip install -e .[test] - @echo - @echo "!!! Please run 'source .venv/bin/activate' to enable the environment !!!" - -.PHONY: release -release: ## Create a new tag for release. - @echo "WARNING: This operation will create s version tag and push to github" - @read -p "Version? (provide the next x.y.z semver) : " TAG - @echo "$${TAG}" > blackmarblepy/VERSION - @$(ENV_PREFIX)gitchangelog > HISTORY.md - @git add blackmarblepy/VERSION HISTORY.md - @git commit -m "release: version $${TAG} 🚀" - @echo "creating git tag : $${TAG}" - @git tag $${TAG} - @git push -u origin HEAD --tags - @echo "Github Actions will detect the new tag and release the new version." - -.PHONY: docs -docs: ## Build the documentation. - @echo "building documentation ..." - @$(ENV_PREFIX)mkdocs build - URL="site/index.html"; xdg-open $$URL || sensible-browser $$URL || x-www-browser $$URL || gnome-open $$URL - -.PHONY: switch-to-poetry -switch-to-poetry: ## Switch to poetry package manager. - @echo "Switching to poetry ..." - @if ! poetry --version > /dev/null; then echo 'poetry is required, install from https://python-poetry.org/'; exit 1; fi - @rm -rf .venv - @poetry init --no-interaction --name=a_flask_test --author=rochacbruno - @echo "" >> pyproject.toml - @echo "[tool.poetry.scripts]" >> pyproject.toml - @echo "blackmarblepy = 'blackmarblepy.__main__:main'" >> pyproject.toml - @cat requirements.txt | while read in; do poetry add --no-interaction "$${in}"; done - @cat requirements-test.txt | while read in; do poetry add --no-interaction "$${in}" --dev; done - @poetry install --no-interaction - @mkdir -p .github/backup - @mv requirements* .github/backup - @mv setup.py .github/backup - @echo "You have switched to https://python-poetry.org/ package manager." - @echo "Please run 'poetry shell' or 'poetry run blackmarblepy'" - -.PHONY: init -init: ## Initialize the project based on an application template. - @./.github/init.sh - - -# This project has been generated from rochacbruno/python-project-template -# __author__ = 'rochacbruno' -# __repo__ = https://github.com/rochacbruno/python-project-template -# __sponsor__ = https://github.com/sponsors/rochacbruno/ diff --git a/README.md b/README.md index 8d178c4..b09912f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# blackmarblepy +# blackmarblepy [![codecov](https://codecov.io/gh/ramarty/blackmarblepy/branch/main/graph/badge.svg?token=blackmarblepy_token_here)](https://codecov.io/gh/ramarty/blackmarblepy) [![CI](https://github.com/ramarty/blackmarblepy/actions/workflows/main.yml/badge.svg)](https://github.com/ramarty/blackmarblepy/actions/workflows/main.yml) @@ -7,7 +7,7 @@ Nighttime Lights Map

-Create Georeferenced Rasters of Nighttime Lights from [NASA Black Marble data](https://blackmarble.gsfc.nasa.gov/). +Create Georeferenced Rasters of Nighttime Lights from [NASA Black Marble data](https://blackmarble.gsfc.nasa.gov/). * [Overview](#overview) * [Installation](#installation) @@ -20,7 +20,7 @@ Create Georeferenced Rasters of Nighttime Lights from [NASA Black Marble data](h * [Quick start](#quickstart) * [Setup](#quickstart-setup) * [Nighttime Lights Raster](#quickstart-raster) - * [Nighttime Lights Trends ](#quickstart-trends) + * [Nighttime Lights Trends](#quickstart-trends) * [Additional Usage](#usage) ## Overview @@ -32,7 +32,7 @@ This package facilitates downloading nighttime lights [Black Marble](https://bla The package can be installed via pip. ```bash -$ pip install git+https://github.com/ramarty/blackmarblepy.git +pip install git+https://github.com/ramarty/blackmarblepy.git ``` ```py @@ -55,67 +55,67 @@ The function requires using a **Bearer Token**; to obtain a token, follow the be The package provides two functions: -* `bm_raster` produces a raster of Black Marble nighttime lights. -* `bm_extract` produces a dataframe of aggregated nighttime lights to a region of interest (e.g., average nighttime lights within US States). +* `bm_raster` produces a raster of Black Marble nighttime lights. +* `bm_extract` produces a dataframe of aggregated nighttime lights to a region of interest (e.g., average nighttime lights within US States). Both functions take the following arguments: ### Required arguments -* __roi_sf:__ Region of interest; geopandas dataframe. Must be in the [WGS 84 (epsg:4326)](https://epsg.io/4326) coordinate reference system. For `bm_extract`, aggregates nighttime lights within each polygon of `roi_sf`. +* **roi_sf:** Region of interest; geopandas dataframe. Must be in the [WGS 84 (epsg:4326)](https://epsg.io/4326) coordinate reference system. For `bm_extract`, aggregates nighttime lights within each polygon of `roi_sf`. -* __product_id:__ One of the following: +* **product_id:** One of the following: - - `"VNP46A1"`: Daily (raw) - - `"VNP46A2"`: Daily (corrected) - - `"VNP46A3"`: Monthly - - `"VNP46A4"`: Annual + * `"VNP46A1"`: Daily (raw) + * `"VNP46A2"`: Daily (corrected) + * `"VNP46A3"`: Monthly + * `"VNP46A4"`: Annual -* __date:__ Date of raster data. Entering one date will produce a raster. Entering multiple dates will produce a raster stack. +* **date:** Date of raster data. Entering one date will produce a raster. Entering multiple dates will produce a raster stack. - - For `product_id`s `"VNP46A1"` and `"VNP46A2"`, a date (eg, `"2021-10-03"`). - - For `product_id` `"VNP46A3"`, a date or year-month (e.g., `"2021-10-01"`, where the day will be ignored, or `"2021-10"`). - - For `product_id` `"VNP46A4"`, year or date (e.g., `"2021-10-01"`, where the month and day will be ignored, or `2021`). + * For `product_id`s `"VNP46A1"` and `"VNP46A2"`, a date (eg, `"2021-10-03"`). + * For `product_id` `"VNP46A3"`, a date or year-month (e.g., `"2021-10-01"`, where the day will be ignored, or `"2021-10"`). + * For `product_id` `"VNP46A4"`, year or date (e.g., `"2021-10-01"`, where the month and day will be ignored, or `2021`). -* __bearer:__ NASA bearer token. For instructions on how to create a token, see [here](https://github.com/ramarty/blackmarblepy#bearer-token-). +* **bearer:** NASA bearer token. For instructions on how to create a token, see [here](https://github.com/ramarty/blackmarblepy#bearer-token-). ### Optional arguments -* __variable:__ Variable to used to create raster (default: `NULL`). For information on all variable choices, see [here](https://ladsweb.modaps.eosdis.nasa.gov/api/v2/content/archives/Document%20Archive/Science%20Data%20Product%20Documentation/VIIRS_Black_Marble_UG_v1.2_April_2021.pdf); for `VNP46A1`, see Table 3; for `VNP46A2` see Table 6; for `VNP46A3` and `VNP46A4`, see Table 9. If `NULL`, uses the following default variables: +* **variable:** Variable to used to create raster (default: `NULL`). For information on all variable choices, see [here](https://ladsweb.modaps.eosdis.nasa.gov/api/v2/content/archives/Document%20Archive/Science%20Data%20Product%20Documentation/VIIRS_Black_Marble_UG_v1.2_April_2021.pdf); for `VNP46A1`, see Table 3; for `VNP46A2` see Table 6; for `VNP46A3` and `VNP46A4`, see Table 9. If `NULL`, uses the following default variables: - - For `product_id` `"VNP46A1"`, uses `DNB_At_Sensor_Radiance_500m`. - - For `product_id` `"VNP46A2"`, uses `Gap_Filled_DNB_BRDF-Corrected_NTL`. - - For `product_id`s `"VNP46A3"` and `"VNP46A4"`, uses `NearNadir_Composite_Snow_Free`. + * For `product_id` `"VNP46A1"`, uses `DNB_At_Sensor_Radiance_500m`. + * For `product_id` `"VNP46A2"`, uses `Gap_Filled_DNB_BRDF-Corrected_NTL`. + * For `product_id`s `"VNP46A3"` and `"VNP46A4"`, uses `NearNadir_Composite_Snow_Free`. -* __quality_flag_rm:__ Quality flag values to use to set values to `NA`. Each pixel has a quality flag value, where low quality values can be removed. Values are set to `NA` for each value in ther `quality_flag_rm` vector. (Default: `[255]`). +* **quality_flag_rm:** Quality flag values to use to set values to `NA`. Each pixel has a quality flag value, where low quality values can be removed. Values are set to `NA` for each value in ther `quality_flag_rm` vector. (Default: `[255]`). - - For `VNP46A1` and `VNP46A2` (daily data): - - `0`: High-quality, Persistent nighttime lights - - `1`: High-quality, Ephemeral nighttime Lights - - `2`: Poor-quality, Outlier, potential cloud contamination, or other issues - - `255`: No retrieval, Fill value (masked out on ingestion) + * For `VNP46A1` and `VNP46A2` (daily data): + * `0`: High-quality, Persistent nighttime lights + * `1`: High-quality, Ephemeral nighttime Lights + * `2`: Poor-quality, Outlier, potential cloud contamination, or other issues + * `255`: No retrieval, Fill value (masked out on ingestion) - - For `VNP46A3` and `VNP46A4` (monthly and annual data): - - `0`: Good-quality, The number of observations used for the composite is larger than 3 - - `1`: Poor-quality, The number of observations used for the composite is less than or equal to 3 - - `2`: Gap filled NTL based on historical data - - `255`: Fill value + * For `VNP46A3` and `VNP46A4` (monthly and annual data): + * `0`: Good-quality, The number of observations used for the composite is larger than 3 + * `1`: Poor-quality, The number of observations used for the composite is less than or equal to 3 + * `2`: Gap filled NTL based on historical data + * `255`: Fill value -* __output_location_type:__ Where output should be stored (default: for `bm_raster`, `"tempfile"`; for `bm_extract`, `"memory"`). Either: +* **output_location_type:** Where output should be stored (default: for `bm_raster`, `"tempfile"`; for `bm_extract`, `"memory"`). Either: - - [For `bm_raster`] `tempfile` where the function will export the raster as a tempfile - - [For `bm_extract`] `memory` where the function will export the data as pandas dataframe - - `file` where the function will export the data as a file. For `bm_raster`, a `.tif` file will be saved; for `bm_extract`, a `.csv` file will be saved. A file is saved for each date. Consequently, if `date = [2018, 2019, 2020]`, three datasets will be saved: one for each year. Saving a dataset for each date can facilitate re-running the function later and only downloading data for dates where data have not been downloaded. + * [For `bm_raster`] `tempfile` where the function will export the raster as a tempfile + * [For `bm_extract`] `memory` where the function will export the data as pandas dataframe + * `file` where the function will export the data as a file. For `bm_raster`, a `.tif` file will be saved; for `bm_extract`, a `.csv` file will be saved. A file is saved for each date. Consequently, if `date = [2018, 2019, 2020]`, three datasets will be saved: one for each year. Saving a dataset for each date can facilitate re-running the function later and only downloading data for dates where data have not been downloaded. If `output_location_type = "file"`, the following arguments can be used: -* __file_dir:__ The directory where data should be exported (default: `NULL`, so the working directory will be used) -* __file_prefix:__ Prefix to add to the file to be saved. The file will be saved as the following: `[file_prefix][product_id]_t[date].[tif/csv]` -* __file_skip_if_exists:__ Whether the function should first check wither the file already exists, and to skip downloading or extracting data if the data for that date if the file already exists (default: `TRUE`). If the function is first run with `date = c(2018, 2019, 2020)`, then is later run with `date = c(2018, 2019, 2020, 2021)`, the function will only download/extract data for 2021. Skipping existing files can facilitate re-running the function at a later date to download only more recent data. +* **file_dir:** The directory where data should be exported (default: `NULL`, so the working directory will be used) +* **file_prefix:** Prefix to add to the file to be saved. The file will be saved as the following: `[file_prefix][product_id]_t[date].[tif/csv]` +* **file_skip_if_exists:** Whether the function should first check wither the file already exists, and to skip downloading or extracting data if the data for that date if the file already exists (default: `TRUE`). If the function is first run with `date = c(2018, 2019, 2020)`, then is later run with `date = c(2018, 2019, 2020, 2021)`, the function will only download/extract data for 2021. Skipping existing files can facilitate re-running the function at a later date to download only more recent data. ### Argument for `bm_extract` only -* __aggregation_fun:__ A vector of functions to aggregate data (default: `"mean"`). The `zonal_stats` function from the `rasterstats` package is used for aggregations; this parameter is passed to `stats` argument in `zonal_stats`. +* **aggregation_fun:** A vector of functions to aggregate data (default: `"mean"`). The `zonal_stats` function from the `rasterstats` package is used for aggregations; this parameter is passed to `stats` argument in `zonal_stats`. ## Quickstart @@ -141,7 +141,7 @@ bearer == "BEARER TOKEN HERE" downloader = GADMDownloader(version="4.0") country_name = "Ghana" -ghana_adm1 = downloader.get_shape_data_by_country_name(country_name=country_name, +ghana_adm1 = downloader.get_shape_data_by_country_name(country_name=country_name, ad_level=1) ``` @@ -153,9 +153,9 @@ The below example shows making an annual raster of nighttime lights for Ghana. ## Raster of nighttime lights r = bm_raster(roi_sf = ghana_adm1, product_id = "VNP46A4", - date = 2022, + date = 2022, bearer = bearer) - + ## Map raster r_np = r.read(1) r_np = np.log(r_np+1) @@ -179,7 +179,7 @@ ntl_df = bm_extract(roi_sf = ghana_adm1, product_id = "VNP46A4", date = list(range(2012, 2023)), bearer = bearer) - + ## Plot Trends sns.set(style="whitegrid") g = sns.catplot(data=ntl_df, kind="bar", x="date", y="ntl_mean", col="NAME_1", height=2.5, col_wrap = 3, aspect=1.2, color = "orange") diff --git a/ntl_usa.png b/docs/ntl_usa.png similarity index 100% rename from ntl_usa.png rename to docs/ntl_usa.png diff --git a/raster_example.png b/docs/raster_example.png similarity index 100% rename from raster_example.png rename to docs/raster_example.png diff --git a/trends_example.png b/docs/trends_example.png similarity index 100% rename from trends_example.png rename to docs/trends_example.png diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 280cf9c..0000000 --- a/mkdocs.yml +++ /dev/null @@ -1,2 +0,0 @@ -site_name: blackmarblepy -theme: readthedocs diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 11ba2ba..0000000 --- a/requirements-test.txt +++ /dev/null @@ -1,10 +0,0 @@ -# This requirements are for development and testing only, not for production. -pytest -coverage -flake8 -black -isort -pytest-cov -mypy -gitchangelog -mkdocs diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8ac4195..0000000 --- a/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -# This template is a low-dependency template. -# By default there is no requirements added here. -# Add the requirements you need to this file. -# or run `make init` to create this file automatically based on the template. -# You can also run `make switch-to-poetry` to use the poetry package manager. - -pandas -numpy -requests -datetime -geopandas -rasterstats -h5py -rasterio -#os -#time -#re -#tempfile -#shutil -#subprocess -#itertools -#glob \ No newline at end of file