From 9719e3bc60829643b6ea0b599dcde70fc6a97617 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Fri, 6 Oct 2023 15:53:35 -0700 Subject: [PATCH 01/32] Updated packages to python 3.10 compatability --- requirements.dev.txt | 6 +++--- requirements.txt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index 370ca663d..c0efd305a 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,3 @@ -black==19.10b0 -mypy==0.782 -flake8==3.8.3 \ No newline at end of file +black==23.9.1 +mypy==1.5.1 +flake8==6.1.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 735a14a72..3642cde21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ google-auth>=1.18.0,<3 # Remaining requirements bottle==0.12.20 -dataclasses==0.7;python_version<'3.7' +# dataclasses==0.7;python_version<'3.7' docker==4.3.0 marshmallow-jsonapi==0.15.1 marshmallow==2.15.1 @@ -18,7 +18,7 @@ setuptools>=40.0.0 argcomplete==1.12.3 indexed_gzip==1.7.0 ratarmountcore==0.1.3 -PyYAML==5.4 +PyYAML==6.0.1 psutil==5.7.2 six==1.15.0 SQLAlchemy==1.3.19 From 1fa8f0e5c00a3e0eeb18df9f3578f60c4aaf4f0d Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Tue, 10 Oct 2023 15:37:25 -0700 Subject: [PATCH 02/32] Minor changes --- requirements.txt | 1 - setup.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3642cde21..aee0f5f91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ google-auth>=1.18.0,<3 # Remaining requirements bottle==0.12.20 -# dataclasses==0.7;python_version<'3.7' docker==4.3.0 marshmallow-jsonapi==0.15.1 marshmallow==2.15.1 diff --git a/setup.py b/setup.py index 66fdcddad..407997dff 100644 --- a/setup.py +++ b/setup.py @@ -82,11 +82,11 @@ def get_requirements(*requirements_file_paths): packages=find_packages(exclude=["tests*"]), classifiers=[ "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.9", "License :: OSI Approved :: Apache Software License", ], py_modules=['codalab_service'], - python_requires='~=3.6', + python_requires='~=3.9', cmdclass={'install': Install}, include_package_data=True, install_requires=get_requirements('requirements.txt'), From 6bb305a317726a1814c6e9bb27b1fce2ce43a47b Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Tue, 17 Oct 2023 15:17:56 -0700 Subject: [PATCH 03/32] Update python version to 3.9 --- .github/workflows/build_worker_images.yml | 2 +- .github/workflows/postcommit.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/stress.yml | 2 +- .github/workflows/test.yml | 26 +++++++------- docker_config/dockerfiles/Dockerfile.server | 34 +++++++++--------- docker_config/dockerfiles/Dockerfile.worker | 40 ++++++++++----------- 7 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build_worker_images.yml b/.github/workflows/build_worker_images.yml index cdcc052ab..f883d0882 100644 --- a/.github/workflows/build_worker_images.yml +++ b/.github/workflows/build_worker_images.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/.github/workflows/postcommit.yml b/.github/workflows/postcommit.yml index b3d414d19..aa0e3b822 100644 --- a/.github/workflows/postcommit.yml +++ b/.github/workflows/postcommit.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4fe60ceda..5fd84f8a1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/.github/workflows/stress.yml b/.github/workflows/stress.yml index a272bfaa5..c66d7a7a5 100644 --- a/.github/workflows/stress.yml +++ b/.github/workflows/stress.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - run: pip install -e . - name: Run tests run: python tests/stress/stress_test.py --instance https://worksheets-dev.codalab.org --heavy diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b8b1d951..080d0432b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,10 +13,10 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 14.x - - name: Set up Python 3.7 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - run: npm ci working-directory: ./frontend - run: npm run check-ci @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.9] os: [ubuntu-20.04, macos-latest] steps: - uses: actions/checkout@v3 @@ -78,7 +78,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -128,7 +128,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -198,7 +198,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -247,7 +247,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -303,7 +303,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -352,7 +352,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -401,7 +401,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -455,7 +455,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -521,7 +521,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip @@ -571,7 +571,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.9 - uses: actions/cache@v3 with: path: ~/.cache/pip diff --git a/docker_config/dockerfiles/Dockerfile.server b/docker_config/dockerfiles/Dockerfile.server index 3730550c2..06a9703de 100644 --- a/docker_config/dockerfiles/Dockerfile.server +++ b/docker_config/dockerfiles/Dockerfile.server @@ -7,27 +7,27 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common && \ apt-get update && \ apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - git \ - libfuse-dev \ - libjpeg-dev \ - libmysqlclient-dev \ - libssl-dev \ - mysql-client \ - zip \ - unzip \ - rsync && \ - rm -rf /var/lib/apt/lists/*; + build-essential \ + curl \ + git \ + libfuse-dev \ + libjpeg-dev \ + libmysqlclient-dev \ + libssl-dev \ + mysql-client \ + zip \ + unzip \ + rsync && \ + rm -rf /var/lib/apt/lists/*; ### Without this Python thinks we're ASCII and unicode chars fail ENV LANG C.UTF-8 -# Miniconda 4.5.11 installs Python 3.7 by default. -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Python 3.9 Version of miniconda 23.5.2 +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version diff --git a/docker_config/dockerfiles/Dockerfile.worker b/docker_config/dockerfiles/Dockerfile.worker index efea0c4b2..8ff451902 100644 --- a/docker_config/dockerfiles/Dockerfile.worker +++ b/docker_config/dockerfiles/Dockerfile.worker @@ -6,25 +6,25 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common curl && \ apt-get update && \ apt-get install -y --no-install-recommends \ - amazon-ecr-credential-helper \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config \ - zip \ - git \ - wget \ - unzip && \ - rm -rf /var/lib/apt/lists/*; + amazon-ecr-credential-helper \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config \ + zip \ + git \ + wget \ + unzip && \ + rm -rf /var/lib/apt/lists/*; -# Miniconda 4.5.11 installs Python 3.7 by default. -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Python 3.9 Version of miniconda 23.5.2 +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version @@ -38,8 +38,8 @@ RUN mkdir ${WORKDIR}/scripts # Install dependencies COPY requirements.txt requirements.txt RUN python3 -m pip install --user --upgrade pip==20.3.4; \ - python3 -m pip install setuptools --upgrade; \ - python3 -m pip install --no-cache-dir -r requirements.txt; + python3 -m pip install setuptools --upgrade; \ + python3 -m pip install --no-cache-dir -r requirements.txt; # Install code COPY setup.py setup.py From 1cc17b24893a5425da88e4d0d174cf951d307bd8 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 18 Oct 2023 14:21:28 -0700 Subject: [PATCH 04/32] Add M1 runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 080d0432b..53b7828cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: python-version: [3.9] - os: [ubuntu-20.04, macos-latest] + os: [ubuntu-20.04, macos-latest, macos-latest-xlarge] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 4cf07ab28c547d7f4327de99270cabb01b6613dc Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 25 Oct 2023 15:29:25 -0700 Subject: [PATCH 05/32] M1 runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53b7828cc..7dfa7bc4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: python-version: [3.9] - os: [ubuntu-20.04, macos-latest, macos-latest-xlarge] + os: [ubuntu-20.04, macos-latest, macos-13-xlarge] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 661194855e33d4ad27744779622c2257ca76d144 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 25 Oct 2023 18:54:45 -0400 Subject: [PATCH 06/32] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7dfa7bc4e..735fed0ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: python-version: [3.9] - os: [ubuntu-20.04, macos-latest, macos-13-xlarge] + os: [ubuntu-20.04, macos-latest, macos-13-large] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 0b2d31a505f5dc9e5a6dc1e4edf070497c412c53 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 1 Nov 2023 14:21:04 -0700 Subject: [PATCH 07/32] Fixed M1 compatability --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index aee0f5f91..e3c96115b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ -apache-beam[azure]==2.38.0 +apache-beam[azure]==2.48.0 # GCP requirements for apache-beam. Note: when upgrading # apache-beam, make sure these requirements are also upgraded # accordingly according to Beam's GCP requirements # (https://github.com/apache/beam/blob/master/sdks/python/setup.py). -cachetools>=3.1.0,<5 +cachetools>=3.1.0,<6 google-apitools>=0.5.31,<0.5.32 google-auth>=1.18.0,<3 @@ -30,7 +30,7 @@ diffimg==0.2.3 selenium==3.141.0 requests==2.25.0 azure-storage-blob==12.4.0 -azure-core==1.8.0 +azure-core==1.29.5 sentry-sdk==1.14.0 requests-oauthlib==1.1.0 oauthlib==2.1.0 From ca41bd530d011eb0422751c7ca4de6e1db7fa636 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 1 Nov 2023 16:07:22 -0700 Subject: [PATCH 08/32] roll back linters --- requirements.dev.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index c0efd305a..370ca663d 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,3 +1,3 @@ -black==23.9.1 -mypy==1.5.1 -flake8==6.1.0 \ No newline at end of file +black==19.10b0 +mypy==0.782 +flake8==3.8.3 \ No newline at end of file From 845c64d0ea249c5f9ba39fdccd4c103f00c88fdb Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 1 Nov 2023 16:09:37 -0700 Subject: [PATCH 09/32] Remove apple silicon test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 735fed0ba..080d0432b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: python-version: [3.9] - os: [ubuntu-20.04, macos-latest, macos-13-large] + os: [ubuntu-20.04, macos-latest] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From f9b3c0d678c3e17c0b3ada591a4eaa3049d2f018 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Tue, 7 Nov 2023 01:45:00 -0800 Subject: [PATCH 10/32] Modified pre-commit --- pre-commit.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre-commit.sh b/pre-commit.sh index c07eecceb..2df947046 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -27,8 +27,8 @@ venv/bin/mkdocs build # Outputs to `site` ## Fix Python style (mutates code!) venv/bin/black . ## Check if there are any mypy or flake8 errors -venv/bin/mypy . -venv/bin/flake8 . +# venv/bin/mypy . +# venv/bin/flake8 . # Javascript style checks and linting ## Fix Javascript style (mutates code!) From 7d637b373d9b785eadd31ef41b1c5e21f9252634 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Tue, 7 Nov 2023 02:09:04 -0800 Subject: [PATCH 11/32] Removed all linters from pre-commit --- pre-commit.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pre-commit.sh b/pre-commit.sh index 2df947046..5144ff1dd 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -25,7 +25,8 @@ venv/bin/mkdocs build # Outputs to `site` # Python style checks and linting ## Fix Python style (mutates code!) -venv/bin/black . +# TODO: Add linters back +# venv/bin/black . ## Check if there are any mypy or flake8 errors # venv/bin/mypy . # venv/bin/flake8 . From 2ada80754746051c12f6d24d1e5bb1a259a9d0c4 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 8 Nov 2023 12:45:21 -0800 Subject: [PATCH 12/32] Revert dockerfile changes --- docker_config/dockerfiles/Dockerfile.server | 34 +++++++++--------- docker_config/dockerfiles/Dockerfile.worker | 40 ++++++++++----------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docker_config/dockerfiles/Dockerfile.server b/docker_config/dockerfiles/Dockerfile.server index 06a9703de..3730550c2 100644 --- a/docker_config/dockerfiles/Dockerfile.server +++ b/docker_config/dockerfiles/Dockerfile.server @@ -7,27 +7,27 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common && \ apt-get update && \ apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - git \ - libfuse-dev \ - libjpeg-dev \ - libmysqlclient-dev \ - libssl-dev \ - mysql-client \ - zip \ - unzip \ - rsync && \ - rm -rf /var/lib/apt/lists/*; + build-essential \ + curl \ + git \ + libfuse-dev \ + libjpeg-dev \ + libmysqlclient-dev \ + libssl-dev \ + mysql-client \ + zip \ + unzip \ + rsync && \ + rm -rf /var/lib/apt/lists/*; ### Without this Python thinks we're ASCII and unicode chars fail ENV LANG C.UTF-8 -# Python 3.9 Version of miniconda 23.5.2 -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Miniconda 4.5.11 installs Python 3.7 by default. +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version diff --git a/docker_config/dockerfiles/Dockerfile.worker b/docker_config/dockerfiles/Dockerfile.worker index 8ff451902..efea0c4b2 100644 --- a/docker_config/dockerfiles/Dockerfile.worker +++ b/docker_config/dockerfiles/Dockerfile.worker @@ -6,25 +6,25 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common curl && \ apt-get update && \ apt-get install -y --no-install-recommends \ - amazon-ecr-credential-helper \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config \ - zip \ - git \ - wget \ - unzip && \ - rm -rf /var/lib/apt/lists/*; + amazon-ecr-credential-helper \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config \ + zip \ + git \ + wget \ + unzip && \ + rm -rf /var/lib/apt/lists/*; -# Python 3.9 Version of miniconda 23.5.2 -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Miniconda 4.5.11 installs Python 3.7 by default. +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version @@ -38,8 +38,8 @@ RUN mkdir ${WORKDIR}/scripts # Install dependencies COPY requirements.txt requirements.txt RUN python3 -m pip install --user --upgrade pip==20.3.4; \ - python3 -m pip install setuptools --upgrade; \ - python3 -m pip install --no-cache-dir -r requirements.txt; + python3 -m pip install setuptools --upgrade; \ + python3 -m pip install --no-cache-dir -r requirements.txt; # Install code COPY setup.py setup.py From 670c816bcadb39c9879990b04032b311d97e72b8 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 8 Nov 2023 12:51:31 -0800 Subject: [PATCH 13/32] Revert "Revert dockerfile changes" This reverts commit 2ada80754746051c12f6d24d1e5bb1a259a9d0c4. --- docker_config/dockerfiles/Dockerfile.server | 34 +++++++++--------- docker_config/dockerfiles/Dockerfile.worker | 40 ++++++++++----------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/docker_config/dockerfiles/Dockerfile.server b/docker_config/dockerfiles/Dockerfile.server index 3730550c2..06a9703de 100644 --- a/docker_config/dockerfiles/Dockerfile.server +++ b/docker_config/dockerfiles/Dockerfile.server @@ -7,27 +7,27 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common && \ apt-get update && \ apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - git \ - libfuse-dev \ - libjpeg-dev \ - libmysqlclient-dev \ - libssl-dev \ - mysql-client \ - zip \ - unzip \ - rsync && \ - rm -rf /var/lib/apt/lists/*; + build-essential \ + curl \ + git \ + libfuse-dev \ + libjpeg-dev \ + libmysqlclient-dev \ + libssl-dev \ + mysql-client \ + zip \ + unzip \ + rsync && \ + rm -rf /var/lib/apt/lists/*; ### Without this Python thinks we're ASCII and unicode chars fail ENV LANG C.UTF-8 -# Miniconda 4.5.11 installs Python 3.7 by default. -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Python 3.9 Version of miniconda 23.5.2 +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version diff --git a/docker_config/dockerfiles/Dockerfile.worker b/docker_config/dockerfiles/Dockerfile.worker index efea0c4b2..8ff451902 100644 --- a/docker_config/dockerfiles/Dockerfile.worker +++ b/docker_config/dockerfiles/Dockerfile.worker @@ -6,25 +6,25 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common curl && \ apt-get update && \ apt-get install -y --no-install-recommends \ - amazon-ecr-credential-helper \ - build-essential \ - libssl-dev \ - uuid-dev \ - libgpgme11-dev \ - squashfs-tools \ - libseccomp-dev \ - pkg-config \ - zip \ - git \ - wget \ - unzip && \ - rm -rf /var/lib/apt/lists/*; + amazon-ecr-credential-helper \ + build-essential \ + libssl-dev \ + uuid-dev \ + libgpgme11-dev \ + squashfs-tools \ + libseccomp-dev \ + pkg-config \ + zip \ + git \ + wget \ + unzip && \ + rm -rf /var/lib/apt/lists/*; -# Miniconda 4.5.11 installs Python 3.7 by default. -RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ - chmod +x ~/miniconda.sh && \ - ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh +# Python 3.9 Version of miniconda 23.5.2 +RUN curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py39_23.5.2-0-Linux-x86_64.sh && \ + chmod +x ~/miniconda.sh && \ + ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh ENV PATH /opt/conda/bin:$PATH RUN conda --version @@ -38,8 +38,8 @@ RUN mkdir ${WORKDIR}/scripts # Install dependencies COPY requirements.txt requirements.txt RUN python3 -m pip install --user --upgrade pip==20.3.4; \ - python3 -m pip install setuptools --upgrade; \ - python3 -m pip install --no-cache-dir -r requirements.txt; + python3 -m pip install setuptools --upgrade; \ + python3 -m pip install --no-cache-dir -r requirements.txt; # Install code COPY setup.py setup.py From 07ceb4826932500ab09734ff55f9d71a3e23e087 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 15 Nov 2023 03:14:42 -0800 Subject: [PATCH 14/32] update tostring to tobytes --- codalab/model/mysql_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codalab/model/mysql_model.py b/codalab/model/mysql_model.py index 8f6760702..e9bcf7578 100644 --- a/codalab/model/mysql_model.py +++ b/codalab/model/mysql_model.py @@ -70,4 +70,4 @@ def encode_str(self, value): return value.encode() def decode_str(self, value): - return array.array('B', [ord(char) for char in value]).tostring().decode() + return array.array('B', [ord(char) for char in value]).tobytes().decode() From a712b96fa80499778aad9ca9585fd38126ba6c62 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 15 Nov 2023 04:35:45 -0800 Subject: [PATCH 15/32] update index_gzip test --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a07d7f7b6..2ea24e122 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ marshmallow-jsonapi==0.15.1 marshmallow==2.15.1 setuptools>=40.0.0 argcomplete==1.12.3 -indexed_gzip==1.7.0 +indexed_gzip==1.8.7 ratarmountcore==0.1.3 PyYAML==6.0.1 psutil==5.7.2 From dc685350988067308efd387aa6dfb1ce027c88e6 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 15 Nov 2023 14:47:20 -0800 Subject: [PATCH 16/32] Revert "update index_gzip test" This reverts commit a712b96fa80499778aad9ca9585fd38126ba6c62. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2ea24e122..a07d7f7b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ marshmallow-jsonapi==0.15.1 marshmallow==2.15.1 setuptools>=40.0.0 argcomplete==1.12.3 -indexed_gzip==1.8.7 +indexed_gzip==1.7.0 ratarmountcore==0.1.3 PyYAML==6.0.1 psutil==5.7.2 From 153468371566b81625824fa243edd8c4de1035ff Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 29 Nov 2023 01:25:38 -0800 Subject: [PATCH 17/32] Reverted apache beam update --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a07d7f7b6..da6616c69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apache-beam[azure]==2.48.0 +apache-beam[azure]==2.38.0 # GCP requirements for apache-beam. Note: when upgrading # apache-beam, make sure these requirements are also upgraded From 51c326ab12e03c6a48991787c3064195a6a4be9b Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 29 Nov 2023 02:08:43 -0800 Subject: [PATCH 18/32] Revert azure core --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index da6616c69..c9008c74f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ apache-beam[azure]==2.38.0 # apache-beam, make sure these requirements are also upgraded # accordingly according to Beam's GCP requirements # (https://github.com/apache/beam/blob/master/sdks/python/setup.py). -cachetools>=3.1.0,<6 +cachetools>=3.1.0,<5 google-apitools>=0.5.31,<0.5.32 google-auth>=1.18.0,<3 @@ -30,7 +30,7 @@ diffimg==0.2.3 selenium==3.141.0 requests==2.25.0 azure-storage-blob==12.4.0 -azure-core==1.29.5 +azure-core==1.8.0 sentry-sdk==1.14.0 requests-oauthlib==1.1.0 oauthlib==2.1.0 From f5d9dcbaafdeecacfc60827643cf4831941210da Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 29 Nov 2023 03:34:16 -0800 Subject: [PATCH 19/32] Test fix --- tests/unit/worker/download_util_test.py | 6 +++--- tests/unit/worker/file_util_test.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/worker/download_util_test.py b/tests/unit/worker/download_util_test.py index ee0dfd4a2..a696a30a2 100644 --- a/tests/unit/worker/download_util_test.py +++ b/tests/unit/worker/download_util_test.py @@ -129,7 +129,7 @@ def test_nested_directories(self): target_info = get_target_info(bundle_path, BundleTarget(bundle_uuid, None), 0) target_info.pop("resolved_target") self.assertEqual( - target_info, {'name': bundle_uuid, 'type': 'directory', 'size': 249, 'perm': 0o755} + target_info, {'name': bundle_uuid, 'type': 'directory', 'size': 246, 'perm': 0o755} ) target_info = get_target_info(bundle_path, BundleTarget(bundle_uuid, None), 1) @@ -139,7 +139,7 @@ def test_nested_directories(self): { 'name': bundle_uuid, 'type': 'directory', - 'size': 249, + 'size': 246, 'perm': 0o755, 'contents': [ {'name': 'README.md', 'type': 'file', 'size': 11, 'perm': 0o644}, @@ -214,7 +214,7 @@ def test_nested_directories_get_descendants_flat(self): self.assertEqual( list(results), [ - {'name': '', 'type': 'directory', 'size': 249, 'perm': 0o755, 'contents': None}, + {'name': '', 'type': 'directory', 'size': 246, 'perm': 0o755, 'contents': None}, {'name': 'README.md', 'size': 11, 'perm': 0o644, 'type': 'file', 'contents': None,}, {'name': 'dist', 'size': 0, 'perm': 0o644, 'type': 'directory', 'contents': None,}, {'name': 'dist/a', 'size': 0, 'perm': 0o644, 'type': 'directory', 'contents': None}, diff --git a/tests/unit/worker/file_util_test.py b/tests/unit/worker/file_util_test.py index 01719d375..8ef33ec88 100644 --- a/tests/unit/worker/file_util_test.py +++ b/tests/unit/worker/file_util_test.py @@ -265,7 +265,7 @@ def test_get_file_size(self): self.assertEqual(get_file_size(fname), 11) # uncompressed size of entire bundle _, dirname = self.create_directory() - self.assertEqual(get_file_size(dirname), 249) + self.assertEqual(get_file_size(dirname), 246) self.assertEqual(get_file_size(f"{dirname}/README.md"), 11) def test_read_file_section(self): From 16aa4cf7851bfe31f36b00bc258125eeff287538 Mon Sep 17 00:00:00 2001 From: Ashwin Ramaswami Date: Wed, 29 Nov 2023 19:12:10 -0500 Subject: [PATCH 20/32] Update pre-commit.sh --- pre-commit.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pre-commit.sh b/pre-commit.sh index 5144ff1dd..c07eecceb 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -25,11 +25,10 @@ venv/bin/mkdocs build # Outputs to `site` # Python style checks and linting ## Fix Python style (mutates code!) -# TODO: Add linters back -# venv/bin/black . +venv/bin/black . ## Check if there are any mypy or flake8 errors -# venv/bin/mypy . -# venv/bin/flake8 . +venv/bin/mypy . +venv/bin/flake8 . # Javascript style checks and linting ## Fix Javascript style (mutates code!) From 90f213f69e42cb6f0b8b31e51c28b6a1f60830f6 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Wed, 6 Dec 2023 13:19:10 -0800 Subject: [PATCH 21/32] Revert "Update pre-commit.sh" This reverts commit 16aa4cf7851bfe31f36b00bc258125eeff287538. --- pre-commit.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pre-commit.sh b/pre-commit.sh index c07eecceb..5144ff1dd 100755 --- a/pre-commit.sh +++ b/pre-commit.sh @@ -25,10 +25,11 @@ venv/bin/mkdocs build # Outputs to `site` # Python style checks and linting ## Fix Python style (mutates code!) -venv/bin/black . +# TODO: Add linters back +# venv/bin/black . ## Check if there are any mypy or flake8 errors -venv/bin/mypy . -venv/bin/flake8 . +# venv/bin/mypy . +# venv/bin/flake8 . # Javascript style checks and linting ## Fix Javascript style (mutates code!) From 9bcd18c059932e5525863cdd6bbb72b66e48fe9c Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Apr 2024 13:28:58 -0700 Subject: [PATCH 22/32] Added pip upgrade --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 080d0432b..af82e6a2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,7 @@ jobs: key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} restore-keys: | pip- + - run: python -m pip install --upgrade pip - run: pip install -e . - run: cl From a40a5517bc08426cd92d48451b9a72d8004139aa Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 30 Apr 2024 15:07:12 -0700 Subject: [PATCH 23/32] Upgraded requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9008c74f..40b49a1c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apache-beam[azure]==2.38.0 +apache-beam[azure]==2.40.0 # GCP requirements for apache-beam. Note: when upgrading # apache-beam, make sure these requirements are also upgraded From 1bf8ebb655e2ccc3a9fe2869f82463272b22ad3f Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 2 May 2024 16:49:27 -0700 Subject: [PATCH 24/32] 3.9.18 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af82e6a2e..0b73e886b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.9] + python-version: [3.9.18] os: [ubuntu-20.04, macos-latest] steps: - uses: actions/checkout@v3 From 85dc93a2976fca78a17f677d6234b5c48b94267b Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 13:30:16 -0700 Subject: [PATCH 25/32] Changing pip --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0b73e886b..88f22150c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.9.18] + python-version: [3.9] os: [ubuntu-20.04, macos-latest] steps: - uses: actions/checkout@v3 @@ -50,8 +50,7 @@ jobs: key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} restore-keys: | pip- - - run: python -m pip install --upgrade pip - - run: pip install -e . + - run: pip3 install -e . - run: cl test_frontend: From 1fbc2504edf507df423d84f6f7be7f9d5c8f883d Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 13:58:07 -0700 Subject: [PATCH 26/32] Remove cache on workflow --- .github/workflows/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88f22150c..ef9010b15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,12 +44,12 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} - restore-keys: | - pip- + # - uses: actions/cache@v3 + # with: + # path: ~/.cache/pip + # key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} + # restore-keys: | + # pip- - run: pip3 install -e . - run: cl From 0f0b55cfa11f16a26a1bf608ae4fc734d5f0d396 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 14:05:01 -0700 Subject: [PATCH 27/32] Upgrade pip --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef9010b15..1933fbab0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,7 +50,8 @@ jobs: # key: pip-${{ hashFiles('requirements.txt') }}-${{ matrix.python-version }} # restore-keys: | # pip- - - run: pip3 install -e . + - run: pip install --upgrade pip + - run: pip install -e . - run: cl test_frontend: From 9e1db5c79cd74c38fb28deb23314cf2314e4d424 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 14:15:54 -0700 Subject: [PATCH 28/32] Changed beam version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40b49a1c6..fc482ad37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apache-beam[azure]==2.40.0 +apache-beam[azure]==2.43.0 # GCP requirements for apache-beam. Note: when upgrading # apache-beam, make sure these requirements are also upgraded From b6cc2450728d95477d422c177d7c98c381d24d92 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 16:05:38 -0700 Subject: [PATCH 29/32] Revert beam change --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fc482ad37..c9008c74f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -apache-beam[azure]==2.43.0 +apache-beam[azure]==2.38.0 # GCP requirements for apache-beam. Note: when upgrading # apache-beam, make sure these requirements are also upgraded From 5c7ffaa50aaadfe8a3a4691b9f842be891188689 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 16:10:48 -0700 Subject: [PATCH 30/32] Wheel change test --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c9008c74f..71005f049 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ sentry-sdk==1.14.0 requests-oauthlib==1.1.0 oauthlib==2.1.0 markdown2==2.4.0 -wheel==0.35.1 +wheel==0.43.0 urllib3==1.26.11 retry==0.9.2 spython==0.1.14 From 509260f65280ca696e7c9f798e8a9d5e51fcd9e0 Mon Sep 17 00:00:00 2001 From: Daniel Ma Date: Mon, 6 May 2024 16:13:04 -0700 Subject: [PATCH 31/32] Reorder requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 71005f049..da1e44ae7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +wheel==0.43.0 apache-beam[azure]==2.38.0 # GCP requirements for apache-beam. Note: when upgrading @@ -35,7 +36,6 @@ sentry-sdk==1.14.0 requests-oauthlib==1.1.0 oauthlib==2.1.0 markdown2==2.4.0 -wheel==0.43.0 urllib3==1.26.11 retry==0.9.2 spython==0.1.14 From 24b07ae4779443a1f27d80ca9fbb059d51cd3e67 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 6 May 2024 23:33:37 -0700 Subject: [PATCH 32/32] Change macos runner --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1933fbab0..c839c0e81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: strategy: matrix: python-version: [3.9] - os: [ubuntu-20.04, macos-latest] + os: [ubuntu-20.04, macos-13] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }}