Update to TensorFlow 2.7 (#609)

.conda/bld.bat

@@ -13,21 +13,22 @@ rem # this out myself, ughhh.
 set PIP_NO_INDEX=False
 set PIP_NO_DEPENDENCIES=False
 set PIP_IGNORE_INSTALLED=False
-rem pip install "numpy>=1.18.1,<1.19.0"
-pip install attrs==19.3
-pip install cattrs==1.0.0rc
+
+rem pip install numpy==1.19.5
+pip install attrs==21.2.0
+pip install cattrs==1.1.1
 pip install jsonpickle==1.2
 pip install networkx
-rem pip install tensorflow==2.3.1
-rem pip install h5py==2.10.0
+rem pip install tensorflow==2.7.0
+rem pip install h5py==3.10.0
 pip install python-rapidjson
 rem pip install opencv-python-headless==4.2.0.34
 pip install pandas
 pip install psutil
-rem pip install "PySide2>=5.12.0,<=5.14.1"
+rem pip install PySide2==5.14.1
 pip install pyzmq
 pip install pyyaml
-pip install imgaug==0.3.0
+pip install imgaug==0.4.0
 pip install "scipy<=1.4.1"
 pip install scikit-image
 pip install scikit-learn

.conda/build.sh

@@ -11,21 +11,21 @@ export PIP_NO_INDEX=False
 export PIP_NO_DEPENDENCIES=False
 export PIP_IGNORE_INSTALLED=False
 
-# pip install "numpy>=1.18.1,<1.19.0"
-pip install attrs==19.3
-pip install cattrs==1.0.0rc
+# pip install numpy==1.19.5
+pip install attrs==21.2.0
+pip install cattrs==1.1.1
 pip install jsonpickle==1.2
 pip install networkx
-# pip install tensorflow==2.3.1
-# pip install h5py==2.10.0
+# pip install tensorflow==2.7.0
+# pip install h5py==3.10.0
 pip install python-rapidjson
 # pip install opencv-python-headless==4.2.0.34
 pip install pandas
 pip install psutil
-# pip install "PySide2>=5.12.0,<=5.14.1"
+# pip install PySide2==5.14.1
 pip install pyzmq
 pip install pyyaml
-pip install imgaug==0.3.0
+pip install imgaug==0.4.0
 pip install "scipy<=1.4.1"
 pip install scikit-image
 pip install scikit-learn

.conda/conda_build_config.yaml

@@ -3,7 +3,7 @@
 # Ref: https://docs.conda.io/projects/conda-build/en/latest/resources/variants.html#conda-build-variant-config-files
 
 python:
-  - 3.6
+  - 3.7
 
 numpy:
-  - 1.18.1
+  - 1.19.5

.conda/meta.yaml

@@ -24,16 +24,16 @@ source:
 
 requirements:
   host:
-    - python=3.6
-    - sleap::tensorflow=2.3.1
+    - python=3.7
+    - sleap::tensorflow=2.7.0
     - sleap::pyside2=5.14.1
     - pip
 
   run:
-    - python=3.6
-    - cudatoolkit=10.1
-    - cudnn=7.6
-    - sleap::tensorflow=2.3.1
+    - python=3.7
+    - cudatoolkit=11.3.1
+    - cudnn=8.2.1
+    - sleap::tensorflow=2.7.0
     - sleap::pyside2=5.14.1
     - Pillow
     - Shapely

.github/workflows/build.yml

@@ -18,9 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # os: ["ubuntu-latest", "windows-latest"]
         os: ["ubuntu-18.04", "windows-2019"]
-        # os: ["ubuntu-18.04"]
     steps:
       # Setup
       - uses: actions/checkout@v2
@@ -36,7 +34,7 @@ jobs:
         # https://github.com/conda-incubator/setup-miniconda
         uses: conda-incubator/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
           use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
           environment-file: environment_build.yml
           activate-environment: sleap
@@ -67,9 +65,9 @@ jobs:
         if: matrix.os == 'ubuntu-18.04'
         shell: bash -l {0}
         run: |
-          sudo apt-get install xvfb libxkbcommon-x11-0
-          sudo Xvfb :1 -screen 0 1024x768x24 </dev/null &
-          export DISPLAY=":1"
+          sudo apt install libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
+          export DISPLAY=":99.0"
+          /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX
           xvfb-run pytest tests/
           conda build .conda --output-folder build -c https://conda.anaconda.org/sleap/
       - name: Build conda package (Windows)

.github/workflows/ci.yml

@@ -6,17 +6,18 @@ on:
     paths:
       - 'sleap/**'
       - 'tests/**'
+      - '.github/workflows/ci.yml'
 
 jobs:
   type_check:
     name: Type Check
     runs-on: "ubuntu-18.04"
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v1
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: Install Dependencies
       run: |
         pip install mypy
@@ -30,10 +31,10 @@ jobs:
     runs-on: "ubuntu-18.04"
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.6
+    - name: Set up Python 3.7
       uses: actions/setup-python@v1
       with:
-        python-version: 3.6
+        python-version: 3.7
     - name: Install Dependencies
       run: |
         pip install black==20.8b1
@@ -62,7 +63,7 @@ jobs:
         # https://github.com/conda-incubator/setup-miniconda
         uses: conda-incubator/[email protected]
         with:
-          python-version: 3.6
+          python-version: 3.7
           use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
           environment-file: environment_build.yml
           activate-environment: sleap
@@ -83,8 +84,14 @@ jobs:
       - name: Test with pytest (Ubuntu)
         if: matrix.os == 'ubuntu-18.04'
         shell: bash -l {0}
+          # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions
+          # sudo apt-get install xvfb libxkbcommon-x11-0
+          # sudo apt install libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
+          # export DISPLAY=":99.0"
+          # /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1920x1200x24 -ac +extension GLX
+          # xvfb-run pytest --cov=sleap --cov-report=xml tests/
         run: |
-          sudo apt-get install xvfb libxkbcommon-x11-0
+          sudo apt install xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xfixes0
           sudo Xvfb :1 -screen 0 1024x768x24 </dev/null &
           export DISPLAY=":1"
           xvfb-run pytest --cov=sleap --cov-report=xml tests/

dev_requirements.txt

@@ -1,6 +1,7 @@
 pytest
 pytest-qt
 pytest-cov
+pytest-xvfb
 ipython
 sphinx
 furo
@@ -12,4 +13,4 @@ pre-commit
 twine==3.3.0
 PyGithub
 jupyterlab
-jedi==0.17.2
+jedi==0.17.2

environment.yml

@@ -1,10 +1,10 @@
 name: sleap
 
 dependencies:
-- python=3.6
-- cudatoolkit=10.1
-- cudnn=7.6
-- sleap::tensorflow=2.3.1
+- python=3.7
+- cudatoolkit=11.3.1
+- cudnn=8.2.1
+- sleap::tensorflow=2.7.0
 - sleap::pyside2=5.14.1
 - Pillow
 - Shapely

environment_build.yml

@@ -1,11 +1,11 @@
 name: sleap
 
 dependencies:
-- python=3.6
+- python=3.7
+- sleap::tensorflow=2.7.0
+- sleap::pyside2=5.14.1
 - Pillow
 - Shapely
-- sleap::tensorflow=2.3.1
-- sleap::pyside2=5.14.1
 - conda-build=3.17
 - anaconda-client
 - conda-verify

environment_mac.yml

@@ -1,14 +1,14 @@
 name: sleap
 
 dependencies:
-- python=3.6
+- python=3.7
 - Pillow
 - Shapely
 - anaconda-client
 - conda-verify
 - pip
 - pip:
-    - tensorflow==2.3.1
+    - tensorflow==2.7.0
     - pyside2==5.14.1
     - "--editable=./"
     - "--requirement=./dev_requirements.txt"

environment_no_cuda.yml

@@ -1,8 +1,8 @@
 name: sleap
 
 dependencies:
-- python=3.6
-- sleap::tensorflow=2.3.1
+- python=3.7
+- sleap::tensorflow=2.7.0
 - sleap::pyside2=5.14.1
 - Pillow
 - Shapely

requirements.txt

@@ -1,18 +1,18 @@
-numpy>=1.18.1,<1.19.0
-attrs==19.3
-cattrs==1.0.0rc
+numpy==1.19.5
+attrs==21.2.0
+cattrs==1.1.1
 jsonpickle==1.2
 networkx
-tensorflow==2.3.1
-h5py==2.10.0
+tensorflow==2.7.0
+h5py==3.1.0
 python-rapidjson
 opencv-python-headless==4.2.0.34
 pandas
 psutil
 PySide2==5.14.1
 pyzmq
 pyyaml
-imgaug==0.3.0
+imgaug==0.4.0
 scipy<=1.4.1
 scikit-image
 scikit-learn

sleap/io/dataset.py

@@ -39,7 +39,7 @@
 """
 import itertools
 import os
-from collections import MutableSequence
+from collections.abc import MutableSequence
 from typing import (
     Callable,
     List,

sleap/io/format/hdf5.py

@@ -80,9 +80,7 @@ def read_headers(
 
         # Extract the Labels JSON metadata and create Labels object with just this
         # metadata.
-        dicts = json_loads(
-            f.require_group("metadata").attrs["json"].tostring().decode()
-        )
+        dicts = json_loads(f.require_group("metadata").attrs["json"].tobytes().decode())
 
         # These items are stored in separate lists because the metadata group got to be
         # too big.
@@ -280,7 +278,7 @@ def write(
 
                 # Otherwise, we need to read the JSON and append to the lists
                 old_labels = labels_json.LabelsJsonAdaptor.from_json_data(
-                    meta_group.attrs["json"].tostring().decode()
+                    meta_group.attrs["json"].tobytes().decode()
                 )
 
                 # A function to join to list but only include new non-dupe entries

sleap/io/format/labels_json.py

@@ -286,12 +286,13 @@ def write(
         if compress is None:
             compress = filename.endswith(".zip")
 
-        # Lets make a temporary directory to store the image frame data or pre-compressed json
-        # in case we need it.
+        # Lets make a temporary directory to store the image frame data or pre-
+        # compressed json in case we need it.
         with tempfile.TemporaryDirectory() as tmp_dir:
 
-            # If we are saving frame data along with the datasets. We will replace videos with
-            # new video object that represent video data from just the labeled frames.
+            # If we are saving frame data along with the datasets. We will replace
+            # videos with new video object that represent video data from just the
+            # labeled frames.
             if save_frame_data:
 
                 # Create a set of new Video objects with imgstore backends. One for each
@@ -314,7 +315,8 @@ def write(
                     # Change to relative path
                     vid.backend.filename = img_store_dir
 
-                # Convert to a dict, not JSON yet, because we need to patch up the videos
+                # Convert to a dict, not JSON yet, because we need to patch up the
+                # videos
                 d = labels.to_dict()
                 d["videos"] = Video.cattr().unstructure(new_videos)
 
@@ -328,16 +330,18 @@ def write(
 
                 # Ensure that filename ends with .json
                 # shutil will append .zip
-                filename = re.sub("(\\.json)?(\\.zip)?$", ".json", filename)
+                filename = re.sub(r"(\.json\.zip)$", ".json", filename)
 
-                # Write the json to the tmp directory, we will zip it up with the frame data.
+                # Write the json to the tmp directory, we will zip it up with the frame
+                # data.
                 full_out_filename = os.path.join(tmp_dir, os.path.basename(filename))
                 json_dumps(d, full_out_filename)
 
                 # Create the archive
                 shutil.make_archive(base_name=filename, root_dir=tmp_dir, format="zip")
 
-            # If the user doesn't want to compress, then just write the json to the filename
+            # If the user doesn't want to compress, then just write the json to the
+            # filename
             else:
                 json_dumps(d, filename)
 

sleap/io/video.py

@@ -238,7 +238,7 @@ def _try_frame_from_source_video(self, idx) -> np.ndarray:
         try:
             return self.source_video.get_frame(idx)
         except:
-            raise ValueError(f"Frame index {idx} not in original index.")
+            raise IndexError(f"Frame index {idx} not in original index.")
 
     # The properties and methods below complete our contract with the higher level
     # Video interface.