Moving PersistentDict from Frambo

user-cont/frambo#14

.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+# HOWTO: https://pre-commit.com/#usage
+# pip3 install pre-commit
+# pre-commit install
+
+repos:
+-   repo: https://github.com/ambv/black
+    rev: 19.3b0
+    hooks:
+    - id: black
+      language_version: python3.7
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    - id: check-added-large-files
+    - id: check-ast
+    - id: check-merge-conflict
+    - id: check-yaml
+    - id: detect-private-key
+      exclude: tests/conftest.py
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+    - id: flake8
+      args:
+        - --max-line-length=100
+        - --per-file-ignores=files/packit.wsgi:F401,E402
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.720
+    hooks:
+    -   id: mypy
+        args: [--no-strict-optional, --ignore-missing-imports]

.travis.yml

@@ -0,0 +1,6 @@
+sudo: required
+language: bash
+services:
+- docker
+script:
+- bash test-with-redis.sh

README.md

@@ -1 +1,32 @@
-# persistentdict
+# persistentdict
+
+Yet another [1] dictionary backed by Redis DB.
+
+We use Redis` 'hash' type [2] and store whole dictionary in one hash.
+
+Usage:
+```python
+from persistentdict import PersistentDict
+
+db = PersistentDict(hash_name="my-persistent-dict")
+
+# add key to the db with a value
+db[key] = value
+
+# show whole dictionary
+print(db)
+
+# iterate over keys & values in db
+for key, value in db.items():
+  do_something(key)
+
+# do sth with key if it is in db
+if key in db:
+  do_something(key)
+
+# delete key from db
+del db[key]
+```
+
+[1] Alternatives: [persistent-dict](https://github.com/richardARPANET/persistent-dict), [durabledict](https://github.com/disqus/durabledict/)
+[2] https://redis.io/topics/data-types-intro#hashes is basically Python's dict, but values can be strings only, so we use json serialization

persistentdict/dict_in_redis.py

@@ -0,0 +1,169 @@
+# MIT License
+#
+# Copyright (c) 2019 Red Hat, Inc.
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import json
+from os import getenv
+
+from redis import Redis
+
+
+class PersistentDict:
+    """
+    Dictionary backed by Redis DB.
+
+    We use Redis` 'hash' type [1] and store whole dictionary in one hash called self.hash.
+
+    Usage:
+    db = PersistentDict(hash_name="my-persistent-dict")
+    # add bug to the db with a value
+    db[key] = value
+    # show whole dictionary
+    print(db)
+    # iterate of bugs in db
+    for key, value in db.items():
+      do_something(key)
+    # do sth with key if it is in db
+    if key in db:
+      do_something(key)
+    # delete bug from db
+    del db[key]
+
+    [1] https://redis.io/topics/data-types-intro#hashes is basically Python's dict, but values
+        can be strings only, so we use json serialization
+    """
+
+    def __init__(self, hash_name="dict-in-redis"):
+        """
+
+        :param hash_name: name of the dictionary/hash [1] we store all the info in
+        """
+        self.db = Redis(
+            host=getenv("REDIS_SERVICE_HOST", "localhost"),
+            port=getenv("REDIS_SERVICE_PORT", "6379"),
+            db=1,  # 0 is used by Celery
+            decode_responses=True,
+        )
+        self.hash = hash_name
+
+    def __contains__(self, key):
+        """
+        Is key in db ?
+
+        Usage:
+        if key in PersistentDict():
+
+        :param key: can be int or string
+        :return: bool
+        """
+        return self.db.hexists(self.hash, key)
+
+    def __getitem__(self, key):
+        """
+        Get info to key
+
+        Usage:
+        xyz = PersistentDict()[key]
+
+        :param key: can be int or string
+        :return: value assigned to the key or None if key not in db
+        """
+        value = self.db.hget(self.hash, key)
+        if value is None:
+            return None
+        return json.loads(value)
+
+    def __len__(self):
+        """
+
+        Number of items in db.
+
+        Usage:
+        len(PersistentDict())
+        """
+        return self.db.hlen(self.hash)
+
+    def __setitem__(self, key, value):
+        """
+        Store key in db along with a value.
+        Because values in a hash can be only strings, we first json serialize the value
+
+        Usage:
+        PersistentDict()[key] = value
+
+        :param key: can be int or string
+        :param value: additional info, can be any json serializable object
+        """
+        self.db.hset(self.hash, key, json.dumps(value))
+
+    def __delitem__(self, key):
+        """
+        Remove key from db
+
+        Usage:
+        del PersistentDict()[key]
+
+        :param key: can be int or string
+        """
+        self.db.hdel(self.hash, key)
+
+    def __repr__(self):
+        """
+        print(PersistentDict())
+
+        :return: string representation
+        """
+        return str(self.get_all())
+
+    def clear(self):
+        """
+        Remove all items from dictionary
+        """
+        for key in self.keys():
+            self.__delitem__(key)
+
+    def get_all(self):
+        """
+        Return whole dictionary
+
+        Usage:
+        all_bugs_dict = PersistentDict().get_all()
+
+        :return: dictionary of {key: value}
+        """
+        return {k: json.loads(v) for k, v in self.db.hgetall(self.hash).items()}
+
+    def items(self):
+        """
+        Return iterator over the (key, value) pairs
+
+        Usage:
+        for key, value in PersistentDict().items():
+
+        :return: iterator over the (key, value) pairs
+        """
+        return self.get_all().items()
+
+    def keys(self):
+        """
+        :return: view object that displays a list of all the keys
+        """
+        return self.get_all().keys()

setup.cfg

@@ -0,0 +1,46 @@
+[metadata]
+name = persistentdict
+url = https://github.com/user-cont/persistentdict
+description = Yet another dictionary backed by Redis DB
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = Red Hat
+author_email = [email protected]
+license = MIT
+license_file = LICENSE
+classifiers =
+    Environment :: Console
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Topic :: Software Development
+    Topic :: Utilities
+keywords =
+    dict
+    durable
+    persistent
+    redis
+
+
+[options]
+packages = find:
+python_requires = >=3.6
+include_package_data = True
+
+setup_requires =
+    setuptools_scm
+    setuptools_scm_git_archive
+
+install_requires =
+    redis
+
+[options.packages.find]
+exclude =
+    tests*
+
+[options.extras_require]
+testing =
+    pytest

setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+
+from setuptools import setup
+
+setup(use_scm_version=True)

test-with-redis.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+
+gc() {
+  retval=$?
+  echo "Stopping redis container"
+  docker stop redis || :
+  echo "Removing test containers"
+  docker rm redis || :
+  exit $retval
+}
+
+trap gc EXIT SIGINT
+
+echo "Starting redis"
+docker run --rm -d -p 6379:6379 --name=redis registry.fedoraproject.org/f28/redis
+
+echo "Starting test suite"
+python3 -m pytest --color=yes --verbose --showlocals
+
+echo "Test suite passed \\o/"

tests/test_dict_in_redis.py

@@ -0,0 +1,41 @@
+from uuid import uuid4
+
+from pytest import fixture
+
+from persistentdict.dict_in_redis import PersistentDict
+
+
+class TestPersistentDict:
+    @fixture
+    def db(self):
+        return PersistentDict(hash_name=str(uuid4()))
+
+    @fixture
+    def dictionary(self):
+        return {
+            "key": "some value",
+            "key2": ["an", "array"],
+            "key3": {"a": "dictionary", "b": 2, "c": ["one"]},
+        }
+
+    def test_one_by_one(self, db, dictionary):
+        for key, value in dictionary.items():
+            assert key not in db
+            db[key] = value
+            assert key in db
+            assert db[key] == value
+            del db[key]
+            assert key not in db
+
+    def test_more_keys(self, db, dictionary):
+        assert len(db) == 0
+        for key, value in dictionary.items():
+            assert key not in db
+            db[key] = value
+        assert len(db) == len(dictionary)
+        assert len(db.get_all()) == len(dictionary)
+        assert db.get_all() == dictionary
+        assert len(db.items()) == len(dictionary)
+        assert db.keys() == dictionary.keys()
+        db.clear()
+        assert len(db) == 0