Merge pull request #40 from Grokzen/unstable

Major refactoring (0.2.0)

.gitignore

@@ -3,9 +3,10 @@
 *.swp
 env27*
 .tox
-rediscluster.egg-info
 .coverage*
 dump.rdb
-redis-git/
-htmlcov/
-dist/
+redis-git
+htmlcov
+dist
+build
+*.egg-info

.travis.yml

@@ -4,7 +4,7 @@ python:
 services:
   - redis-server
 install:
-  - make travis-install
+  - make redis-install
   - pip install coverage python-coveralls tox
   - pip install -e .
 script: 

CHANGES

@@ -1,3 +1,18 @@
+* 0.2.0
+    * Moved pipeline code into new file.
+    * Code now uses a proper cluster connection pool class that handles 
+      all nodes and connections similar to how redis-py do.
+    * Better support for pubsub. All clients will now talk to the same server because 
+      pubsub commands do not work reliably if it talks to a random server in the cluster.
+    * Better result callbacks and node routing support. No more ugly decorators.
+    * Fix keyslot command when using non ascii characters.
+    * Add bitpos support, redis-py 2.10.2 or higher required.
+    * Fixed a bug where vagrant users could not build the package via shared folder.
+    * Better support for CLUSTERDOWN error. (Neuront)
+    * Parallel pipeline execution using threads. (72squared)
+    * Added vagrant support for testing and development. (72squared)
+    * Improve stability of client during resharding operations (72squared)
+
 * 0.1.0
     * Initial release
     * First release uploaded to pypi

Makefile

@@ -73,12 +73,42 @@ cluster-enabled yes
 cluster-config-file /tmp/redis_cluster_node6.conf
 endef
 
+define REDIS_CLUSTER_NODE7_CONF
+daemonize yes
+port 7006
+cluster-node-timeout 5000
+pidfile /tmp/redis_cluster_node7.pid
+logfile /tmp/redis_cluster_node7.log
+save ""
+appendonly no
+cluster-enabled yes
+cluster-config-file /tmp/redis_cluster_node7.conf
+endef
+
+define REDIS_CLUSTER_NODE8_CONF
+daemonize yes
+port 7007
+cluster-node-timeout 5000
+pidfile /tmp/redis_cluster_node8.pid
+logfile /tmp/redis_cluster_node8.log
+save ""
+appendonly no
+cluster-enabled yes
+cluster-config-file /tmp/redis_cluster_node8.conf
+endef
+
+ifndef REDIS_TRIB_RB
+	REDIS_TRIB_RB=redis-git/src/redis-trib.rb
+endif
+
 export REDIS_CLUSTER_NODE1_CONF
 export REDIS_CLUSTER_NODE2_CONF
 export REDIS_CLUSTER_NODE3_CONF
 export REDIS_CLUSTER_NODE4_CONF
 export REDIS_CLUSTER_NODE5_CONF
 export REDIS_CLUSTER_NODE6_CONF
+export REDIS_CLUSTER_NODE7_CONF
+export REDIS_CLUSTER_NODE8_CONF
 
 help:
 	@echo "Please use 'make <target>' where <target> is one of"
@@ -96,7 +126,7 @@ help:
 	@echo "  cleanup         cleanup files after running a test cluster"
 	@echo "  test            starts/activates the test cluster nodes and runs tox test"
 	@echo "  tox             run all tox environments and combine coverage report after"
-	@echo "  travis-install  checkout latest redis commit --> build --> install ruby dependencies"
+	@echo "  redis-install  checkout latest redis commit --> build --> install ruby dependencies"
 
 clean:
 	-rm -f MANIFEST
@@ -134,6 +164,11 @@ start: cleanup
 	echo "$$REDIS_CLUSTER_NODE4_CONF" | redis-server -
 	echo "$$REDIS_CLUSTER_NODE5_CONF" | redis-server -
 	echo "$$REDIS_CLUSTER_NODE6_CONF" | redis-server -
+	echo "$$REDIS_CLUSTER_NODE7_CONF" | redis-server -
+	echo "$$REDIS_CLUSTER_NODE8_CONF" | redis-server -
+	sleep 5
+	echo "yes" | ruby $(REDIS_TRIB_RB) create --replicas 1 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005
+	sleep 5
 
 cleanup:
 	- rm -vf /tmp/redis_cluster_node*.conf 2>/dev/null
@@ -146,18 +181,19 @@ stop:
 	kill `cat /tmp/redis_cluster_node4.pid` || true
 	kill `cat /tmp/redis_cluster_node5.pid` || true
 	kill `cat /tmp/redis_cluster_node6.pid` || true
+	kill `cat /tmp/redis_cluster_node7.pid` || true
+	kill `cat /tmp/redis_cluster_node8.pid` || true
 	rm -f /tmp/redis_cluster_node1.conf
 	rm -f /tmp/redis_cluster_node2.conf
 	rm -f /tmp/redis_cluster_node3.conf
 	rm -f /tmp/redis_cluster_node4.conf
 	rm -f /tmp/redis_cluster_node5.conf
 	rm -f /tmp/redis_cluster_node6.conf
+	rm -f /tmp/redis_cluster_node7.conf
+	rm -f /tmp/redis_cluster_node8.conf
 
 test:
 	make start
-	sleep 5
-	echo "yes" | ruby redis-git/src/redis-trib.rb create --replicas 1 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005
-	sleep 5
 	make tox
 	make stop
 
@@ -167,8 +203,8 @@ tox:
 	coverage combine
 	coverage report
 
-travis-install:
-	[ ! -e redis-git ] && git clone https://github.com/antirez/redis.git redis-git || true
+redis-install:
+	[ ! -e redis-git ] && git clone --depth 1 https://github.com/antirez/redis.git redis-git || true
 	make -C redis-git -j4
 	gem install redis
 	sleep 3

README.md

@@ -4,21 +4,26 @@ Redis cluster client in python for the official cluster support targeted for red
 
 This project is a port of `redis-rb-cluster` by antirez, with alot of added functionality. The original source can be found at https://github.com/antirez/redis-rb-cluster
 
-[![Build Status](https://travis-ci.org/Grokzen/redis-py-cluster.svg?branch=master)](https://travis-ci.org/Grokzen+/redis-py-cluster) [![Coverage Status](https://coveralls.io/repos/Grokzen/redis-py-cluster/badge.png)](https://coveralls.io/r/Grokzen/redis-py-cluster) [![Latest Version](https://pypip.in/version/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![Downloads](https://pypip.in/download/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![Supported Python versions](https://pypip.in/py_versions/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![License](https://pypip.in/license/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/)
+[![Build Status](https://travis-ci.org/Grokzen/redis-py-cluster.svg?branch=master)](https://travis-ci.org/Grokzen/redis-py-cluster) [![Coverage Status](https://coveralls.io/repos/Grokzen/redis-py-cluster/badge.png)](https://coveralls.io/r/Grokzen/redis-py-cluster) [![Latest Version](https://pypip.in/version/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![Downloads](https://pypip.in/download/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![Supported Python versions](https://pypip.in/py_versions/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![License](https://pypip.in/license/redis-py-cluster/badge.svg)](https://pypi.python.org/pypi/redis-py-cluster/) [![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/Grokzen/redis-py-cluster?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Code Health](https://landscape.io/github/Grokzen/redis-py-cluster/unstable/landscape.svg)](https://landscape.io/github/Grokzen/redis-py-cluster/unstable)
 
 
 
 ## Dependencies & supported python versions
 
-- redis >= 2.9.1
+- redis >= 2.10.2
 - Cluster enabled redis servers. Only Redis 3.0 beta.7 and above is supported because of CLUSTER SLOTS command was introduced in that release.
+- Optional: hiredis >= 0.1.3
 
-Current python support is
+Hiredis is tested and supported on all supported python versions.
 
-- 2.7 + hiredis
-- 3.2 + hiredis
-- 3.3 + hiredis
-- 3.4 + hiredis
+Supported python versions:
+
+- 2.7.x
+- 3.2.x
+- 3.3.x
+- 3.4.1+
+
+Python 3.4.0 do not not work with pubsub because of segfault issues (Same as redis-py has). If rediscluster is runned on 3.4.0 it will raise RuntimeError exception and exit. If you get this error locally when running tox, consider using `pyenv` to fix this problem.
 
 
 
@@ -52,6 +57,12 @@ True
 'bar'
 ```
 
+The following imports can be imported from `redis` package. 
+
+- `RedisCluster` 
+- `StrictClusterPipeline` 
+- `ClusterPubSub` 
+
 
 
 ## Testing
@@ -75,7 +86,6 @@ To run all environments you need all supported python versions installed on your
 To run a specific python version use either `tox -e py27` or `tox -e py34`
 
 
-
 ## More documentation
 
 More detailed documentation can be found in `docs` folder.
@@ -85,6 +95,7 @@ More detailed documentation can be found in `docs` folder.
 - [Command differences](docs/Commands.md)
 - [Limitations and differences](docs/Limits_and_differences.md)
 - [Redisco support (Django ORM)](docs/Redisco.md)
+- [Threaded Pipeline support](docs/Threads.md)
 - [Authors](docs/Authors)
 
 

Vagrantfile

@@ -0,0 +1,57 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+=begin
+
+DOCUMENTATION
+=============
+
+This vagrant instance installs all the various versions of python needed to run the tests and installs redis-server cluster.
+
+do:
+
+```
+vagrant up
+vagrant ssh
+```
+
+once inside the vagrant instance you should be able to do:
+```
+cd /vagrant
+make test
+```
+
+This will put you in this current directory and run the tests inside of vagrant.
+
+=end
+
+VAGRANTFILE_API_VERSION = "2"
+
+$script = <<SCRIPT
+set -ex
+echo "" | sudo add-apt-repository ppa:fkrull/deadsnakes
+sudo apt-get update
+sudo apt-get install -y git curl python-dev python-pip python3.2-dev python3.3-dev make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm
+sudo pip install pep8 tox hiredis pyopenssl coverage
+cd /vagrant && sudo make redis-install
+curl -L -s https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash
+export PATH="$HOME/.pyenv/bin:$PATH"
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+pyenv install 3.4.1
+pyenv shell 3.4.1
+
+cat >> "$HOME/.bashrc" <<'EOF'
+export PATH="$HOME/.pyenv/bin:$PATH"
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+EOF
+
+SCRIPT
+
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = "redis-py-cluster"
+  config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box"
+  config.vm.provision "shell", inline: $script, :privileged => false
+end

benchmarks/simple.py

@@ -25,7 +25,7 @@ def loop(rc, reset_last_key=None):
             print("error {0}".format(e))
             time.sleep(1)
 
-    for i in xrange(last, 1000000000):
+    for i in xrange(last, 1000000000):  # noqa
         try:
             print("SET foo{0} {1}".format(i, i))
             rc.set("foo{0}".format(i), i)
@@ -42,7 +42,7 @@ def timeit(rc, itterations=50000):
     """ Time how long it take to run a number of set/get:s
     """
     t0 = time.time()
-    for i in xrange(0, itterations):
+    for i in xrange(0, itterations):  # noqa
         try:
             s = "foo{0}".format(i)
             rc.set(s, i)

dev-requirements.txt

@@ -0,0 +1,7 @@
+-r requirements.txt
+
+coverage >= 3.7.1
+hiredis >= 0.1.3
+pytest >= 2.5.0
+testfixtures >= 4.0.1
+mock == 1.0.1

docs/ALPHA.md

@@ -1,6 +1,6 @@
-# Alpha notes
+# Beta notes
 
-Release 0.1.0 of redis-cluster is a Alpha release.
+Release 0.2.0 of redis-cluster is a Beta release.
 
 Before using this lib please read the documentation and understand the major differences and problems that exists with redis clustering. The following documents is best to read.
 

docs/Authors

@@ -16,3 +16,4 @@ Authors who contributed code or testing:
 
  - Dobrite - https://github.com/dobrite
  - 72squared - https://github.com/72squared
+ - Neuron Teckid - https://github.com/neuront

docs/Cluster_Setup.md

@@ -16,12 +16,26 @@ See repo README for detailed instructions how to setup.
 
 # Vagrant
 
-Alternatively, you can also use vagrant to spin up redis cluster in a vm for testing.
+You can also use vagrant to spin up redis cluster in a vm for testing. The vm also provides all the python libraries needed to run the tests.
 
-A fully functional vagrant machine can be obtained at [Vagrant-redis-cluster](https://github.com/72squared/vagrant-redis-cluster).
+To use the vm, first install vagrant on your system (Instructions can be found at: http://www.vagrantup.com/).
+Navigate to root of this project in your ssh terminal and run these commands:
 
-See repo README for detailed instructions how to setup.
 
+```
+vagrant up && vagrant ssh
+```
+
+This will print out a bunch of debugging output as it installs redis-server and all the python libraries.
+If all is successful, you will be logged into the vagrant instance at the end.
+
+Once inside the vagrant instance you should be able to do:
+
+```
+cd /vagrant && make test
+```
+
+This will put you in the root directory of this project from within vagrant and run all of the tests.
 
 
 # Simple makefile

docs/Pubsub.md

@@ -10,16 +10,20 @@ Because of this, if there is some functionality that relies on an exact and corr
 
 Currently the only known workarounds is to:
 
-- Ignore the returned value or t
-- All clients talk to the same server in pubsub mode
+- Ignore the returned value
+- All clients talk to the same server
 - Use a non clustered redis server for pubsub operations
 
 Discussion on this topic can be found here: https://groups.google.com/forum/?hl=sv#!topic/redis-db/BlwSOYNBUl8
 
 
 
-## How this lib handels this problem
+# How pubsub works in RedisCluster
 
-Currently there is no special logic running on pubsub commands.
+In 0.2.0 a first solution to pubsub problem was implemented, but it contains some limitations.
 
-All tests have been adapted to run on just 1 server to verify that the pubsub functionality still works. In some situations like `PUBLISH` command, tests will be done to verify it works across the cluster but it will ignore returned value because of reasons described in this document.
+When a new RedisCluster instance is created it will now just after all slots is initialized determine what one node will be the pubsub node. Currently it will use the node with the highest port number.
+
+With this solution, pubsub will work in a cluster without any other major workarounds.
+
+All pubsub tests pass with this setup.