cleaned up

deploy/Vagrantfile

@@ -4,7 +4,7 @@
 require 'fileutils'
 
 # number of instances : First one is master.
-$spark_num_instances = 5
+$spark_num_instances = 2
 $c_num = 1
 
 Vagrant.configure("2") do |config|
@@ -30,9 +30,11 @@ Vagrant.configure("2") do |config|
                 d.name = "scale#{i}"
                 d.create_args = ["--privileged=true"]
                 d.remains_running = true
+                # d.cmd = "echo xyz && if [[ `hostname` = 'scale1.docker' ]] ; then /opt/run.sh master ; else /opt/run.sh slave ;  fi"
                 if "#{i}" == "1"
                     d.ports = [ "4040:4040", "7707:7707" ] 
                 else
+                    #puts "ASDF";
                     ### Note that there should be a cassandra1.docker listening in cassandra.yaml.
                     d.create_args = d.create_args << "--link" << "scale1:scale1.docker" << "--link" << "cassandra1:cassandra1.docker"
                 end

deploy/spark/Dockerfile

@@ -24,5 +24,5 @@ RUN echo "JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk/" >> /opt/spark-1.2.0-bin-ha
 #CMD echo "XXXXXX" && cat /etc/hosts && echo "YYYY" && ip addr | grep 17 && hostname && if [[ `hostname` = 'scale1.docker' ]] ; then /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; else ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ; fi ; tailf /opt/spark-1.2.0-bin-hadoop2.4/logs/*
 
 ADD run.sh /opt/run.sh
-# CMD /opt/run.sh
+CMD if [[ `hostname` = 'scale1.docker' ]] ; then /opt/run.sh master ; else /opt/run.sh slave ;  fi
 # CMD /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh

deploy/spark/run.sh

@@ -6,28 +6,13 @@ if [ $# -eq 0 ]; then
         exit 1
 fi   
 
-### Vagrant : This is untested, but it should generally work.
-if [[ `hostname -s` =~ scale[0-9]*docker ]]; then
-    if [ $1 = "master" ] ; then 
-        /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; 
-    # slave
-    else
-        ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ; 
-    fi 
+echo "running docker spark startup"
 
-### Kubernetes
+if [ $1 = "master" ] ; then 
+        /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; 
+# slave
 else
-
-    # One arg expected when launching kubernetes (master or slave)
-    if [ $1 = "slave" ]; then 
-       echo "START SLAVE pointing to ${SPARK_MASTER_SERVICE_HOST} " 
-       # SPARK_MASTER_SERVICE_HOST is 1.2.3.4:7077
-       /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://${SPARK_MASTER_SERVICE_HOST}:7077
-    # master
-    else 
-       /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh
-
-    fi
+        ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ;  
 fi
 ### In all cases, we tail the logs as the final task...
 tailf /opt/spark-1.2.0-bin-hadoop2.4/logs/*

deploy/vagrant-run-up-test.sh

@@ -18,9 +18,13 @@ vagrant destroy --force && vagrant up --no-parallel
 
 echo "RUNNING smoke tests..." 
 
-docker exec -i -t scale1 \
-/opt/spark-1.2.0-bin-hadoop2.4/bin/spark-submit \
---class org.apache.spark.examples.SparkPi \
---master spark://scale1.docker:7077 /scale-shared/spark-examples_2.10-1.1.1.jar 10000
+
+if `sleep 2 && docker ps | grep -q scale1`; then
+    docker exec -i -t scale1 /opt/spark-1.2.0-bin-hadoop2.4/bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://scale1.docker:7077 /scale-shared/spark-examples_2.10-1.1.1.jar 10000
+else
+    echo "FAILED: Looks like vagrant creation of the spark containers failed ..."
+    docker ps
+fi
+
 
 echo "DONE TESTING .  RESULTS ^ "