diff --git a/deploy/spark/Dockerfile b/deploy/spark/Dockerfile index fa6ecdb..41ff55f 100644 --- a/deploy/spark/Dockerfile +++ b/deploy/spark/Dockerfile @@ -24,5 +24,6 @@ RUN echo "JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk/" >> /opt/spark-1.2.0-bin-ha #CMD echo "XXXXXX" && cat /etc/hosts && echo "YYYY" && ip addr | grep 17 && hostname && if [[ `hostname` = 'scale1.docker' ]] ; then /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; else ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ; fi ; tailf /opt/spark-1.2.0-bin-hadoop2.4/logs/* ADD run.sh /opt/run.sh -CMD if [[ `hostname` = 'scale1.docker' ]] ; then /opt/run.sh master ; else /opt/run.sh slave ; fi + +CMD echo `hostname` && if [[ `hostname` = 'scale1.docker' ]] ; then /opt/run.sh master ; else /opt/run.sh slave ; fi # CMD /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh diff --git a/deploy/spark/run.sh b/deploy/spark/run.sh index 4bbf6d2..819c7d9 100755 --- a/deploy/spark/run.sh +++ b/deploy/spark/run.sh @@ -1,18 +1,10 @@ #/bin/bash - -if [ $# -eq 0 ]; then - echo "No arguments supplied" - echo "USAGE ./run.sh master|slave" - exit 1 -fi - -echo "running docker spark startup" - +echo "starting ........" if [ $1 = "master" ] ; then - /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; -# slave + /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-master.sh ; else - ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ; -fi -### In all cases, we tail the logs as the final task... -tailf /opt/spark-1.2.0-bin-hadoop2.4/logs/* + echo "pinging slave........" + ping -c 2 scale1.docker && /opt/spark-1.2.0-bin-hadoop2.4/sbin/start-slave.sh -h spark://scale1.docker:7077 ; +fi + +tail -f /opt/spark-1.2.0-bin-hadoop2.4/logs/* diff --git a/deploy/vagrant-run-up-test.sh b/deploy/vagrant-run-up-test.sh index 5cb1504..7e16391 100755 --- a/deploy/vagrant-run-up-test.sh +++ b/deploy/vagrant-run-up-test.sh @@ -20,7 +20,7 @@ echo "RUNNING smoke tests..." if `sleep 2 && docker ps | grep -q scale1`; then - docker exec -i -t scale1 /opt/spark-1.2.0-bin-hadoop2.4/bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://scale1.docker:7077 /scale-shared/spark-examples_2.10-1.1.1.jar 10000 + docker exec -i -t scale1 /opt/spark-1.2.0-bin-hadoop2.4/bin/spark-submit --class org.apache.spark.examples.SparkPi --master spark://scale1.docker:7077 /scale-shared/spark-examples_2.10-1.1.1.jar 100 else echo "FAILED: Looks like vagrant creation of the spark containers failed ..." docker ps