REVERT BACK -- BAD DAY

kristiyanto · Oct 10, 2016 · 8e8edaf · 8e8edaf
1 parent f6dcce3
commit 8e8edaf
Show file tree

Hide file tree

Showing 63 changed files with 145 additions and 291 deletions.
diff --git a/README.md b/README.md
diff --git a/elasticsearch/RESET DB.ipynb b/elasticsearch/RESET DB.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 236,
+   "execution_count": 144,
    "metadata": {
     "collapsed": false
    },
@@ -59,8 +59,7 @@
     "        'altdest2': {'type': 'geo_point', 'lat_lon': 'true', \"include_in_all\": \"false\"},\n",
     "        'altdestid2':{'type': 'string', \"include_in_all\": \"false\"}, \n",
     "        'origin': {'type': 'geo_point', 'lat_lon': 'true', \"include_in_all\": \"false\"},\n",
-    "        'path': {'type': 'string', \"include_in_all\": \"false\"},\n",
-    "        'tripdistance': {'type': 'string', \"include_in_all\": \"false\"}\n",
+    "        'path': {'type': 'string', \"include_in_all\": \"false\"}\n",
     "      }\n",
     "    }\n",
     "  }\n",
@@ -71,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 237,
+   "execution_count": 145,
    "metadata": {
     "collapsed": false
    },
@@ -82,7 +81,7 @@
        "{u'acknowledged': True}"
       ]
      },
-     "execution_count": 237,
+     "execution_count": 145,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -94,7 +93,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 238,
+   "execution_count": 146,
    "metadata": {
     "collapsed": false
    },
@@ -105,7 +104,7 @@
        "{u'acknowledged': True}"
       ]
      },
-     "execution_count": 238,
+     "execution_count": 146,
      "metadata": {},
      "output_type": "execute_result"
     }

diff --git a/elasticsearch/elasticschema.txt b/elasticsearch/elasticschema.txt
diff --git a/flask/app/__init__.py b/flask/app/__init__.py
diff --git a/flask/app/__init__.pyc b/flask/app/__init__.pyc
diff --git a/flask/app/static/dist/css/.Rhistory b/flask/app/static/dist/css/.Rhistory
diff --git a/flask/app/static/qr.jpg b/flask/app/static/qr.jpg
diff --git a/flask/app/templates/index.html b/flask/app/templates/index.html
@@ -292,7 +292,19 @@ <h1 class="page-header">http://venturo.kristiyanto.me</h1>
     var DATA;
 
 
-
+    function drawPath()
+    {
+        $.getJSON("lines", function(path){
+        polyline = new L.Polyline(path.path)
+        mymap.addLayer(polyline)
+        });
+
+        var circle = L.circle(path.location, 50, {
+        color: 'yellow',
+        fillOpacity: 0.5
+        }).addTo(mymap);
+
+    }
 
     function currentTime()
     {
@@ -337,6 +349,11 @@ <h1 class="page-header">http://venturo.kristiyanto.me</h1>
       color: 'black',
       fillOpacity: 0.2
       }).addTo(mymap);
+
+      var circle = L.circle(data.arrMsgLoc, 250, {
+      color: 'yellow',
+      fillOpacity: 0.5
+      }).addTo(mymap);
 
       var circle = L.circle(data.arrMsgLoc, 30, {
       color: 'black',
@@ -397,6 +414,14 @@ <h1 class="page-header">http://venturo.kristiyanto.me</h1>
     }
 
 
+    $(function () {
+    setInterval(currentTime, 500);
+    setInterval(clearMap, 4999);
+    setInterval(drawPassengers, 5000);
+    setInterval(drawDrivers, 5000);
+    setInterval(drawPath, 1000);
+
+    });
 
     function centerIt(){
      mymap.panTo(DATA.mLoc);
@@ -414,39 +439,7 @@ <h1 class="page-header">http://venturo.kristiyanto.me</h1>
      mymap.panTo(DATA.arrMsgLoc);
     }
 
-    function drawPath()
-    {
-        var path
-        var _first
-        var _last
-        $.getJSON("lines", function(p){
-
-        if (p.path) {
-        polyline = new L.Polyline(p.path)
-        mymap.addLayer(polyline)
-
-        var circle = L.circle(p.location, 50, {
-        color: 'yellow',
-        fillOpacity: 0.5
-        }).addTo(mymap);
-
-        var circle = L.circle(p.path[0], 50, {
-        color: 'purple',
-        fillOpacity: 0.5
-        }).addTo(mymap);
-        }
 
-        });
-    }
-
-    $(function () {
-    setInterval(currentTime, 500);
-    setInterval(clearMap, 4999);
-    setInterval(drawPassengers, 5000);
-    setInterval(drawDrivers, 5000);
-    setInterval(drawPath, 1000);
-
-    });
 
     </script>
 

diff --git a/flask/app/views.py b/flask/app/views.py
@@ -89,6 +89,9 @@ def lines():
     res = arrivedMsg()
     if len(res['hits']['hits']) > 0:
         res = res['hits']['hits'][0]['_source']
+        path = res['path']
+        sorted(path , key=lambda k: [k[1], k[0]])
+        res['path'] = path
     else:
         res = []
     return json.dumps(res)

diff --git a/flask/app/views.pyc b/flask/app/views.pyc
diff --git a/flask/run.py b/flask/run.py
diff --git a/flask/tornadoapp.py b/flask/tornadoapp.py
diff --git a/kafka/archive/Driver_Generator.ipynb b/kafka/archive/Driver_Generator.ipynb
diff --git a/kafka/archive/Kafka_Producer.ipynb b/kafka/archive/Kafka_Producer.ipynb
diff --git a/kafka/archive/Passenger_Generator.ipynb b/kafka/archive/Passenger_Generator.ipynb
diff --git a/kafka/archive/Untitled.ipynb b/kafka/archive/Untitled.ipynb
diff --git a/kafka/archive/consumer.py b/kafka/archive/consumer.py
diff --git a/kafka/archive/producer.py b/kafka/archive/producer.py
diff --git a/kafka/boundaries.csv b/kafka/boundaries.csv
diff --git a/kafka/create_topic.txt b/kafka/create_topic.txt
diff --git a/kafka/d.sh b/kafka/d.sh
diff --git a/kafka/destinations.csv b/kafka/destinations.csv
diff --git a/kafka/driver_producer.py b/kafka/driver_producer.py
@@ -1,6 +1,6 @@
 # This is the script to populate driver's Data
 # {driver_id, time, curr_lat, curr_long, dest, load}
-total_drivers = 5000
+total_drivers = 2000
 
 
 import time
@@ -32,12 +32,7 @@
 es = Elasticsearch(cluster, port=9200)
 
 def convertTime(tm):
-    try:
-        t = datetime.strptime("{}".format(tm),'%Y-%m-%dT%H:%M:%S.%fZ')
-        t = t.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
-    except:
-        t = datetime.strptime("{}".format(tm),'%Y-%m-%d %H:%M:%S.%f')
-        t = t.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+    t = datetime.strptime("{}".format(tm),'%Y-%m-%dT%H:%M:%S.%fZ')
     return t
 
 
@@ -68,13 +63,13 @@ def simulateTrip(c, d):
 
 
 def generateDriver(city):
-    driverID = random.randint(1, total_drivers)
+    d_id = random.randint(1, total_drivers)
     driver_mapping ={ 
             'city': city,
-            'name': 'driver_{}'.format(driverID),
-            'id': driverID,
+            'name': 'driver_{}'.format(d_id),
+            'id': d_id,
             'status': 'idle',
-            'location': randomLoc(driverID, city),
+            'location': randomLoc(d_id, city),
             'ctime': str(datetime.now()),
             'p1': None,
             'p2': None,
@@ -87,17 +82,15 @@ def generateDriver(city):
             'origin': None,
         }
 
-    q = es.get(index='driver', doc_type='rolling', id=driverID, ignore=[404, 400])
-    if q['found'] and (q['_source']['status'] in ['ontrip', 'pickup', '']): 
+    q = es.get(index='driver', doc_type='rolling', id=d_id, ignore=[404, 400])
+    if q['found'] and (q['_source']['status'] in ['ontrip', 'pickup']): 
         driver_mapping = q['_source']
         driver_mapping['location'] = simulateTrip(driver_mapping['location'], driver_mapping['destination'])
     if q['found'] and (q['_source']['status'] in ['arrived']): 
         driver_mapping['ctime'] = convertTime(driver_mapping['ctime'])
         doc = json.dumps(driver_mapping)
-        doc = '{{"doc": {},  "doc_as_upsert" : "true"}}'.format(doc)
-
-        q = es.update(index='driver', doc_type='rolling', id=driverID, ignore=[404, 400], body=doc)
-        return False
+        q = es.update(index='driver', doc_type='rolling', id=d_id, ignore=[404, 400],
+                     body={'doc': doc, "doc_as_upsert" : "true"})
 
     return(driver_mapping)
 
@@ -108,11 +101,10 @@ def main():
     for n in range(total_drivers):
         city = random.choice(['CHI','NYC'])
         driver = generateDriver(city)
-        if driver:
-            u_json = json.dumps(driver).encode('utf-8')
-            key = json.dumps(driver['id']).encode('utf-8')
-            #print('{}'.format(driver))
-            producer.send(b'drv', key, u_json)
+        u_json = json.dumps(driver).encode('utf-8')
+        key = json.dumps(driver['id']).encode('utf-8')
+        print('{}'.format(driver))
+        producer.send(b'drv', key, u_json)
     kafka.close()
 
 if __name__ == "__main__":

diff --git a/kafka/passenger_producer.py b/kafka/passenger_producer.py
@@ -3,7 +3,7 @@
 # for the stream. Ideally, this would come from User.
 # Boundaries.csv contains information about the location boundaries for the generation.
 
-totalPassenger = 15000
+totalPassenger = 20000
 # Once generated, the request then sent to Kafka.
 
 import csv
@@ -69,14 +69,13 @@ def retention(passanger):
     res = es.search(index='passenger', doc_type='rolling', body=q, ignore=[404, 400])
     return res['hits']["total"]
 
-def convertTime(tm):
+def convertTime(ctime):
     try:
-        t = datetime.strptime("{}".format(tm),'%Y-%m-%dT%H:%M:%S.%fZ')
-        t = t.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
+        tmp = datetime.strptime("{}".format(ctime), '%Y-%m-%d %H:%M:%S.%f')
+        ctime = tmp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
     except:
-        t = datetime.strptime("{}".format(tm),'%Y-%m-%d %H:%M:%S.%f')
-        t = t.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
-    return t
+        print "Time conversion failed"
+    return ctime
 
 def generatePassenger(city, ID):
     last_uid = ID
@@ -101,7 +100,7 @@ def generatePassenger(city, ID):
             'altdest2': att[2][1],
             'altdest2id': att[2][0],
             'origin': [curr_lat, curr_long],
-            'path': [[curr_lat, curr_long]]
+            'path': [curr_lat, curr_long]
           }
 
     q = es.get(index='passenger', doc_type='rolling', id=ID, ignore=[404, 400])

diff --git a/kafka/read_topic.txt b/kafka/read_topic.txt
diff --git a/kafka/start_secor.sh b/kafka/start_secor.sh
diff --git a/media/architecture.png b/media/architecture.png
diff --git a/media/docker.png b/media/docker.png
diff --git a/media/insight.png b/media/insight.png
diff --git a/media/screen_big.png b/media/screen_big.png
diff --git a/media/screenshot.png b/media/screenshot.png
diff --git a/media/youtube.png b/media/youtube.png
diff --git a/media/youtube_big.png b/media/youtube_big.png
diff --git a/recommendation/als.ipynb b/recommendation/als.ipynb
diff --git a/recommendation/als.scala b/recommendation/als.scala
diff --git a/recommendation/data/data_row_number.txt b/recommendation/data/data_row_number.txt
diff --git a/recommendation/data/gps.csv.zip b/recommendation/data/gps.csv.zip
diff --git a/spark/archive/.ipynb_checkpoints/SCRAPBOOK-checkpoint.ipynb b/spark/archive/.ipynb_checkpoints/SCRAPBOOK-checkpoint.ipynb