diff --git a/.gitignore b/.gitignore
index f6f295c..a2bee49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,9 @@ email_password.txt
__pycache__
glove.twitter.27B.*
numberbatch-en-17.04b.*
+bae_docker.sh
+password.txt
docker-command-smile.sh
docker-command-bae.sh
-test_orca
\ No newline at end of file
+test_orca
+checkpoint-17315-epoch-5
\ No newline at end of file
diff --git a/batch/batch_classification_predict.py b/batch/batch_classification_predict.py
deleted file mode 100644
index bf0df20..0000000
--- a/batch/batch_classification_predict.py
+++ /dev/null
@@ -1,159 +0,0 @@
-import csv
-import os
-from os.path import join, dirname
-import pickle
-import numpy as np
-from plotly.offline import plot
-import plotly.graph_objs as go
-from collections import Counter
-import json
-import writeToS3 as s3
-import argparse
-import deleteDir as d
-import notification as n
-
-class Classification:
-
- def __init__(self, awsPath, localSavePath, filename):
-
- self.localSavePath = localSavePath
- self.awsPath = awsPath
- self.filename = filename
-
- def predict(self):
-
- # load classification model
- pkl_model = os.path.join(self.localSavePath,'classification_pipeline.pickle')
- with open(pkl_model,'rb') as f:
- text_clf = pickle.load(f)
-
- # load text set
- data = []
- try:
- with open(self.localSavePath + 'UNLABELED_' + self.filename + '.csv','r',encoding='utf-8') as f:
- reader = list(csv.reader(f))
- for row in reader[1:]:
- try:
- data.extend(row)
- except Exception as e:
- pass
- except:
- with open(self.localSavePath + 'UNLABELED_' + self.filename + '.csv','r',encoding='ISO-8859-1') as f:
- reader = list(csv.reader(f))
- for row in reader[1:]:
- try:
- data.extend(row)
- except Exception as e:
- pass
-
- # predict using trained model
- self.predicted = text_clf.predict(data)
-
- # save result
- fname = 'PREDICTED_' + self.filename + '.csv'
- try:
- with open(self.localSavePath + fname,'w',newline="",encoding='utf-8') as f:
- writer = csv.writer(f)
- writer.writerow(['text','category'])
- for i in range(len(data)):
- try:
- writer.writerow([data[i],self.predicted[i]])
- except:
- pass
- except:
- with open(self.localSavePath + fname,'w',newline="",encoding='ISO-8859-1') as f:
- writer = csv.writer(f)
- writer.writerow(['text','category'])
- for i in range(len(data)):
- try:
- writer.writerow([data[i],self.predicted[i]])
- except:
- pass
- s3.upload(self.localSavePath, self.awsPath, fname)
- return s3.generate_downloads(self.awsPath, fname)
-
-
- def plot(self):
- y_pred_dict = Counter(self.predicted)
- labels = []
- values = []
- for i in y_pred_dict.keys():
- labels.append("class: " + str(i))
- values.append(y_pred_dict[i])
- trace = go.Pie(labels=labels, values = values, textinfo='label')
- div_comp = plot([trace], output_type='div',image='png',auto_open=False, image_filename='plot_img')
-
- fname_div_comp = 'div_comp.html'
- with open(self.localSavePath + fname_div_comp,"w") as f:
- f.write(div_comp)
- s3.upload(self.localSavePath, self.awsPath, fname_div_comp)
- return s3.generate_downloads(self.awsPath, fname_div_comp)
-
-
-
-if __name__ == '__main__':
-
- output = dict()
-
- parser = argparse.ArgumentParser(description="processing...")
- parser.add_argument('--remoteReadPath', required=True)
- parser.add_argument('--uuid',required=True)
- parser.add_argument('--s3FolderName',required=True)
- parser.add_argument('--email',required=True)
- args = parser.parse_args()
-
- uid = args.uuid
- awsPath = args.s3FolderName + '/ML/classification/' + uid +'/'
- localSavePath = '/tmp/' + args.s3FolderName + '/ML/classification/' + uid + '/'
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
-
- # download config to local folder
- fname_config = 'config.json'
- if s3.checkExist(awsPath, fname_config):
- s3.downloadToDisk(fname_config, localSavePath, awsPath)
- with open(localSavePath + fname_config, "r") as fp:
- data = json.load(fp)
- for key in vars(args).keys():
- if key not in data.keys():
- data[key] = vars(args)[key]
- with open(localSavePath + fname_config,"w") as f:
- json.dump(data,f)
- s3.upload(localSavePath, awsPath, fname_config)
- output['config'] = s3.generate_downloads(awsPath, fname_config)
- output['uuid'] = uid
-
- else:
- raise ValueError('This session ID is invalid!')
- exit()
-
-
- # download unlabeled data to local folder
- filename = args.remoteReadPath.split('/')[-2]
- fname_unlabeled = 'UNLABELED_' + filename +'.csv'
- if s3.checkExist(awsPath, fname_unlabeled):
- s3.downloadToDisk(fname_unlabeled, localSavePath, awsPath)
- else:
- raise ValueError('You\'re requesting ' + fname_unlabeled + ' file, and it\'s not found in your remote directory!\
- It is likely that you have not yet performed step 1 -- split the dataset into training and predicting set, or you have provided the wrong sessionID.')
- exit()
-
- #download pickle model to local folder
- fname_pickle = 'classification_pipeline.pickle'
- if s3.checkExist(awsPath, fname_pickle):
- s3.downloadToDisk(fname_pickle, localSavePath, awsPath)
- else:
- raise ValueError('You\'re requesting ' + fname_pickle + ' file, and it\'s not found in your remote directory! \
- It is likely that you have not yet performed step 2 -- model training, or you have provided the wrong sessionID.')
- exit()
-
-
- classification = Classification(awsPath, localSavePath, filename)
- output['predict'] = classification.predict()
- output['div'] = classification.plot()
-
- d.deletedir('/tmp')
- n.notification(args.email,case=3,filename=awsPath)
-
diff --git a/batch/batch_classification_split.py b/batch/batch_classification_split.py
deleted file mode 100644
index e650405..0000000
--- a/batch/batch_classification_split.py
+++ /dev/null
@@ -1,177 +0,0 @@
-import csv
-import pandas
-import random
-import uuid
-import os
-from plotly.offline import plot
-import plotly.graph_objs as go
-from os.path import join, dirname
-import json
-import re
-import writeToS3 as s3
-import deleteDir as d
-import notification as n
-import argparse
-
-class Classification:
-
- def __init__(self,awsPath, localSavePath, localReadPath, remoteReadPath):
-
- self.localSavePath = localSavePath
- self.awsPath = awsPath
-
- # download remote socialmedia data into a temp folder
- # load it into csv
- filename = remoteReadPath.split('/')[-2] + '.csv'
- self.filename = filename # save it so split function can reuse this name
- s3.downloadToDisk(filename=filename,localpath=localReadPath, remotepath=remoteReadPath)
-
- Array = []
- try:
- with open(localReadPath + filename,'r',encoding='utf-8') as f:
- reader = csv.reader(f)
- for row in reader:
- try:
- Array.append(row)
- except Exception as e:
- pass
- except:
- with open(localReadPath + filename,'r',encoding='ISO-8859-1') as f:
- reader = csv.reader(f)
- for row in reader:
- try:
- Array.append(row)
- except Exception as e:
- pass
-
- df = pandas.DataFrame(Array[1:], columns=Array[0])
-
- # remoteReadPath always follows format of sessionID/folderID/datasetName/
- # example: local/GraphQL/twitter-Tweet/trump/ => ['local','GraphQL', 'twitter-Tweet','trump','']
- source = remoteReadPath.split('/')[2]
-
- if (source == 'twitter-Tweet') and ('text' in Array[0]):
- self.corpus = list(set(df[df['text']!='']['text'].dropna().astype('str').tolist()))
- elif (source == 'twitter-Stream') and ('_source.text' in Array[0]):
- self.corpus = list(set(df[df['_source.text']!='']['_source.text'].dropna().astype('str').tolist()))
-
- # find the unique content in crimson hexagon
- elif (source=='crimson-Hexagon') and ('contents' in Array[0]):
- self.corpus = list(set(df[df['contents'] != '']['contents'].dropna().astype('str').tolist()))
-
- # find the unique title in reddit posts
- elif (source=='reddit-Search' or source=='reddit-Post') and 'title' in Array[0]:
- self.corpus = list(set(df[df['title']!='']['title'].dropna().astype('str').tolist()))
- elif source =='reddit-Historical-Post' and '_source.title' in Array[0]:
- self.corpus = list(set(df[df['_source.title']!='']['_source.title'].dropna().astype('str').tolist()))
-
- # find the unique body in reddit comments
- elif (source == 'reddit-Comment' or source == 'reddit-Historical-Comment') and 'body' in Array[0]:
- self.corpus = list(set(df[df['body']!='']['body'].dropna().astype('str').tolist()))
-
- # TODO: switch reddit comment to elasticsearch endpoint
- # elif source == 'reddit-Historical-Comment' and '_source.body' in Array[0]:
- # self.corpus = list(set(df[df['_source.body']!='']['_source.body'].dropna().astype('str').tolist()))
-
- # strip http in the corpus
- self.corpus = [ re.sub(r"http\S+","",text) for text in self.corpus]
-
- def split(self,ratio):
- training_set = list(random.sample(self.corpus, int(len(self.corpus)*ratio/100)))
- testing_set = [item for item in self.corpus if item not in training_set]
-
- # plot a pie chart of the split
- labels = ['training set data points','unlabeled data points']
- values = [len(training_set), len(testing_set)]
- trace = go.Pie(labels=labels, values = values, textinfo='value')
- div_split = plot([trace], output_type='div',image='png',auto_open=False, image_filename='plot_img')
- fname_div_split = 'div_split.html'
- with open(self.localSavePath + fname_div_split,"w") as f:
- f.write(div_split)
- s3.upload(self.localSavePath, self.awsPath, fname_div_split)
- div_url = s3.generate_downloads(self.awsPath, fname_div_split)
-
- fname1 = 'TRAINING_' + self.filename
- try:
- with open(self.localSavePath + fname1,'w',newline="",encoding='utf-8') as f:
- writer = csv.writer(f)
- writer.writerow(['text','category'])
- for row in training_set:
- try:
- writer.writerow([row])
- except UnicodeDecodeError:
- pass
- except:
- with open(self.localSavePath + fname1,'w',newline="",encoding='ISO-8859-1') as f:
- writer = csv.writer(f)
- writer.writerow(['text','category'])
- for row in training_set:
- try:
- writer.writerow([row])
- except UnicodeDecodeError:
- pass
- s3.upload(self.localSavePath, self.awsPath, fname1)
- training_url = s3.generate_downloads(self.awsPath, fname1)
-
-
-
- fname2 = 'UNLABELED_' + self.filename
- try:
- with open(self.localSavePath + fname2,'w',newline="",encoding='utf-8') as f:
- writer = csv.writer(f)
- writer.writerow(['text'])
- for row in testing_set:
- try:
- writer.writerow([row])
- except UnicodeDecodeError:
- pass
- except:
- with open(self.localSavePath + fname2,'w',newline="",encoding='ISO-8859-1') as f:
- writer = csv.writer(f)
- writer.writerow(['text'])
- for row in testing_set:
- try:
- writer.writerow([row])
- except UnicodeDecodeError:
- pass
- s3.upload(self.localSavePath, self.awsPath, fname2)
- unlabeled_url = s3.generate_downloads(self.awsPath, fname2)
-
-
- return {'div': div_url, 'training':training_url, 'testing': unlabeled_url}
-
-
-if __name__ == '__main__':
-
- output = dict()
-
- parser = argparse.ArgumentParser(description="processing...")
- parser.add_argument('--remoteReadPath', required=True)
- parser.add_argument('--ratio',required=True)
- parser.add_argument('--s3FolderName',required=True)
- parser.add_argument('--email',required=True)
- args = parser.parse_args()
-
- # arranging the paths
- uid = str(uuid.uuid4())
- awsPath = args.s3FolderName + '/ML/classification/' + uid +'/'
- localSavePath = '/tmp/' + args.s3FolderName + '/ML/classification/' + uid + '/'
- localReadPath = '/tmp/' + args.s3FolderName + '/' + uid + '/'
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- fname = 'config.json'
- with open(localSavePath + fname,"w") as f:
- json.dump(vars(args),f)
- s3.upload(localSavePath, awsPath, fname)
- output['config'] = s3.generate_downloads(awsPath, fname)
- output['uuid'] = uid
-
- classification = Classification(awsPath, localSavePath, localReadPath, args.remoteReadPath)
- output.update(classification.split(int(args.ratio)))
-
- d.deletedir('/tmp')
- n.notification(args.email,case=3,filename=awsPath)
-
diff --git a/batch/batch_classification_train.py b/batch/batch_classification_train.py
deleted file mode 100644
index cd288dd..0000000
--- a/batch/batch_classification_train.py
+++ /dev/null
@@ -1,328 +0,0 @@
-import csv
-import os
-from sklearn.pipeline import Pipeline
-from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
-from sklearn.naive_bayes import MultinomialNB
-from sklearn.linear_model import Perceptron
-from sklearn.linear_model import SGDClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.neighbors import KNeighborsClassifier
-from sklearn.linear_model import PassiveAggressiveClassifier
-from sklearn import metrics
-from sklearn.metrics import roc_curve, auc
-from sklearn.preprocessing import label_binarize
-from sklearn.model_selection import cross_val_predict
-from sklearn.model_selection import cross_val_score
-from scipy import interp
-from itertools import cycle
-import pickle
-import numpy as np
-from plotly.offline import plot
-import plotly.graph_objs as go
-import json
-import writeToS3 as s3
-import deleteDir as d
-import notification as n
-import argparse
-
-
-
-class Classification:
-
- def __init__(self, awsPath, localSavePath, localReadPath, remoteReadPath,filename):
-
- self.localSavePath = localSavePath
- self.awsPath = awsPath
-
- # download remote socialmedia data into a temp folder
- # load it into csv
- s3.downloadToDisk(filename=filename,localpath=localReadPath, remotepath=remoteReadPath)
-
- Array = []
- try:
- with open(localReadPath + filename,'r',encoding="utf-8") as f:
- reader = csv.reader(f)
- for row in reader:
- try:
- Array.append(row)
- except Exception as e:
- pass
- except:
- with open(localReadPath + filename,'r',encoding="ISO-8859-1") as f:
- reader = csv.reader(f)
- for row in reader:
- try:
- Array.append(row)
- except Exception as e:
- pass
-
- self.data = []
- self.target = []
- for a in Array[1:]:
- if len(a) == 2:
- self.data.append(a[0])
- self.target.append(a[1])
-
-
- def classify(self, model):
-
- if model == 'NaiveBayes':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',MultinomialNB())])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.predict_proba(self.data)
- elif model == 'Perceptron':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',Perceptron())])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.decision_function(self.data)
- elif model == 'SGD':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',SGDClassifier())])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.decision_function(self.data)
- elif model == 'RandomForest':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',RandomForestClassifier(n_estimators=100))])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.predict_proba(self.data)
- elif model == 'KNN':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',KNeighborsClassifier(n_neighbors=10))])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.predict_proba(self.data)
- elif model == 'passiveAggressive':
- text_clf = Pipeline([('vect', CountVectorizer(stop_words='english')),
- ('tfidf', TfidfTransformer()),
- ('clf',PassiveAggressiveClassifier(n_iter=50))])
- # 10 fold cross validation
- self.predicted = cross_val_predict(text_clf, self.data, self.target, cv=10)
- # fit the model
- text_clf.fit(self.data, self.target)
- y_score = text_clf.decision_function(self.data)
-
- # get 10 fold cross validation accuracy score
- fold_scores = cross_val_score(text_clf, self.data, self.target, cv=10)
- fname_folds = 'accuracy_score.csv'
- with open(self.localSavePath + fname_folds,'w',newline="") as f:
- writer = csv.writer(f)
- writer.writerow(['fold_1','fold_2','fold_3','fold_4','fold_5',
- 'fold_6','fold_7','fold_8','fold_9','fold_10'])
- writer.writerow([ '%.4f' % elem for elem in fold_scores ])
- s3.upload(self.localSavePath, self.awsPath, fname_folds)
- accuracy_url = s3.generate_downloads(self.awsPath, fname_folds)
-
- # pickle the Pipeline for future use
- fname_pickle = 'classification_pipeline.pickle'
- with open(self.localSavePath + fname_pickle,'wb') as f:
- pickle.dump(text_clf,f)
- s3.upload(self.localSavePath, self.awsPath, fname_pickle)
- pickle_url = s3.generate_downloads(self.awsPath, fname_pickle)
-
- # plotting the roc curve
- self.labels = text_clf.classes_
- y = label_binarize(self.target,classes = self.labels)
-
-
- # binary class
- if len(self.labels) <= 2:
- if model == 'Perceptron' or model == 'SGD' or model == 'passiveAggressive':
- fpr, tpr, _ = roc_curve(y[:, 0], y_score)
- else:
- y = []
- for label in self.target:
- item = []
- for i in range(len(text_clf.classes_)):
- if label == text_clf.classes_[i]:
- item.append(1)
- else:
- item.append(0)
- y.append(item)
- y = np.array(y)
- fpr, tpr, _ = roc_curve(y.ravel(), y_score.ravel())
-
- roc_auc = auc(fpr, tpr)
- trace = go.Scatter(
- x = fpr,
- y = tpr,
- name = 'ROC curve (area =' + str(roc_auc) + ' )',
- line = dict(color=('deeppink'), width = 4)
- )
- data = [trace]
-
- # multiclasses
- else:
- fpr = {}
- tpr = {}
- roc_auc = {}
- for i in range(len(self.labels)):
- fpr[self.labels[i]], tpr[self.labels[i]], _ = roc_curve(y[:, i], y_score[:, i])
- roc_auc[self.labels[i]] = auc(fpr[self.labels[i]], tpr[self.labels[i]])
-
- # Compute micro-average ROC curve and ROC area
- fpr["micro"], tpr["micro"], _ = roc_curve(y.ravel(), y_score.ravel())
- roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
-
- # First aggregate all false positive rates
- all_fpr = np.unique(np.concatenate([fpr[self.labels[i]] for i in range(len(self.labels))]))
-
- # Then interpolate all ROC curves at this points
- mean_tpr = np.zeros_like(all_fpr)
- for i in range(len(self.labels)):
- mean_tpr += interp(all_fpr, fpr[self.labels[i]], tpr[self.labels[i]])
-
- # Finally average it and compute AUC
- mean_tpr /= len(self.labels)
-
- fpr["macro"] = all_fpr
- tpr["macro"] = mean_tpr
- roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])
-
- # plotting
- trace0 = go.Scatter(
- x = fpr['micro'],
- y = tpr['micro'],
- name = 'micro-average ROC curve (area =' + str(roc_auc["micro"]) + ' )',
- line = dict(color=('deeppink'), width = 4)
- )
- trace1 = go.Scatter(
- x = fpr['macro'],
- y = tpr['macro'],
- name = 'macro-average ROC curve (area =' + str(roc_auc["macro"]) + ' )',
- line = dict(
- color = ('navy'),
- width = 4,)
- )
- data = [trace0, trace1]
- colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
- for i, color in zip(range(len(self.labels)), colors):
- trace = go.Scatter(
- x = fpr[self.labels[i]],
- y = tpr[self.labels[i]],
- name = 'ROC curve of class {0} (area = {1:0.2f})'.format(self.labels[i], roc_auc[self.labels[i]]),
- line = dict(
- color = (color),
- width = 4,
- dash = 'dash')
- )
- data.append(trace)
-
-
- layout = dict(title = model + ' model ROC curve',
- xaxis = dict(title = 'False Positive Rate'),
- yaxis = dict(title = 'True Positive Rate'),
- )
-
- fig = dict(data=data, layout=layout)
- div = plot(fig, output_type='div',image='png',auto_open=False, image_filename='plot_img')
-
- # print the graph file
- fname_div ='div.html'
- with open(self.localSavePath + fname_div,'w') as f:
- f.write(div)
- s3.upload(self.localSavePath, self.awsPath, fname_div)
- div_url = s3.generate_downloads(self.awsPath, fname_div)
-
- return {'accuracy':accuracy_url, 'pickle':pickle_url, 'div':div_url }
-
-
-
- def metrics(self):
- report = np.array(metrics.precision_recall_fscore_support(self.target,self.predicted,labels=self.labels)).T
- avg_report = list(metrics.precision_recall_fscore_support(self.target,self.predicted,average='weighted'))
- avg_report.insert(0,'AVG')
-
- # save metrics report
- fname_metrics = 'classification_report.csv'
- with open(self.localSavePath + fname_metrics,'w',newline="") as f:
- writer = csv.writer(f)
- writer.writerow(['label','precision','recall','f1-score','support'])
- for i in range(len(report)):
- writer.writerow([self.labels[i],
- round(report[i][0],4),
- round(report[i][1],4),
- round(report[i][2],4),
- round(report[i][3],4)])
- writer.writerow(avg_report)
- s3.upload(self.localSavePath, self.awsPath, fname_metrics)
- return {'metrics': s3.generate_downloads(self.awsPath, fname_metrics)}
-
-
-
-
-if __name__ == '__main__':
-
- output = dict()
-
- parser = argparse.ArgumentParser(description="processing...")
- parser.add_argument('--remoteReadPath', required=True)
- parser.add_argument('--model',required=True)
- parser.add_argument('--uuid',required=True)
- parser.add_argument('--labeledFilename',required=True)
- parser.add_argument('--s3FolderName',required=True)
- parser.add_argument('--email',required=True)
- args = parser.parse_args()
-
- # arranging the paths
- uid = args.uuid
-
- # check if this awsPath exist!!! if not exist, exit with error
- awsPath = args.s3FolderName + '/ML/classification/' + uid +'/'
-
- localSavePath = '/tmp/' + args.s3FolderName + '/ML/classification/' + uid + '/'
- localReadPath = '/tmp/' + args.s3FolderName + '/'
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- fname_config = 'config.json'
- if s3.checkExist(awsPath, fname_config):
- s3.downloadToDisk(fname_config, localSavePath, awsPath)
- with open(localSavePath + fname_config, "r") as fp:
- data = json.load(fp)
- for key in vars(args).keys():
- if key not in data.keys():
- data[key] = vars(args)[key]
- with open(localSavePath + fname_config,"w") as f:
- json.dump(data,f)
- s3.upload(localSavePath, awsPath, fname_config)
- output['config'] = s3.generate_downloads(awsPath, fname_config)
- output['uuid'] = uid
-
- else:
- raise ValueError('This session ID is invalid!')
- exit()
-
-
- # download the labeled data from s3 to tmp
- classification = Classification(awsPath, localSavePath, localReadPath, args.remoteReadPath,args.labeledFilename)
-
- output.update(classification.classify(args.model))
- output.update(classification.metrics())
-
- d.deletedir('/tmp')
- n.notification(args.email,case=3,filename=awsPath)
-
-
-
diff --git a/batch/batch_network_analysis/batch_function.py b/batch/batch_network_analysis/batch_function.py
index 63ba1ea..04493dc 100644
--- a/batch/batch_network_analysis/batch_function.py
+++ b/batch/batch_network_analysis/batch_function.py
@@ -1,4 +1,4 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
@@ -26,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -46,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_network_analysis/dataset.py b/batch/batch_network_analysis/dataset.py
index c1bb7b5..23cf0a6 100644
--- a/batch/batch_network_analysis/dataset.py
+++ b/batch/batch_network_analysis/dataset.py
@@ -4,145 +4,150 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
-
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
-
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
-
- return url
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
+
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
+
+ return url
diff --git a/batch/batch_network_analysis/writeToS3.py b/batch/batch_network_analysis/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_network_analysis/writeToS3.py
+++ b/batch/batch_network_analysis/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_pipeline/batch_function.py b/batch/batch_pipeline/batch_function.py
index 4337f90..04493dc 100644
--- a/batch/batch_pipeline/batch_function.py
+++ b/batch/batch_pipeline/batch_function.py
@@ -1,8 +1,7 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
-from algorithm import StemmedCountVectorizer
if __name__ == '__main__':
@@ -27,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -47,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_pipeline/dataset.py b/batch/batch_pipeline/dataset.py
index c1bb7b5..f9728b5 100644
--- a/batch/batch_pipeline/dataset.py
+++ b/batch/batch_pipeline/dataset.py
@@ -4,145 +4,150 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
- return url
+ return url
diff --git a/batch/batch_pipeline/writeToS3.py b/batch/batch_pipeline/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_pipeline/writeToS3.py
+++ b/batch/batch_pipeline/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_preprocessing/batch_function.py b/batch/batch_preprocessing/batch_function.py
index 63ba1ea..04493dc 100644
--- a/batch/batch_preprocessing/batch_function.py
+++ b/batch/batch_preprocessing/batch_function.py
@@ -1,4 +1,4 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
@@ -26,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -46,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_preprocessing/dataset.py b/batch/batch_preprocessing/dataset.py
index c1bb7b5..f9728b5 100644
--- a/batch/batch_preprocessing/dataset.py
+++ b/batch/batch_preprocessing/dataset.py
@@ -4,145 +4,150 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
- return url
+ return url
diff --git a/batch/batch_preprocessing/writeToS3.py b/batch/batch_preprocessing/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_preprocessing/writeToS3.py
+++ b/batch/batch_preprocessing/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_reddit_comment/RedditComment.py b/batch/batch_reddit_comment/RedditComment.py
new file mode 100644
index 0000000..8331bf0
--- /dev/null
+++ b/batch/batch_reddit_comment/RedditComment.py
@@ -0,0 +1,177 @@
+import argparse
+import csv
+import os
+import shutil
+import uuid
+import zipfile
+
+import notification as n
+import pandas as pd
+import praw
+from writeToS3 import WriteToS3
+
+
+def deletedir(dir):
+ shutil.rmtree(dir)
+
+
+def getFolderSize(folder):
+ total_size = os.path.getsize(folder)
+ for item in os.listdir(folder):
+ itempath = os.path.join(folder, item)
+ if os.path.isfile(itempath):
+ total_size += os.path.getsize(itempath)
+ elif os.path.isdir(itempath):
+ total_size += getFolderSize(itempath)
+ return total_size
+
+
+def zipdir(path, ziph):
+ # ziph is zipfile handle
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ ziph.write(os.path.join(root, file), file)
+
+
+def bfs(submission, id, directory):
+ # expand comments
+ submission.comments.replace_more(limit=None)
+ comment_queue = submission.comments[:] # Seed with top-level
+ comments_no_order = [['author', 'body', 'created_utc', 'id', 'link_id',
+ 'parent_id', 'score', 'subreddit_display_name', 'subreddit_name_prefixed', 'subreddit_id']]
+ while comment_queue:
+ comment = comment_queue.pop(0)
+ comments_no_order.append([str(comment.author),
+ comment.body, comment.created_utc, comment.id, comment.link_id,
+ comment.parent_id, comment.score, comment.subreddit.display_name,
+ comment.subreddit_name_prefixed, comment.subreddit_id])
+ comment_queue.extend(comment.replies)
+
+ # save to csv
+ with open(os.path.join(directory, id + '.csv'), 'w', newline="", encoding='utf-8') as f:
+ writer = csv.writer(f, delimiter=',')
+ for c in comments_no_order:
+ try:
+ writer.writerow(c)
+ except:
+ print('encoding error')
+
+
+if __name__ == '__main__':
+
+ parser = argparse.ArgumentParser(description="Processing...")
+ parser.add_argument('--email', required=True)
+ parser.add_argument('--remoteReadPath', required=True)
+ parser.add_argument('--s3FolderName', required=True)
+ parser.add_argument('--sessionURL', required=True)
+
+ # user specified parameters
+ parsed, unknown = parser.parse_known_args()
+ for arg in unknown:
+ if arg.startswith("--"):
+ parser.add_argument(arg, required=False)
+
+ params = vars(parser.parse_args())
+
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ uid = str(uuid.uuid4())
+ temp_dir = os.path.join('/tmp', params['s3FolderName'], uid)
+ if not os.path.exists(temp_dir):
+ os.makedirs(temp_dir)
+
+ # configure output directory
+ # save it in download/temp/xxx-xxxxxxxxxxxxx-xxxxx/aww-comments
+
+ file = params['remoteReadPath'].split('/')[-2]
+ fname_zip = file + '.zip'
+ comments_folder = os.path.join(temp_dir, file + '-comments')
+ if not os.path.exists(comments_folder):
+ os.makedirs(comments_folder)
+
+ s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ s3.downloadToDisk(filename=file + '.csv', localpath=temp_dir, remotepath=params['remoteReadPath'])
+ Array = []
+ try:
+ with open(os.path.join(temp_dir, file + '.csv'), 'r', encoding='utf-8') as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ pass
+
+ except:
+ with open(os.path.join(temp_dir, file + '.csv'), 'r', encoding="ISO-8859-1") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ pass
+
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+ headers = df.columns.values.tolist()
+ if 'permalink' in headers and 'id' in headers:
+ urls = df['permalink'].dropna().astype('str').tolist()
+ ids = df['id'].dropna().astype('str').tolist()
+ elif '_source.permalink' in headers and '_source.id' in headers:
+ urls = df['_source.permalink'].dropna().astype('str').tolist()
+ ids = df['_source.id'].dropna().astype('str').tolist()
+ else:
+ deletedir('/tmp')
+ n.notification(params['email'], case=0, filename='', links='', sessionURL=params['sessionURL'])
+ raise ValueError('The Reddit post collection provided does not have permalink and/or id in it.')
+
+ # praw construct submission
+ # here need to subtitute reddit client id and reddit client secret in the container with real token
+ reddit = praw.Reddit(user_agent='Comment Extraction (by /u/USERNAME)',
+ client_id=os.environ['REDDIT_CLIENT_ID'], client_secret=os.environ['REDDIT_CLIENT_SECRET'])
+
+ # loop through the id and store their comments
+ for url, id in zip(urls, ids):
+ url = "https://www.reddit.com" + url
+ try:
+ submission = reddit.submission(url=url)
+ bfs(submission, id, comments_folder)
+ except:
+ print("The url " + url + " is not valid.")
+
+ # success and send email notification
+ # zip goes here
+ zipf = zipfile.ZipFile(os.path.join(temp_dir, fname_zip), 'w')
+ zipdir(comments_folder, zipf)
+ zipf.close()
+
+ # upload this zip to the s3 corresponding folder
+ s3.upload(temp_dir, params['remoteReadPath'], fname_zip)
+ url = s3.generate_downloads(params['remoteReadPath'], fname_zip)
+
+ # delete the files
+ deletedir('/tmp')
+
+ # send out email notification
+ n.notification(params['email'], case=2, filename=params['remoteReadPath'], links=url, sessionURL=params['sessionURL'])
diff --git a/batch/batch_reddit_comment/dockerfile b/batch/batch_reddit_comment/dockerfile
new file mode 100644
index 0000000..1417079
--- /dev/null
+++ b/batch/batch_reddit_comment/dockerfile
@@ -0,0 +1,16 @@
+FROM ubuntu:18.04
+
+RUN apt-get -qq -y update && apt-get -qq -y install cron
+
+RUN mkdir -p /scripts
+WORKDIR /scripts
+
+# copy paste python scripts
+COPY . ./
+
+# install dependency libraries
+RUN apt-get update
+RUN apt-get -y install python3-pip
+
+# install dependency libraries and download required data
+RUN pip3 install -r requirement.txt
\ No newline at end of file
diff --git a/image_crawler/notification.py b/batch/batch_reddit_comment/notification.py
similarity index 100%
rename from image_crawler/notification.py
rename to batch/batch_reddit_comment/notification.py
diff --git a/batch/batch_reddit_comment/requirement.txt b/batch/batch_reddit_comment/requirement.txt
new file mode 100644
index 0000000..4ec9e6e
--- /dev/null
+++ b/batch/batch_reddit_comment/requirement.txt
@@ -0,0 +1,4 @@
+requests==2.21.0
+pandas==0.24.1
+boto3==1.6.11
+praw>=6.5.1
\ No newline at end of file
diff --git a/batch/batch_reddit_comment/writeToS3.py b/batch/batch_reddit_comment/writeToS3.py
new file mode 100644
index 0000000..51dbf78
--- /dev/null
+++ b/batch/batch_reddit_comment/writeToS3.py
@@ -0,0 +1,83 @@
+import mimetypes
+import os
+
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_sentiment_analysis/batch_function.py b/batch/batch_sentiment_analysis/batch_function.py
index 63ba1ea..04493dc 100644
--- a/batch/batch_sentiment_analysis/batch_function.py
+++ b/batch/batch_sentiment_analysis/batch_function.py
@@ -1,4 +1,4 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
@@ -26,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -46,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_sentiment_analysis/dataset.py b/batch/batch_sentiment_analysis/dataset.py
index c1bb7b5..f9728b5 100644
--- a/batch/batch_sentiment_analysis/dataset.py
+++ b/batch/batch_sentiment_analysis/dataset.py
@@ -4,145 +4,150 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
- return url
+ return url
diff --git a/batch/batch_sentiment_analysis/dockerfile b/batch/batch_sentiment_analysis/dockerfile
index ab50d03..3dd6487 100644
--- a/batch/batch_sentiment_analysis/dockerfile
+++ b/batch/batch_sentiment_analysis/dockerfile
@@ -12,4 +12,5 @@ RUN apt-get -y install python3-pip
# install dependency libraries and download required data
RUN pip3 install -r requirement.txt
-RUN python3 -m nltk.downloader -d /usr/local/share/nltk_data punkt vader_lexicon sentiwordnet
\ No newline at end of file
+RUN python3 -m nltk.downloader -d /usr/local/share/nltk_data stopwords wordnet punkt averaged_perceptron_tagger \
+vader_lexicon sentiwordnet
\ No newline at end of file
diff --git a/batch/batch_sentiment_analysis/writeToS3.py b/batch/batch_sentiment_analysis/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_sentiment_analysis/writeToS3.py
+++ b/batch/batch_sentiment_analysis/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_topic_modeling/batch_function.py b/batch/batch_topic_modeling/batch_function.py
index 63ba1ea..04493dc 100644
--- a/batch/batch_topic_modeling/batch_function.py
+++ b/batch/batch_topic_modeling/batch_function.py
@@ -1,4 +1,4 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
@@ -26,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -46,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_topic_modeling/dataset.py b/batch/batch_topic_modeling/dataset.py
index c1bb7b5..f9728b5 100644
--- a/batch/batch_topic_modeling/dataset.py
+++ b/batch/batch_topic_modeling/dataset.py
@@ -4,145 +4,150 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
- return url
+ return url
diff --git a/batch/batch_topic_modeling/writeToS3.py b/batch/batch_topic_modeling/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_topic_modeling/writeToS3.py
+++ b/batch/batch_topic_modeling/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/batch_twitter_ner/batch_function.py b/batch/batch_twitter_ner/batch_function.py
index 63ba1ea..04493dc 100644
--- a/batch/batch_twitter_ner/batch_function.py
+++ b/batch/batch_twitter_ner/batch_function.py
@@ -1,4 +1,4 @@
-import dataset
+from dataset import Dataset
import argparse
from notification import notification
from algorithm import algorithm
@@ -26,17 +26,43 @@
params = vars(parser.parse_args())
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
# arranging the paths
- path = dataset.organize_path_lambda(params)
+ path = d.organize_path_lambda(params)
# save the config file
- urls['config'] = dataset.save_remote_output(path['localSavePath'],
+ urls['config'] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
'config',
params)
# prepare input dataset
- df = dataset.get_remote_input(path['remoteReadPath'],
+ df = d.get_remote_input(path['remoteReadPath'],
path['filename'],
path['localReadPath'])
@@ -46,7 +72,7 @@
# upload object to s3 bucket and return the url
for key, value in output.items():
if key != 'uid':
- urls[key] = dataset.save_remote_output(path['localSavePath'],
+ urls[key] = d.save_remote_output(path['localSavePath'],
path['remoteSavePath'],
key,
value)
diff --git a/batch/batch_twitter_ner/dataset.py b/batch/batch_twitter_ner/dataset.py
index c1bb7b5..5476234 100644
--- a/batch/batch_twitter_ner/dataset.py
+++ b/batch/batch_twitter_ner/dataset.py
@@ -4,145 +4,149 @@
import pickle
import pandas as pd
import types
-import writeToS3 as s3
-
-
-def organize_path_lambda(event):
- """
- parse the lambda handler parameter event to construct necessary paths for reading and storing data
- :param event: aws lambda parameters from handler
- :return: path dictionary
- """
- # arranging the paths
- localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
- localSavePath = os.path.join('/tmp',
- event['s3FolderName'] + event['resultPath'],
- event['uid'])
- remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
- event['uid'])
- if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
- remoteReadPath = remoteSavePath
- filename = event['labeledFilename']
- else:
- remoteReadPath = event['remoteReadPath']
- filename = remoteReadPath.split('/')[-2] + '.csv'
-
- if not os.path.exists(localSavePath):
- os.makedirs(localSavePath)
- if not os.path.exists(localReadPath):
- os.makedirs(localReadPath)
-
- path = {
- 'remoteReadPath': remoteReadPath,
- 'localReadPath': localReadPath,
- 'localSavePath': localSavePath,
- 'remoteSavePath': remoteSavePath,
- 'filename': filename
- }
-
- return path
-
-
-def get_remote_input(remoteReadPath, filename, localReadPath):
- """
- download input file from s3 bucket to a local location, and then load
- it to a pandas dataframe
- :param remoteReadPath: remote path in s3 to store the data
- :param localReadPath: local location to store the data, usually in /tmp
- :return: df: dataframe that contains the complete input file
- """
- s3.downloadToDisk(filename, localReadPath, remoteReadPath)
-
- # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
- # Array = 2D nested list holding column and row data
- Array = []
- try:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='utf-8', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
- except Exception:
- with open(os.path.join(localReadPath, filename), 'r',
- encoding='ISO-8859-1', errors="ignore") as f:
- reader = csv.reader(f)
- try:
- for row in reader:
- Array.append(row)
- except Exception as e:
- print(e)
-
- # load to pandas dataframe
- df = pd.DataFrame(Array[1:], columns=Array[0])
-
- return df
-
-
-def save_remote_output(localSavePath, remoteSavePath, fname, output_data):
- """
- save output in memory first to local file, then upload to remote S3 bucket
- :param localSavePath: local saved file
- :param remoteSavePath: remote save file path
- :param fname: filename
- :param output_data: the actual data
- :return: url of the file saved in S3 bucket
- """
-
- # json
- if isinstance(output_data, dict):
- fname += '.json'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- json.dump(output_data, f)
-
- # dataframe to csv
- elif isinstance(output_data, pd.DataFrame):
- fname += '.csv'
- output_data.to_csv(fname, encoding='utf-8')
-
- # string to html
- elif isinstance(output_data, str):
- fname += '.html'
- with open(os.path.join(localSavePath, fname), 'w') as f:
- f.write(output_data)
-
- # list(list) to csv
- elif isinstance(output_data, list) \
- and (isinstance(output_data[0], list) or isinstance(output_data[0],
- tuple)):
- fname += '.csv'
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- writer = csv.writer(f)
- for row in output_data:
+from writeToS3 import WriteToS3
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ localReadPath = os.path.join('/tmp', event['s3FolderName'], event['uid'])
+ localSavePath = os.path.join('/tmp',
+ event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ remoteSavePath = os.path.join(event['s3FolderName'] + event['resultPath'],
+ event['uid'])
+ if 'remoteReadPath' not in event.keys() or event['remoteReadPath'] == 'undefined':
+ remoteReadPath = remoteSavePath
+ filename = event['labeledFilename']
+ else:
+ remoteReadPath = event['remoteReadPath']
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8', errors="ignore") as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1', errors="ignore") as f:
+ reader = csv.reader(f)
try:
- writer.writerow(row)
- except UnicodeEncodeError as e:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
print(e)
- # special case
- elif isinstance(output_data, types.GeneratorType):
- if fname == 'gephi':
- fname += '.gml'
- elif fname == 'pajek':
- fname += '.net'
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname, output_data):
+ """
+ save output in memory first to local file, then upload to remote S3 bucket
+ :param localSavePath: local saved file
+ :param remoteSavePath: remote save file path
+ :param fname: filename
+ :param output_data: the actual data
+ :return: url of the file saved in S3 bucket
+ """
+
+ # json
+ if isinstance(output_data, dict):
+ fname += '.json'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ json.dump(output_data, f)
+
+ # dataframe to csv
+ elif isinstance(output_data, pd.DataFrame):
+ fname += '.csv'
+ output_data.to_csv(fname, encoding='utf-8')
+
+ # string to html
+ elif isinstance(output_data, str):
+ fname += '.html'
+ with open(os.path.join(localSavePath, fname), 'w') as f:
+ f.write(output_data)
+
+ # list(list) to csv
+ elif isinstance(output_data, list) \
+ and (isinstance(output_data[0], list) or isinstance(output_data[0],
+ tuple)):
+ fname += '.csv'
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ writer = csv.writer(f)
+ for row in output_data:
+ try:
+ writer.writerow(row)
+ except UnicodeEncodeError as e:
+ print(e)
+
+ # special case
+ elif isinstance(output_data, types.GeneratorType):
+ if fname == 'gephi':
+ fname += '.gml'
+ elif fname == 'pajek':
+ fname += '.net'
+ else:
+ fname += '.unknown'
+
+ with open(os.path.join(localSavePath, fname), 'w', newline='',
+ encoding='utf-8') as f:
+ for line in output_data:
+ f.write(line + '\n')
+
+ # else pickle the object
else:
- fname += '.unknown'
-
- with open(os.path.join(localSavePath, fname), 'w', newline='',
- encoding='utf-8') as f:
- for line in output_data:
- f.write(line + '\n')
-
- # else pickle the object
- else:
- fname += '.pickle'
- with open(os.path.join(localSavePath, fname), 'wb') as f:
- pickle.dump(output_data, f)
+ fname += '.pickle'
+ with open(os.path.join(localSavePath, fname), 'wb') as f:
+ pickle.dump(output_data, f)
- s3.upload(localSavePath, remoteSavePath, fname)
- url = s3.generate_downloads(remoteSavePath, fname)
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
- return url
+ return url
diff --git a/batch/batch_twitter_ner/models.py b/batch/batch_twitter_ner/models.py
index 412b3ee..3a60d40 100644
--- a/batch/batch_twitter_ner/models.py
+++ b/batch/batch_twitter_ner/models.py
@@ -1,45 +1,36 @@
from __future__ import (absolute_import, division,
print_function, unicode_literals)
+
from builtins import *
-from io import open
import matplotlib
matplotlib.use("Agg")
import numpy as np
-import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
+
sns.set_context("poster")
sns.set_style("ticks")
-from itertools import chain
-
-from sklearn.metrics import make_scorer
-
import sklearn_crfsuite
-from sklearn_crfsuite import scorers
from sklearn_crfsuite import metrics
-import regex as re
-from collections import namedtuple, defaultdict, Counter, OrderedDict
-from IPython.display import display
-from joblib import load, dump, Parallel, delayed
+from collections import Counter
-import os, string, sys
import time, datetime
-import subprocess
from utils import *
+
class CRFModel(object):
- def __init__(self, labels= None, skip_label="O", **kwargs):
+ def __init__(self, labels=None, skip_label="O", **kwargs):
self.labels = labels
self.skip_label = skip_label
self.model = crf = sklearn_crfsuite.CRF(**kwargs)
-
+
def fit(self, X_train, y_train):
start = time.time()
self.model.fit(X_train, y_train)
@@ -47,82 +38,78 @@ def fit(self, X_train, y_train):
process_time = end - start
print("Model fitting took: %s" % datetime.timedelta(seconds=process_time))
self._gen_labels()
-
+
def _gen_labels(self):
self.labels = list(self.model.classes_)
self.labels.remove(self.skip_label)
# group B and I results
self.labels = sorted(
- self.labels,
+ self.labels,
key=lambda name: (name[1:], name[0])
)
self.boundary_labels = set([k[0] for k in self.labels]) - set([self.skip_label])
self.category_labels = set([k[2:] for k in self.labels if k != self.skip_label])
- self.possible_labels = ["%s-%s" % (k1,k) for k in self.category_labels
- for k1 in self.boundary_labels] + [self.skip_label]
+ self.possible_labels = ["%s-%s" % (k1, k) for k in self.category_labels
+ for k1 in self.boundary_labels] + [self.skip_label]
-
def predict(self, X_test):
y_pred = self.model.predict(X_test)
return y_pred
-
+
def evaluate(self, y_true, y_pred):
- print("Training accuracy: ", metrics.flat_f1_score(y_true, y_pred,
- average='weighted', labels=self.labels))
+ print("Training accuracy: ", metrics.flat_f1_score(y_true, y_pred,
+ average='weighted', labels=self.labels))
report = metrics.flat_classification_report(
y_true, y_pred, labels=self.labels, digits=3
)
return classification_report_to_df(report)
-
-
+
def print_transitions(self, top_n=25):
trans_features = Counter(self.model.transition_features_).most_common(top_n)
print("Top %s likely transitions:" % top_n)
for (label_from, label_to), weight in trans_features:
print("%-6s -> %-7s %0.6f" % (label_from, label_to, weight))
-
+
def print_state_features(self, top_n=10):
state_features = Counter(self.model.state_features_).most_common(top_n)
for (attr, label), weight in state_features:
- print("%0.6f %-8s %s" % (weight, label, attr))
-
+ print("%0.6f %-8s %s" % (weight, label, attr))
+
def show_transition_matrix(self):
transition_matrix = np.zeros((len(self.possible_labels), len(self.possible_labels)))
for k in self.possible_labels:
for k1 in self.possible_labels:
- if (k,k1) in self.model.transition_features_:
- i,j = self.possible_labels.index(k), self.possible_labels.index(k1)
- transition_matrix[i,j] = self.model.transition_features_[(k,k1)]
+ if (k, k1) in self.model.transition_features_:
+ i, j = self.possible_labels.index(k), self.possible_labels.index(k1)
+ transition_matrix[i, j] = self.model.transition_features_[(k, k1)]
with plt.rc_context(rc={'xtick.labelsize': 10, 'ytick.labelsize': 10,
'font.size': 10,
- 'figure.figsize': (10,8)}):
+ 'figure.figsize': (10, 8)}):
ax = sns.heatmap(data=transition_matrix,
- xticklabels=self.possible_labels, yticklabels=self.possible_labels,
- cmap="RdGy")
+ xticklabels=self.possible_labels, yticklabels=self.possible_labels,
+ cmap="RdGy")
return ax
def plot_cm(self, y_test, y_pred, axis=1):
labels = self.labels + [self.skip_label]
- labels_s = dict((k,i) for i,k in enumerate(labels))
+ labels_s = dict((k, i) for i, k in enumerate(labels))
cm = np.zeros((len(labels), len(labels)))
- for i,j in zip(sum(y_test, []), sum(y_pred, [])):
+ for i, j in zip(sum(y_test, []), sum(y_pred, [])):
i = labels_s[i]
j = labels_s[j]
- cm[i,j] += 1
+ cm[i, j] += 1
with plt.rc_context(rc={'xtick.labelsize': 12, 'ytick.labelsize': 12,
- 'figure.figsize': (16,14)}):
- ax = sns.heatmap(cm * 100/ cm.sum(axis=axis, keepdims=True),
- #cmap=sns.cubehelix_palette(n_colors=100, rot=-.4, as_cmap=True),
- cmap="Greys",
- xticklabels=labels,
- yticklabels=labels)
+ 'figure.figsize': (16, 14)}):
+ ax = sns.heatmap(cm * 100 / cm.sum(axis=axis, keepdims=True),
+ # cmap=sns.cubehelix_palette(n_colors=100, rot=-.4, as_cmap=True),
+ cmap="Greys",
+ xticklabels=labels,
+ yticklabels=labels)
plt.ylabel("True labels")
plt.xlabel("Predicted labels")
title = "Precision Plot"
- if axis== 0:
+ if axis == 0:
title = "Recall Plot"
plt.title(title)
return ax, cm
-
-
diff --git a/batch/batch_twitter_ner/writeToS3.py b/batch/batch_twitter_ner/writeToS3.py
index 75b1999..51dbf78 100644
--- a/batch/batch_twitter_ner/writeToS3.py
+++ b/batch/batch_twitter_ner/writeToS3.py
@@ -1,72 +1,83 @@
-import boto3
-import os
import mimetypes
+import os
-client = boto3.client('s3')
-bucket_name = 'macroscope-smile'
-
-def upload(localpath, remotepath, filename):
- content_type = mimetypes.guess_type(os.path.join(localpath,filename))[0]
- print(filename, content_type)
- if content_type == None:
- extra_args = {'ContentType':'application/octet-stream'}
- else:
- extra_args = {'ContentType':content_type}
-
- client.upload_file(os.path.join(localpath, filename),
- bucket_name,
- os.path.join(remotepath, filename),
- ExtraArgs=extra_args)
-
-
-def createDirectory(DirectoryName):
- client.put_object(Bucket=bucket_name, Key=DirectoryName)
-
-
-def generate_downloads(remotepath, filename):
- url = client.generate_presigned_url(
- ClientMethod='get_object',
- Params={
- 'Bucket': bucket_name,
- 'Key': os.path.join(remotepath, filename)
- },
- ExpiresIn=604800 # one week
- )
-
- return url
-
-
-def downloadToDisk(filename, localpath, remotepath):
- with open(os.path.join(localpath, filename), 'wb') as f:
- client.download_fileobj(bucket_name,
- os.path.join(remotepath, filename), f)
-
-
-def getObject(remoteKey):
- obj = client.get_object(Bucket=bucket_name, Key=remoteKey)
-
-
-def putObject(body, remoteKey):
- # bytes or seekable file-like object
- obj = client.put_object(Bucket=bucket_name,
- Body=body, Key=remoteKey)
- print(obj['Body'].read())
-
-def listDir(remoteClass):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=remoteClass,
- Delimiter='/')
- foldernames = []
- for o in objects.get('CommonPrefixes'):
- foldernames.append(o.get('Prefix'))
-
- # only return the list of foldernames
- return foldernames
-
-
-def listFiles(foldernames):
- objects = client.list_objects(Bucket=bucket_name,
- Prefix=foldernames)
-
- # return rich information about the files
- return objects.get('Contents')
+import boto3
+from botocore.client import Config
+
+
+class WriteToS3:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+
+ # local minio s3
+ if HOST_IP and AWS_ACCESSKEY and AWS_ACCESSKEYSECRET and BUCKET_NAME:
+ self.client = boto3.client('s3', endpoint_url='http://' + HOST_IP + ':9000',
+ aws_access_key_id=AWS_ACCESSKEY,
+ aws_secret_access_key=AWS_ACCESSKEYSECRET,
+ config=Config(signature_version='s3v4'))
+ self.bucket_name = BUCKET_NAME
+
+ # remote aws s3
+ else:
+ self.client = boto3.client('s3')
+ self.bucket_name = 'macroscope-smile'
+
+ def upload(self, localpath, remotepath, filename):
+ content_type = mimetypes.guess_type(os.path.join(localpath, filename))[0]
+ print(filename, content_type)
+ if content_type == None:
+ extra_args = {'ContentType': 'application/octet-stream'}
+ else:
+ extra_args = {'ContentType': content_type}
+
+ self.client.upload_file(os.path.join(localpath, filename),
+ self.bucket_name,
+ os.path.join(remotepath, filename),
+ ExtraArgs=extra_args)
+
+ def createDirectory(self, DirectoryName):
+ self.client.put_object(Bucket=self.bucket_name, Key=DirectoryName)
+
+ def generate_downloads(self, remotepath, filename):
+ url = self.client.generate_presigned_url(
+ ClientMethod='get_object',
+ Params={
+ 'Bucket': self.bucket_name,
+ 'Key': os.path.join(remotepath, filename)
+ },
+ ExpiresIn=604800 # one week
+ )
+
+ return url
+
+ def downloadToDisk(self, filename, localpath, remotepath):
+ with open(os.path.join(localpath, filename), 'wb') as f:
+ self.client.download_fileobj(self.bucket_name,
+ os.path.join(remotepath, filename), f)
+
+ def getObject(self, remoteKey):
+ obj = self.client.get_object(Bucket=self.bucket_name, Key=remoteKey)
+
+ def putObject(self, body, remoteKey):
+ # bytes or seekable file-like object
+ obj = self.client.put_object(Bucket=self.bucket_name,
+ Body=body, Key=remoteKey)
+ print(obj['Body'].read())
+
+ def listDir(self, remoteClass):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=remoteClass,
+ Delimiter='/')
+ foldernames = []
+ for o in objects.get('CommonPrefixes'):
+ foldernames.append(o.get('Prefix'))
+
+ # only return the list of foldernames
+ return foldernames
+
+ def listFiles(self, foldernames):
+ objects = self.client.list_objects(Bucket=self.bucket_name,
+ Prefix=foldernames)
+
+ # return rich information about the files
+ return objects.get('Contents')
diff --git a/batch/image_crawler/batch_function.py b/batch/image_crawler/batch_function.py
new file mode 100644
index 0000000..cb52f7c
--- /dev/null
+++ b/batch/image_crawler/batch_function.py
@@ -0,0 +1,88 @@
+from dataset import Dataset
+import argparse
+from notification import notification
+from image_crawler import image_crawler as ic
+
+if __name__ == '__main__':
+
+ # entrance to invoke Batch
+ urls = {}
+
+ # default parameters
+ parser = argparse.ArgumentParser(description="processing...")
+ parser.add_argument('--remoteReadPath', required=True)
+ parser.add_argument('--email', required=True)
+ parser.add_argument('--sessionURL', required=True)
+
+ # user specified parameters
+ parsed, unknown = parser.parse_known_args()
+ for arg in unknown:
+ if arg.startswith("--"):
+ parser.add_argument(arg, required=False)
+
+
+ params = vars(parser.parse_args())
+
+ if 'HOST_IP' in params.keys():
+ HOST_IP = params['HOST_IP']
+ params.pop('HOST_IP', None)
+ else:
+ HOST_IP = None
+
+ if 'AWS_ACCESSKEY' in params.keys():
+ AWS_ACCESSKEY = params['AWS_ACCESSKEY']
+ params.pop('AWS_ACCESSKEY', None)
+ else:
+ AWS_ACCESSKEY = None
+
+ if 'AWS_ACCESSKEYSECRET' in params.keys():
+ AWS_ACCESSKEYSECRET = params['AWS_ACCESSKEYSECRET']
+ params.pop('AWS_ACCESSKEYSECRET', None)
+ else:
+ AWS_ACCESSKEYSECRET = None
+
+ if 'BUCKET_NAME' in params.keys():
+ BUCKET_NAME = params['BUCKET_NAME']
+ params.pop('BUCKET_NAME', None)
+ else:
+ BUCKET_NAME = None
+
+ d = Dataset(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ # arranging the paths
+ path = d.organize_path_lambda(params)
+
+ # prepare input dataset
+ df = d.get_remote_input(path['remoteReadPath'],
+ path['filename'],
+ path['localReadPath'])
+
+ img_urls = []
+ source = path['remoteReadPath'].split('/')[-3]
+ if source == "reddit-Search" or source == "reddit-Post" \
+ or source == "crimson-Hexagon" or source == "reddit-Historical-Post"\
+ and 'url' in list(df.columns):
+ img_urls = df['url'].dropna().tolist()
+
+ elif source == "twitter-Tweet" or source == "twitter-Timeline" \
+ and 'entities.media.media_url' in list(df.columns):
+ img_urls = df['entities.media.media_url'].dropna().tolist()
+
+ elif source == 'flickr-Photo' and 'size.source' in list(df.columns):
+ img_urls = df['size.source'].dropna().tolist()
+
+ else:
+ raise ValueError("This data source does not support image collection!")
+
+ urls = {}
+ for img_url in img_urls:
+ if ic.is_image(img_url):
+ filename, binary = ic.crawler(img_url)
+ d.save_local_output(path['localSavePath'], filename, binary)
+ urls['images.zip'] = d.save_remote_output(
+ path['localSavePath'], path['remoteSavePath'], 'images.zip')
+
+ # push notification email
+ notification(toaddr=params['email'], case=3,
+ filename=path['remoteSavePath'],
+ links=urls, sessionURL=params['sessionURL'])
diff --git a/image_crawler/command.txt b/batch/image_crawler/command.txt
similarity index 100%
rename from image_crawler/command.txt
rename to batch/image_crawler/command.txt
diff --git a/batch/image_crawler/dataset.py b/batch/image_crawler/dataset.py
new file mode 100644
index 0000000..5ae90ac
--- /dev/null
+++ b/batch/image_crawler/dataset.py
@@ -0,0 +1,110 @@
+import csv
+import os
+import pandas as pd
+import zipfile
+from writeToS3 import WriteToS3
+
+class Dataset:
+
+ def __init__(self, HOST_IP=None, AWS_ACCESSKEY=None, AWS_ACCESSKEYSECRET=None, BUCKET_NAME=None):
+ self.s3 = WriteToS3(HOST_IP, AWS_ACCESSKEY, AWS_ACCESSKEYSECRET, BUCKET_NAME)
+
+ def organize_path_lambda(self, event):
+ """
+ parse the lambda handler parameter event to construct necessary paths for reading and storing data
+ :param event: aws lambda parameters from handler
+ :return: path dictionary
+ """
+ # arranging the paths
+ remoteReadPath = event['remoteReadPath']
+ localReadPath = os.path.join('/tmp', remoteReadPath)
+ localSavePath = os.path.join('/tmp', remoteReadPath)
+ remoteSavePath = os.path.join(remoteReadPath)
+ filename = remoteReadPath.split('/')[-2] + '.csv'
+
+ if not os.path.exists(localSavePath):
+ os.makedirs(localSavePath)
+ os.makedirs(os.path.join(localSavePath, 'img'))
+ if not os.path.exists(localReadPath):
+ os.makedirs(localReadPath)
+
+ path = {
+ 'remoteReadPath': remoteReadPath,
+ 'localReadPath': localReadPath,
+ 'localSavePath': localSavePath,
+ 'remoteSavePath': remoteSavePath,
+ 'filename': filename
+ }
+
+ return path
+
+
+ def get_remote_input(self, remoteReadPath, filename, localReadPath):
+ """
+ download input file from s3 bucket to a local location, and then load
+ it to a pandas dataframe
+ :param remoteReadPath: remote path in s3 to store the data
+ :param localReadPath: local location to store the data, usually in /tmp
+ :return: df: dataframe that contains the complete input file
+ """
+ self.s3.downloadToDisk(filename, localReadPath, remoteReadPath)
+
+ # quick fix for decoding error, sometimes the data is coded in ISO-8859-1
+ # Array = 2D nested list holding column and row data
+ Array = []
+ try:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='utf-8') as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+ except Exception:
+ with open(os.path.join(localReadPath, filename), 'r',
+ encoding='ISO-8859-1') as f:
+ reader = csv.reader(f)
+ try:
+ for row in reader:
+ Array.append(row)
+ except Exception as e:
+ print(e)
+
+ # load to pandas dataframe
+ df = pd.DataFrame(Array[1:], columns=Array[0])
+
+ return df
+
+
+ def save_local_output(self, localSavePath, fname, output_data):
+ with open(os.path.join(localSavePath, 'img', fname), 'wb') as f:
+ f.write(output_data)
+
+
+ def zipdir(self, path, ziph):
+ # ziph is zipfile handle
+ for root, dirs, files in os.walk(path):
+ for file in files:
+ print(file)
+ ziph.write(os.path.join(root, file), file)
+
+
+ def save_remote_output(self, localSavePath, remoteSavePath, fname):
+ """
+
+ :param localSavePath:
+ :param remoteSavePath:
+ :param fname:
+ :param output_data:
+ :return:
+ """
+ zipf = zipfile.ZipFile(os.path.join(localSavePath, fname), 'w',
+ zipfile.ZIP_DEFLATED)
+ self.zipdir(os.path.join(localSavePath,'img'), zipf)
+ zipf.close()
+
+ self.s3.upload(localSavePath, remoteSavePath, fname)
+ url = self.s3.generate_downloads(remoteSavePath, fname)
+
+ return url
diff --git a/image_crawler/dockerfile b/batch/image_crawler/dockerfile
similarity index 100%
rename from image_crawler/dockerfile
rename to batch/image_crawler/dockerfile
diff --git a/image_crawler/image_crawler.py b/batch/image_crawler/image_crawler.py
similarity index 100%
rename from image_crawler/image_crawler.py
rename to batch/image_crawler/image_crawler.py
diff --git a/batch/image_crawler/notification.py b/batch/image_crawler/notification.py
new file mode 100644
index 0000000..3374856
--- /dev/null
+++ b/batch/image_crawler/notification.py
@@ -0,0 +1,167 @@
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+
+def notification(toaddr,case,filename,links,sessionURL):
+ # toaddr -- email address to send to
+ # text content to send
+ # subject
+ host = 'smtp.mail.us-east-1.awsapps.com'
+ port = '465'
+ fromaddr = 'smile@socialmediamacroscope.awsapps.com'
+
+
+ # map the fpath component to History panel names
+ # local/NLP/sentiment/xxxxxxxxxxxxxxxxxxxxxxxx/ => [local,nlp,sentiment,xxxx,space]
+ # [local, GraphQL, reddit-post, aww, space]
+ # 0 1 2 3
+ if filename != '':
+ fpath = filename.split('/')
+
+ if fpath[1] == 'GraphQL':
+ fpath[1] == 'Social Media Data'
+ elif fpath[1] == 'NLP':
+ fpath[1] = 'Nature Language Processing'
+ elif fpath[1] == 'ML':
+ fpath[1] = 'Machine Learning ML'
+ elif fpath[1] == 'NW':
+ fpath[1] = 'Network Visualization and Analysis'
+
+ if fpath[2] == 'reddit-Post':
+ fpath[2] = 'Subreddit Posts Title'
+ elif fpath[2] == 'reddit-Historical-Post':
+ fpath[2] = 'Reddit Historical Post'
+ elif fpath[2] == 'reddit-Search':
+ fpath[2] = 'Reddit Search Posts Title'
+ elif fpath[2] == 'sentiment':
+ fpath[2] = 'Sentiment Analysis'
+ elif fpath[2] == 'preprocessing':
+ fpath[2] = 'NLP Preprocessing'
+ elif fpath[2] == 'networkx':
+ fpath[2] = 'Python NetworkX'
+ elif fpath[2] == 'classification':
+ fpath[2] = 'Text Classification'
+
+
+ if case == 0 or case == 'comment-fail':
+ html = """
+
+
+
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection has been terminated.
+
We are using the id and permalink from your Reddit Submission dataset
+ to collect comments and replies. It is most likely you have provide an incomplete Reddit Submission dataset missing these two fields.
+
Please try to reproduce the Reddit Submission with id and permalink, or switch to another dataset.
+
Go to your session...
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is exceeding 400 Megabyte, and is terminated due to lack of disk space.
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. The partial comments we manage to collect and save will be compressed for you in an .zip file named """ + fpath[3] + """-comments.zip (click)
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is ready for you!
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. It will be compressed for you in an .zip file named """+ fpath[3] +"""-comments.zip
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] +"""
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your """ + fpath[2] + """ results are ready for you! (job ID: """ + fpath[3] + """)
+
+ - You can view the visualization and download the results at HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """ tab
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click view
+
+
+ - You can also click the link below to download part of the results:
+
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection has been terminated.
+
We are using the id and permalink from your Reddit Submission dataset
+ to collect comments and replies. It is most likely you have provide an incomplete Reddit Submission dataset missing these two fields.
+
Please try to reproduce the Reddit Submission with id and permalink, or switch to another dataset.
+
Go to your session...
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is exceeding 400 Megabyte, and is terminated due to lack of disk space.
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. The partial comments we manage to collect and save will be compressed for you in an .zip file named """ + fpath[3] + """-comments.zip (click)
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is ready for you!
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. It will be compressed for you in an .zip file named """+ fpath[3] +"""-comments.zip
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] +"""
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your """ + fpath[2] + """ results are ready for you! (job ID: """ + fpath[3] + """)
+
+ - You can view the visualization and download the results at HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """ tab
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click view
+
+
+ - You can also click the link below to download part of the results:
+
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection has been terminated.
+
We are using the id and permalink from your Reddit Submission dataset
+ to collect comments and replies. It is most likely you have provide an incomplete Reddit Submission dataset missing these two fields.
+
Please try to reproduce the Reddit Submission with id and permalink, or switch to another dataset.
+
Go to your session...
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is exceeding 400 Megabyte, and is terminated due to lack of disk space.
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. The partial comments we manage to collect and save will be compressed for you in an .zip file named """ + fpath[3] + """-comments.zip (click)
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is ready for you!
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. It will be compressed for you in an .zip file named """+ fpath[3] +"""-comments.zip
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] +"""
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your """ + fpath[2] + """ results are ready for you! (job ID: """ + fpath[3] + """)
+
+ - You can view the visualization and download the results at HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """ tab
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click view
+
+
+ - You can also click the link below to download part of the results:
+
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection has been terminated.
+
We are using the id and permalink from your Reddit Submission dataset
+ to collect comments and replies. It is most likely you have provide an incomplete Reddit Submission dataset missing these two fields.
+
Please try to reproduce the Reddit Submission with id and permalink, or switch to another dataset.
+
Go to your session...
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is exceeding 400 Megabyte, and is terminated due to lack of disk space.
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. The partial comments we manage to collect and save will be compressed for you in an .zip file named """ + fpath[3] + """-comments.zip (click)
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your Reddit Comment collection is ready for you!
+
+ - You have requested comments and replies for the Reddit Submission (Post):""" + fpath[3] + """. It will be compressed for you in an .zip file named """+ fpath[3] +"""-comments.zip
+ - In order to download this file, you need to first locate the original submission in the HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] +"""
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click VIEW
+ - --> in the Overview table under the downloadables column, you will find these comments in a zip file.
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+
+
Dear user (session ID: """ + fpath[0] + """),
+
Your """ + fpath[2] + """ results are ready for you! (job ID: """ + fpath[3] + """)
+
+ - You can view the visualization and download the results at HISTORY page in SMILE.
+ Go to your session...
+
+ - Go to History
+ - --> under """ + fpath[1] + """ tab
+ - --> click """ + fpath[2] + """
+ - --> then find """ + fpath[3] + """
+ - --> click view
+
+
+ - You can also click the link below to download part of the results:
+
+
+
+
+
Best Regards,
+
Social Media Macroscope - SMILE
+