From 0cd7ad8945c9effdc47971bef916927db054b5a2 Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma Date: Sat, 19 Sep 2020 00:19:47 +0530 Subject: [PATCH 1/6] test --- libra/preprocessing/data_preprocesser.py | 12 ++++++------ libra/query/feedforward_nn.py | 10 +++++----- libra/query/unused_functions.py | 6 +++--- tools/data/nlp_data/sentimentAnalysisData.csv | 6 +++--- tools/materials/Reference Manual.md | 4 ++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libra/preprocessing/data_preprocesser.py b/libra/preprocessing/data_preprocesser.py index c9347c72..b64a6e78 100644 --- a/libra/preprocessing/data_preprocesser.py +++ b/libra/preprocessing/data_preprocesser.py @@ -133,7 +133,7 @@ def structured_preprocesser(data, ca_threshold, text): accept_sparse=True)), ('embedder', FunctionTransformer( - text_embedder, + textembedder, accept_sparse=True))]), text[x])) @@ -146,14 +146,14 @@ def structured_preprocesser(data, ca_threshold, text): if too_many_values(combined[categorical_columns], ca_threshold): cat_pipeline = Pipeline([ ('imputer', SimpleImputer(strategy="constant", fill_value="")), - ('one_hot_encoder', OneHotEncoder(handle_unknown='ignore')), + ('one_hotencoder', OneHotEncoder(handle_unknown='ignore')), ('transformer', FunctionTransformer(lambda x: x.toarray(), accept_sparse=True)), ('ca', CA(n_components=-1)) ]) else: cat_pipeline = Pipeline([ ('imputer', SimpleImputer(strategy="constant", fill_value="")), - ('one_hot_encoder', OneHotEncoder(handle_unknown='ignore')) + ('one_hotencoder', OneHotEncoder(handle_unknown='ignore')) ]) full_pipeline.transformers.append( @@ -230,7 +230,7 @@ def text_preprocessing(data, text_cols): data['test'] = combined.iloc[len(data['train']):] -def text_embedder(text): +def textembedder(text): total = list() for i in text: @@ -253,7 +253,7 @@ def generate_column_labels(full_pipeline, numeric_cols, text_cols): else: try: encoded_cols = full_pipeline.named_transformers_[ - 'cat']['one_hot_encoder'].get_feature_names() + 'cat']['one_hotencoder'].get_feature_names() cols = [*list(numeric_cols), *encoded_cols, *text_cols] except Exception as error: @@ -281,7 +281,7 @@ def clustering_preprocessor(data): # pipeline for categorical columns cat_pipeline = Pipeline([ ('imputer', SimpleImputer(strategy="constant", fill_value="")), - ('one_hot_encoder', OneHotEncoder()), + ('one_hotencoder', OneHotEncoder()), ]) # combine the two pipelines diff --git a/libra/query/feedforward_nn.py b/libra/query/feedforward_nn.py index 9f9b0ba5..ce1c1aa9 100644 --- a/libra/query/feedforward_nn.py +++ b/libra/query/feedforward_nn.py @@ -11,7 +11,7 @@ from keras import Model from keras.models import Sequential, model_from_json from keras.layers import (Dense, Conv2D, Flatten, MaxPooling2D, Dropout, GlobalAveragePooling2D) -from keras.applications import VGG16, VGG19, ResNet50, ResNet101, ResNet152, MobileNet, MobileNetV2, DenseNet121, DenseNet169, DenseNet201, Xception, +from keras.applications import VGG16, VGG19, ResNet50, ResNet101, ResNet152, MobileNet, MobileNetV2, DenseNet121, DenseNet169, DenseNet201 import pandas as pd import json @@ -325,14 +325,14 @@ def classification_ann(instruction, if num_classes >= 2: # ANN needs target one hot encoded for classification - one_hot_encoder = OneHotEncoder() + one_hotencoder = OneHotEncoder() y = pd.DataFrame( - one_hot_encoder.fit_transform( + one_hotencoder.fit_transform( np.reshape( y.values, (-1, 1))).toarray(), - columns=one_hot_encoder.get_feature_names()) + columns=one_hotencoder.get_feature_names()) y_train = y.iloc[:len(X_train)] y_test = y.iloc[len(X_train):] @@ -475,7 +475,7 @@ def classification_ann(instruction, "plots": plots, "target": remove, "preprocesser": full_pipeline, - "interpreter": one_hot_encoder, + "interpreter": one_hotencoder, 'test_data': {'X': X_test, 'y': y_test}, 'losses': { 'training_loss': final_hist.history['loss'], diff --git a/libra/query/unused_functions.py b/libra/query/unused_functions.py index c0dcf843..840bf77c 100644 --- a/libra/query/unused_functions.py +++ b/libra/query/unused_functions.py @@ -357,7 +357,7 @@ def get_snet_layer(num_out=2): # # best_hps : best Hyperprameters obtained after tuning, stored as map # # history : history of the data executed from the given model # """ -# return tuner,best_model.best_estimator_.get_params()['C'], history +# return tuner,best_model.bestestimator_.get_params()['C'], history # # def tuneClass( @@ -431,7 +431,7 @@ def get_snet_layer(num_out=2): # # # best_hps : best Hyperprameters obtained after tuning, stored as array # # # history : history of the data executed from the given model # # """ -# # return tuner,best_model.best_estimator_.get_params()['C'], history +# # return tuner,best_model.bestestimator_.get_params()['C'], history # # #return models[0], hyp, history # # # # """""" @@ -767,7 +767,7 @@ def insert_layer(model,layer_name,layer_index): # learning_rate=learning_rate, # restore_from='fresh', # run_name='run1', -# print_every=10, +# printevery=10, # sample_every=200, # save_every=save_every, # checkpoint_dir=save_path diff --git a/tools/data/nlp_data/sentimentAnalysisData.csv b/tools/data/nlp_data/sentimentAnalysisData.csv index 14ce6b0b..6766f932 100644 --- a/tools/data/nlp_data/sentimentAnalysisData.csv +++ b/tools/data/nlp_data/sentimentAnalysisData.csv @@ -888,7 +888,7 @@ Label, Num, Date, Query, Username, Tweet 0,"1468032862","Mon Apr 06 23:21:35 PDT 2009","NO_QUERY","lamarmcarter","happy for Coach Stringer (HOF c/o 2009!)...now if I can only finish my term paper on her " 0,"1468032869","Mon Apr 06 23:21:35 PDT 2009","NO_QUERY","urbanrocker","I feel like shit. This is NOT the way I want to spend my birthday's eve " 0,"1468033025","Mon Apr 06 23:21:39 PDT 2009","NO_QUERY","strifethe3rd","so far i have 311 on all my sites put together, most of them were me checking out the updates i made......" -0,"1468033122","Mon Apr 06 23:21:41 PDT 2009","NO_QUERY","Ember_X","has a lot on her mind. I needs to make some moneys. And ideas. Nowhere is hiring! " +0,"1468033122","Mon Apr 06 23:21:41 PDT 2009","NO_QUERY","Emberx","has a lot on her mind. I needs to make some moneys. And ideas. Nowhere is hiring! " 0,"1468033152","Mon Apr 06 23:21:41 PDT 2009","NO_QUERY","xGabii","@emiliexclarkex miss you " 0,"1468033194","Mon Apr 06 23:21:44 PDT 2009","NO_QUERY","tygerbaby","@stephenkruiser I'm so sorry to hear about your dog " 0,"1468033322","Mon Apr 06 23:21:45 PDT 2009","NO_QUERY","nubblecakes","Too much internet. How it plagues me " @@ -7853,7 +7853,7 @@ Label, Num, Date, Query, Username, Tweet 1,"1573858603","Tue Apr 21 03:11:26 PDT 2009","NO_QUERY","FredByrne","@PhilByrne65 Oh my god I had the most drunken weekend thanks to the footy! Wasn't gonna go out sunday but after the penos got hammered " 1,"1573858615","Tue Apr 21 03:11:26 PDT 2009","NO_QUERY","bobbigmac","@robertkneschke I think you just claimed the first ever use of the #picNiche hashtag Hooray " 1,"1573858656","Tue Apr 21 03:11:27 PDT 2009","NO_QUERY","Simo_C","@cabert17 cheer up! tomorrow you'll be at home " -1,"1573858700","Tue Apr 21 03:11:27 PDT 2009","NO_QUERY","timetraveller_x","@lowleeta Hello to you too " +1,"1573858700","Tue Apr 21 03:11:27 PDT 2009","NO_QUERY","timetravellerx","@lowleeta Hello to you too " 1,"1573858710","Tue Apr 21 03:11:28 PDT 2009","NO_QUERY","freetz","still playing with my pet " 1,"1573858774","Tue Apr 21 03:11:28 PDT 2009","NO_QUERY","bakees","blah. getting ready. school all day. lets hope it gets better out before last block. whoot gym " 1,"1573858788","Tue Apr 21 03:11:28 PDT 2009","NO_QUERY","frgt10kyur","I'm off to work and then the computer lab loving life!" @@ -8472,7 +8472,7 @@ Label, Num, Date, Query, Username, Tweet 1,"1573984889","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","jonmack","just had a nice brunch now it's time for work " 1,"1573984899","Tue Apr 21 03:46:09 PDT 2009","NO_QUERY","snaglepus","@ADDN2X I have been using more Skype - except for now when I should " 1,"1573984901","Tue Apr 21 03:46:09 PDT 2009","NO_QUERY","AndrewChapman","Back onto surf4wine later to write blog post and upload new wines/photos. Bit behind with online wine life " -1,"1573984911","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","geoff_garcia","I'm a twitter virgin " +1,"1573984911","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","geoffgarcia","I'm a twitter virgin " 1,"1573984923","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","DasKreative","Learning how to communicate DKS thro twitter! feel free to ad us onto ur list! lotsa action comin up cheers,raoul/eddie" 1,"1573984925","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","mizdi","@dennis_siuu3p u cannot make me dizzy again.. never ( mahal pa rin kita kapatid, don't worry add kita sa facebook bukas)" 1,"1573984968","Tue Apr 21 03:46:10 PDT 2009","NO_QUERY","sheckie7","Let's start this day!! Love to wake up to the rain! " diff --git a/tools/materials/Reference Manual.md b/tools/materials/Reference Manual.md index f23aa4f8..1ba17cd6 100644 --- a/tools/materials/Reference Manual.md +++ b/tools/materials/Reference Manual.md @@ -1285,10 +1285,10 @@ None Executes code to tokenize & normalize text, autocorrects spelling, and lemmatizes all columns comprised of textual data -### text_embedder ### +### textembedder ### ``` python -libra.text_embedder(text) +libra.textembedder(text) ``` Embeds data in text column by summing all of the numerical vector representations of textual contents in column into suitable scalar format From 642940c89896605700745e0a1a6972196e871b7e Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma Date: Sun, 20 Sep 2020 00:14:38 +0530 Subject: [PATCH 2/6] =?UTF-8?q?Tensorflow=20js=20support=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libra/queries.py | 6 ++++-- libra/query/feedforward_nn.py | 11 +++++++++-- requirements.txt | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/libra/queries.py b/libra/queries.py index 89d814a5..8e04efce 100644 --- a/libra/queries.py +++ b/libra/queries.py @@ -758,7 +758,8 @@ def convolutional_query(self, epochs=10, height=None, width=None, - show_feature_map=False): + show_feature_map=False, + save_as_tfjs=False): ''' Calls the body of the convolutional neural network query which is located in the feedforward.py file :param instruction: The objective that you want to model (str). @@ -794,7 +795,8 @@ def convolutional_query(self, pretrained=pretrained, epochs=epochs, height=height, - width=width) + width=width, + save_as_tfjs=False) if show_feature_map: model = self.models["convolutional_NN"]["model"] diff --git a/libra/query/feedforward_nn.py b/libra/query/feedforward_nn.py index ce1c1aa9..c3c197ff 100644 --- a/libra/query/feedforward_nn.py +++ b/libra/query/feedforward_nn.py @@ -1,6 +1,7 @@ from colorama import Fore, Style from tensorflow.keras.callbacks import EarlyStopping import os +import tensorflowjs as tfjs from libra.preprocessing.image_preprocesser import (setwise_preprocessing, csv_preprocessing, classwise_preprocessing, @@ -504,7 +505,8 @@ def convolutional(instruction=None, pretrained=None, epochs=10, height=None, - width=None): + width=None, + save_as_tfjs=False): ''' Body of the convolutional function used that is called in the neural network query if the data is presented in images. @@ -790,8 +792,13 @@ def convolutional(instruction=None, # storing values the model dictionary logger("Stored model under 'convolutional_NN' key") - clearLog() + + if save_as_tfjs: + tfjs.converters.save_keras_model(model, "tfjsmodel") + logger("Saved tfjs model under 'tfjsmodel' directory") + clearLog() + K.clear_session() return { diff --git a/requirements.txt b/requirements.txt index 1ca6f65f..9a0a941a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ tabulate textblob seaborn keras-tuner +tensorflowjs jellyfish spacy autocorrect From 29e748a95df26768abc5a8974d9e27243983d73c Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma Date: Sun, 20 Sep 2020 00:35:34 +0530 Subject: [PATCH 3/6] =?UTF-8?q?Added=20Tensorflow=20js=20and=20lite=20supp?= =?UTF-8?q?ort=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libra/queries.py | 6 ++++-- libra/query/feedforward_nn.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/libra/queries.py b/libra/queries.py index 8e04efce..53b60424 100644 --- a/libra/queries.py +++ b/libra/queries.py @@ -759,7 +759,8 @@ def convolutional_query(self, height=None, width=None, show_feature_map=False, - save_as_tfjs=False): + save_as_tfjs=False, + save_as_tflite=False): ''' Calls the body of the convolutional neural network query which is located in the feedforward.py file :param instruction: The objective that you want to model (str). @@ -796,7 +797,8 @@ def convolutional_query(self, epochs=epochs, height=height, width=width, - save_as_tfjs=False) + save_as_tfjs=False, + save_as_tflite=False) if show_feature_map: model = self.models["convolutional_NN"]["model"] diff --git a/libra/query/feedforward_nn.py b/libra/query/feedforward_nn.py index c3c197ff..f4cb8738 100644 --- a/libra/query/feedforward_nn.py +++ b/libra/query/feedforward_nn.py @@ -1,6 +1,7 @@ from colorama import Fore, Style from tensorflow.keras.callbacks import EarlyStopping import os +import tensorflow as tf import tensorflowjs as tfjs from libra.preprocessing.image_preprocesser import (setwise_preprocessing, csv_preprocessing, @@ -506,7 +507,8 @@ def convolutional(instruction=None, epochs=10, height=None, width=None, - save_as_tfjs=False): + save_as_tfjs=False, + save_as_tflite=False): ''' Body of the convolutional function used that is called in the neural network query if the data is presented in images. @@ -797,8 +799,14 @@ def convolutional(instruction=None, tfjs.converters.save_keras_model(model, "tfjsmodel") logger("Saved tfjs model under 'tfjsmodel' directory") + if save_as_tflite: + converter = tf.lite.TFLiteConverter.from_keras_model(model) + tflite_model = converter.convert() + open ("model.tflite" , "wb") .write(tflite_model) + logger("Saved tflite model as 'model.tflite' ") + clearLog() - + K.clear_session() return { From 2351df328002f181b9588ccc18d41689d487414e Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma <44338419+sukkritsharmaofficial@users.noreply.github.com> Date: Sun, 20 Sep 2020 01:35:03 +0530 Subject: [PATCH 4/6] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9a0a941a..60451625 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ colorama transformers==3.0.2 -tensorflow==2.2.0 +tensorflow keras==2.4.3 numpy pandas From f56b484d24011c75e1902ea75fdea6a9b5eb1dc4 Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma <44338419+sukkritsharmaofficial@users.noreply.github.com> Date: Mon, 21 Sep 2020 11:17:09 +0530 Subject: [PATCH 5/6] =?UTF-8?q?Tensorflow=20js=20and=20lite=20queries=20fi?= =?UTF-8?q?x=20=F0=9F=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libra/queries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libra/queries.py b/libra/queries.py index 53b60424..ab88e062 100644 --- a/libra/queries.py +++ b/libra/queries.py @@ -759,8 +759,8 @@ def convolutional_query(self, height=None, width=None, show_feature_map=False, - save_as_tfjs=False, - save_as_tflite=False): + save_as_tfjs=None, + save_as_tflite=None): ''' Calls the body of the convolutional neural network query which is located in the feedforward.py file :param instruction: The objective that you want to model (str). @@ -797,8 +797,8 @@ def convolutional_query(self, epochs=epochs, height=height, width=width, - save_as_tfjs=False, - save_as_tflite=False) + save_as_tfjs=save_as_tfjs, + save_as_tflite=save_as_tflite) if show_feature_map: model = self.models["convolutional_NN"]["model"] From 7ac0bc95666f1836b7466e03c9a00756757e8495 Mon Sep 17 00:00:00 2001 From: Sukkrit Sharma <44338419+sukkritsharmaofficial@users.noreply.github.com> Date: Mon, 21 Sep 2020 11:18:28 +0530 Subject: [PATCH 6/6] =?UTF-8?q?Tensorflow=20js=20and=20lite=20fix=20?= =?UTF-8?q?=F0=9F=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libra/query/feedforward_nn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libra/query/feedforward_nn.py b/libra/query/feedforward_nn.py index f4cb8738..fd093bea 100644 --- a/libra/query/feedforward_nn.py +++ b/libra/query/feedforward_nn.py @@ -507,8 +507,8 @@ def convolutional(instruction=None, epochs=10, height=None, width=None, - save_as_tfjs=False, - save_as_tflite=False): + save_as_tfjs=None, + save_as_tflite=None): ''' Body of the convolutional function used that is called in the neural network query if the data is presented in images.