Tdey/intel xpu

Pilot1/Attn/attn_abstention_keras2.py

@@ -21,6 +21,7 @@
 import attn_viz_utils as attnviz
 
 from attn_baseline_keras2 import build_attention_model
+from keras_utils import PerformanceReportCallback
 
 np.set_printoptions(precision=4)
 tf.compat.v1.disable_eager_execution()
@@ -292,6 +293,8 @@ def run(params):
 
     epochs = params['epochs']
     batch_size = params['batch_size']
+    perf_callback = PerformanceReportCallback(batch_size)
+    callbacks.append(perf_callback)
     history = model.fit(X_train, Y_train, class_weight=d_class_weights,
                         batch_size=batch_size,
                         epochs=epochs,

Pilot1/Attn/attn_baseline_keras2.py

@@ -23,6 +23,7 @@
 import candle
 
 import attn_viz_utils as attnviz
+from keras_utils import PerformanceReportCallback
 
 np.set_printoptions(precision=4)
 tf.compat.v1.disable_eager_execution()
@@ -247,6 +248,8 @@ def run(params):
 
     epochs = params['epochs']
     batch_size = params['batch_size']
+    perf_callback = PerformanceReportCallback(batch_size)
+    callbacks.append(perf_callback)
     history = model.fit(X_train, Y_train, class_weight=d_class_weights,
                         batch_size=batch_size,
                         epochs=epochs,

Pilot1/Attn/attn_bin_working_jan7_h5.py

@@ -26,11 +26,12 @@
 from sklearn.utils.class_weight import compute_class_weight
 from sklearn.metrics import roc_auc_score, auc, roc_curve, f1_score, precision_recall_curve
 
-
 file_path = os.path.dirname(os.path.realpath(__file__))
 lib_path = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
 sys.path.append(lib_path)
 
+import candle
+from keras_utils import PerformanceReportCallback
 psr = argparse.ArgumentParser(description='input agg csv file')
 psr.add_argument('--in', default='in_file')
 psr.add_argument('--ep', type=int, default=400)
@@ -220,13 +221,14 @@ def load_data():
 reduce_lr = ReduceLROnPlateau(monitor='val_tf_auc', factor=0.20, patience=40, verbose=1, mode='auto', min_delta=0.0001, cooldown=3, min_lr=0.000000001)
 early_stop = EarlyStopping(monitor='val_tf_auc', patience=200, verbose=1, mode='auto')
 
+perf_callback = PerformanceReportCallback(BATCH)
 # history = parallel_model.fit(X_train, Y_train,
 history = model.fit(X_train, Y_train, class_weight=d_class_weights,
                     batch_size=BATCH,
                     epochs=EPOCH,
                     verbose=1,
                     validation_data=(X_val, Y_val),
-                    callbacks=[checkpointer, csv_logger, reduce_lr, early_stop])
+                    callbacks=[checkpointer, csv_logger, reduce_lr, early_stop, perf_callback])
 
 score = model.evaluate(X_test, Y_test, verbose=0)
 

Pilot1/Combo/combo_baseline_keras2.py

@@ -24,12 +24,18 @@
 from sklearn.model_selection import StratifiedKFold, GroupKFold
 from scipy.stats.stats import pearsonr
 
+import tensorflow as tf
+options = tf.profiler.experimental.ProfilerOptions(host_tracer_level = 3, python_tracer_level = 1, device_tracer_level = 1)
+tf.compat.v1.disable_eager_execution()
+
 import matplotlib as mpl
 mpl.use('Agg')
 
 import NCI60
 import combo
 import candle
+from keras_utils import PerformanceReportCallback
+
 
 logger = logging.getLogger(__name__)
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
@@ -702,6 +708,8 @@ def warmup_scheduler(epoch):
 
         # callbacks = [history_logger, model_recorder]
         callbacks = [candle_monitor, timeout_monitor, history_logger, model_recorder]
+        perf_callback = PerformanceReportCallback(args.batch_size)
+        callbacks.append(perf_callback)
         if args.reduce_lr:
             callbacks.append(reduce_lr)
         if args.warmup_lr:

Pilot1/NT3/nt3_baseline_keras2.py

@@ -15,7 +15,7 @@
 
 import nt3 as bmk
 import candle
-
+from keras_utils import PerformanceReportCallback
 
 def initialize_parameters(default_model='nt3_default_model.txt'):
 
@@ -193,15 +193,15 @@ def run(gParameters):
     reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)
     candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters)
     timeoutMonitor = candle.TerminateOnTimeOut(gParameters['timeout'])
-
+    perf_callback = PerformanceReportCallback(gParameters['batch_size'])
     history = model.fit(X_train, Y_train,
                         batch_size=gParameters['batch_size'],
                         epochs=gParameters['epochs'],
                         initial_epoch=initial_epoch,
                         verbose=1,
                         validation_data=(X_test, Y_test),
                         callbacks=[csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor,
-                                   ckpt])
+                                   ckpt, perf_callback])
 
     score = model.evaluate(X_test, Y_test, verbose=0)
 

Pilot1/P1B1/p1b1_baseline_keras2.py

@@ -24,7 +24,7 @@
 
 import p1b1
 import candle
-
+from keras_utils import PerformanceReportCallback
 np.set_printoptions(precision=4)
 
 
@@ -324,6 +324,8 @@ def warmup_scheduler(epoch):
         callbacks.append(checkpointer)
     if params['tb']:
         callbacks.append(tensorboard)
+    perf_callback = PerformanceReportCallback(params['batch_size'])
+    callbacks.append(perf_callback)
 
     x_val2 = np.copy(x_val)
     np.random.shuffle(x_val2)

Pilot1/P1B2/p1b2_baseline_keras2.py

@@ -9,7 +9,7 @@
 
 import p1b2
 import candle
-
+from keras_utils import PerformanceReportCallback
 
 def initialize_parameters(default_model='p1b2_default_model.txt'):
 
@@ -109,10 +109,12 @@ def run(gParameters):
     # Seed random generator for training
     np.random.seed(seed)
 
+    perf_callback = PerformanceReportCallback(gParameters['batch_size'])
     mlp.fit(X_train, y_train,
             batch_size=gParameters['batch_size'],
             epochs=gParameters['epochs'],
-            validation_data=(X_val, y_val)
+            validation_data=(X_val, y_val),
+            callbacks=[perf_callback]
             )
 
     # model save

Pilot1/P1B3/p1b3_baseline_keras2.py

@@ -22,6 +22,7 @@
 import p1b3 as benchmark
 import candle
 import tensorflow as tf
+from keras_utils import PerformanceReportCallback
 tf.compat.v1.disable_eager_execution()
 
 
@@ -344,14 +345,15 @@ def run(gParameters):
     np.random.seed(seed)
 
     candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters)
+    perf_callback = PerformanceReportCallback(gParameters['batch_size'])
 
     # history = model.fit(train_gen, steps_per_epoch=train_steps, # this should be the deprecation fix
     history = model.fit(train_gen, steps_per_epoch=train_steps,
                         epochs=gParameters['epochs'],
                         validation_data=val_gen,
                         validation_steps=val_steps,
                         verbose=0,
-                        callbacks=[checkpointer, loss_history, progbar, candleRemoteMonitor],
+                        callbacks=[checkpointer, loss_history, progbar, candleRemoteMonitor, perf_callback],
                         )
     # callbacks=[checkpointer, loss_history, candleRemoteMonitor], # this just caused the job to hang on Biowulf
 

Pilot1/TC1/tc1_baseline_keras2.py

@@ -17,6 +17,7 @@
 
 import tc1 as bmk
 import candle
+from keras_utils import PerformanceReportCallback
 
 
 def initialize_parameters(default_model='tc1_default_model.txt'):
@@ -122,12 +123,13 @@ def run(gParameters):
     reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10,
                                   verbose=1, mode='auto', min_delta=0.0001, cooldown=0, min_lr=0)
 
+    perf_callback = PerformanceReportCallback(gParameters['batch_size'])
     history = model.fit(X_train, Y_train,
                         batch_size=gParameters['batch_size'],
                         epochs=gParameters['epochs'],
                         verbose=1,
                         validation_data=(X_test, Y_test),
-                        callbacks=[checkpointer, csv_logger, reduce_lr])
+                        callbacks=[checkpointer, csv_logger, reduce_lr, perf_callback])
 
     score = model.evaluate(X_test, Y_test, verbose=0)
 

Pilot2/P2B1/p2b1.py

@@ -334,7 +334,7 @@ def train_ac(self):
                 # for frame in random.sample(range(len(xt_all)), int(self.sampling_density*len(xt_all))):
                 for frame in range(len(xt_all)):
                     history = self.molecular_model.fit(xt_all[frame], yt_all[frame], epochs=1,
-                                                       batch_size=self.batch_size, callbacks=self.callbacks[:2])
+                                                       batch_size=self.batch_size, callbacks=self.callbacks)
                     frame_loss.append(history.history['loss'])
                     frame_mse.append(history.history['mean_squared_error'])
 

Pilot2/P2B1/p2b1_baseline_keras2.py

@@ -23,6 +23,7 @@
 import candle
 
 import p2b1_AE_models as AE_models
+from keras_utils import PerformanceReportCallback
 
 HOME = os.environ['HOME']
 
@@ -249,7 +250,8 @@ def step_decay(epoch):
     history_logger = candle.LoggingCallback(logger.debug)
     candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP)
     timeoutMonitor = candle.TerminateOnTimeOut(TIMEOUT)
-    callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor]
+    perfCallback = PerformanceReportCallback(GP['batch_size'])
+    callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor, perfCallback]
     # loss = 0.
 
 # ### Save the Model to disk

Pilot3/P3B1/p3b1_baseline_keras2.py

@@ -11,7 +11,7 @@
 
 import p3b1 as bmk
 import candle
-
+from keras_utils import PerformanceReportCallback
 
 def initialize_parameters(default_model='p3b1_default_model.txt'):
 
@@ -160,10 +160,11 @@ def train_model(gParameters, models,
             gParameters['run_id'] = base_run_id + ".{}.{}.{}".format(fold, epoch, k)
             candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters)
             timeoutMonitor = candle.TerminateOnTimeOut(gParameters['timeout'])
+            perf_callback = PerformanceReportCallback(gParameters['batch_size'])
 
             model.fit({'input': X_train[k]}, {'out_' + str(k): Y_train[k]},
                       epochs=1, verbose=verbose,
-                      callbacks=[candleRemoteMonitor, timeoutMonitor],
+                      callbacks=[candleRemoteMonitor, timeoutMonitor, perf_callback],
                       batch_size=gParameters['batch_size'],
                       validation_data=(X_test[k], Y_test[k]))
 

Pilot3/P3B2/p3b2_baseline_keras2.py

@@ -1,3 +1,4 @@
+import tensorflow as tf
 import tensorflow.keras as keras
 from tensorflow.keras.models import Sequential
 from tensorflow.keras.layers import Dense, Activation
@@ -11,7 +12,7 @@
 
 import p3b2 as bmk
 import candle
-
+from keras_utils import PerformanceReportCallback
 
 def initialize_parameters(default_model='p3b2_default_model.txt'):
 
@@ -126,13 +127,33 @@ def run(gParameters):
             ret_seq = True
         else:
             ret_seq = False
-
-        if k == 0:
-            model.add(LSTM(rnn_size, input_shape=(maxlen, len(chars)), return_sequences=ret_seq,
-                           dropout=dropout, recurrent_dropout=recurrent_dropout))
+        if gParameters['itexlstm'] is not None and gParameters['itexlstm'] is True:
+            try:
+                import intel_extension_for_tensorflow as itex
+                print('Using ITEX-LSTM')
+                if k == 0:
+                    model.add(itex.ops.ItexLSTM(rnn_size, input_shape=(maxlen, len(chars)), return_sequences=ret_seq,
+                              dropout=dropout, recurrent_dropout=recurrent_dropout))
+                else:
+                    model.add(itex.ops.ItexLSTM(rnn_size, dropout=dropout, recurrent_dropout=recurrent_dropout,
+                              return_sequences=ret_seq))
+            except ImportError:
+                from tensorflow.keras.layers import LSTM
+                print('Using Keras-LSTM')
+                if k == 0:
+                    model.add(LSTM(rnn_size, input_shape=(maxlen, len(chars)), return_sequences=ret_seq,
+                                   dropout=dropout, recurrent_dropout=recurrent_dropout))
+                else:
+                    model.add(LSTM(rnn_size, dropout=dropout, recurrent_dropout=recurrent_dropout,
+                              return_sequences=ret_seq))
         else:
-            model.add(LSTM(rnn_size, dropout=dropout, recurrent_dropout=recurrent_dropout,
-                      return_sequences=ret_seq))
+            from tensorflow.keras.layers import LSTM
+            if k == 0:
+                model.add(LSTM(rnn_size, input_shape=(maxlen, len(chars)), return_sequences=ret_seq,
+                               dropout=dropout, recurrent_dropout=recurrent_dropout))
+            else:
+                model.add(LSTM(rnn_size, dropout=dropout, recurrent_dropout=recurrent_dropout,
+                          return_sequences=ret_seq))
 
     model.add(Dense(len(chars)))
     model.add(Activation(gParameters['activation']))
@@ -153,7 +174,8 @@ def run(gParameters):
             print('Iteration', iteration)
 
         history = LossHistory()
-        model.fit(X, y, batch_size=100, epochs=1, callbacks=[history])
+        perf_callback = PerformanceReportCallback(gParameters['batch_size'])
+        model.fit(X, y, batch_size=gParameters['batch_size'], epochs=gParameters['epochs'], callbacks=[history, perf_callback])
 
         loss = history.losses[-1]
         if verbose:

Pilot3/P3B2/p3b2_default_model.txt

@@ -2,6 +2,7 @@
 data_url = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P3B2/'
 train_data = 'P3B2_data.tgz'
 model_name = 'p3b2'
+batch_size = 100
 rnn_size = 64
 epochs = 2
 n_layers = 1
@@ -16,3 +17,4 @@ primetext = 'Diagnosis'
 length = 1000
 do_sample = True
 verbose = True
+itexlstm = True

Pilot3/P3B3/p3b3_baseline_keras2.py

@@ -17,7 +17,7 @@
 import p3b3 as bmk
 import keras_mt_shared_cnn
 import candle
-
+from keras_utils import PerformanceReportCallback
 
 def initialize_parameters(default_model='p3b3_default_model.txt'):
 
@@ -105,15 +105,15 @@ def run_cnn(GP, train_x, train_y, test_x, test_y,
 
     candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP)
     timeoutMonitor = candle.TerminateOnTimeOut(GP['timeout'])
-
+    perf_callback = PerformanceReportCallback(batch_size)
     history = cnn.fit(
         x=np.array(train_x),
         y=train_labels,
         batch_size=batch_size,
         epochs=epochs,
         verbose=2,
         validation_data=validation_data,
-        callbacks=[candleRemoteMonitor, timeoutMonitor]
+        callbacks=[candleRemoteMonitor, timeoutMonitor, perf_callback]
     )
 
     return history

Pilot3/P3B4/dense_attention.py

@@ -22,6 +22,7 @@
 from __future__ import division
 from __future__ import print_function
 
+import tensorflow as tf
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import smart_cond
@@ -172,7 +173,11 @@ def compute_mask(self, inputs, mask=None):
             q_mask = mask[0]
             if q_mask is None:
                 return None
-            return ops.convert_to_tensor_v2(q_mask)
+            if tf.__version__ < '2.13.0':
+              return ops.convert_to_tensor_v2(q_mask)
+            else:
+              from tensorflow.python.framework import tensor_conversion
+              return tensor_conversion.convert_to_tensor_v2(q_mask)
         return None
 
     def _validate_call_args(self, inputs, mask):