From 3d27013097dc3257ca8f8533f0476617ba61e05a Mon Sep 17 00:00:00 2001 From: Bartosz Kuncer Date: Sat, 9 Apr 2022 01:56:53 +0200 Subject: [PATCH 1/4] Add sorting of chunks to evaluation --- scripts/question_answering/run_squad.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/question_answering/run_squad.py b/scripts/question_answering/run_squad.py index 521ee15a47..a47c5231a5 100644 --- a/scripts/question_answering/run_squad.py +++ b/scripts/question_answering/run_squad.py @@ -837,6 +837,7 @@ def evaluate(args, last=True): logging.info('Prepare dev data') dev_features = get_squad_features(args, tokenizer, segment='dev') + dev_features.sort(key=lambda x: x.qas_id) dev_data_path = os.path.join(args.data_dir, 'dev-v{}.json'.format(args.version)) dataset_processor = SquadDatasetProcessor(tokenizer=tokenizer, doc_stride=args.doc_stride, @@ -848,6 +849,7 @@ def evaluate(args, last=True): chunk_features = dataset_processor.process_sample(feature) dev_all_chunk_features.extend(chunk_features) dev_chunk_feature_ptr.append(dev_chunk_feature_ptr[-1] + len(chunk_features)) + dev_all_chunk_features.sort(key=lambda x: x.valid_length) def eval_validation(ckpt_name, best_eval): """ @@ -912,6 +914,8 @@ def eval_validation(ckpt_name, best_eval): all_predictions = collections.OrderedDict() all_nbest_json = collections.OrderedDict() no_answer_score_json = collections.OrderedDict() + all_results.sort(key=lambda x: x.qas_id) + dev_all_chunk_features.sort(key=lambda x: x.qas_id) for index, (left_index, right_index) in enumerate(zip(dev_chunk_feature_ptr[:-1], dev_chunk_feature_ptr[1:])): chunked_features = dev_all_chunk_features[left_index:right_index] From fa89f9ea0ad6000a4fecc4c86ee811e95665cbd7 Mon Sep 17 00:00:00 2001 From: Bartosz Kuncer Date: Wed, 20 Apr 2022 12:49:13 +0200 Subject: [PATCH 2/4] Fix sorting chunks with quantization --- scripts/question_answering/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/question_answering/run_squad.py b/scripts/question_answering/run_squad.py index a47c5231a5..3dfabdac58 100644 --- a/scripts/question_answering/run_squad.py +++ b/scripts/question_answering/run_squad.py @@ -849,7 +849,7 @@ def evaluate(args, last=True): chunk_features = dataset_processor.process_sample(feature) dev_all_chunk_features.extend(chunk_features) dev_chunk_feature_ptr.append(dev_chunk_feature_ptr[-1] + len(chunk_features)) - dev_all_chunk_features.sort(key=lambda x: x.valid_length) + dev_all_chunk_features.sort(key=lambda x: x.valid_length, reverse=True) def eval_validation(ckpt_name, best_eval): """ From d1c078ded0d5af7adbfc45350bf425063536234e Mon Sep 17 00:00:00 2001 From: Bartosz Kuncer Date: Tue, 12 Jul 2022 16:28:39 +0200 Subject: [PATCH 3/4] Add sort flag --- .../commands/run_squad2_uncased_bert_base.sh | 3 +-- scripts/question_answering/run_squad.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh b/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh index ee3d8d0208..a66eef5bed 100644 --- a/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh +++ b/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh @@ -22,7 +22,7 @@ RUN_SQUAD_PATH=$(dirname "$0")/../run_squad.py # Run the script if [ ${USE_HOROVOD} -eq 0 ]; then - RUN_COMMAND="python3 ${RUN_SQUAD_PATH} --gpus 0,1,2,3" + RUN_COMMAND="python3 ${RUN_SQUAD_PATH} --gpus -1" else RUN_COMMAND="horovodrun -np 4 -H localhost:4 python3 ${RUN_SQUAD_PATH} --comm_backend horovod" fi @@ -32,7 +32,6 @@ ${RUN_COMMAND} \ --output_dir fintune_${MODEL_NAME}_squad_${VERSION} \ --version ${VERSION} \ --do_eval \ - --do_train \ --batch_size ${BATCH_SIZE} \ --num_accumulated ${NUM_ACCUMULATED} \ --layerwise_decay ${LAYERWISE_DECAY} \ diff --git a/scripts/question_answering/run_squad.py b/scripts/question_answering/run_squad.py index 3dfabdac58..01e1ae1f65 100644 --- a/scripts/question_answering/run_squad.py +++ b/scripts/question_answering/run_squad.py @@ -126,6 +126,8 @@ def parse_args(): 'may increase for mixed precision training on GPUs with TensorCores.') parser.add_argument('--overwrite_cache', action='store_true', help='Whether to overwrite the feature cache.') + parser.add_argument('--sort_input_data', action='store_true', + help='Whether to sort input data before evaluation.') # Evaluation hyperparameters parser.add_argument('--start_top_n', type=int, default=5, help='Number of start-position candidates') @@ -837,7 +839,8 @@ def evaluate(args, last=True): logging.info('Prepare dev data') dev_features = get_squad_features(args, tokenizer, segment='dev') - dev_features.sort(key=lambda x: x.qas_id) + if args.sort_input_data: + dev_features.sort(key=lambda x: x.qas_id) dev_data_path = os.path.join(args.data_dir, 'dev-v{}.json'.format(args.version)) dataset_processor = SquadDatasetProcessor(tokenizer=tokenizer, doc_stride=args.doc_stride, @@ -849,7 +852,8 @@ def evaluate(args, last=True): chunk_features = dataset_processor.process_sample(feature) dev_all_chunk_features.extend(chunk_features) dev_chunk_feature_ptr.append(dev_chunk_feature_ptr[-1] + len(chunk_features)) - dev_all_chunk_features.sort(key=lambda x: x.valid_length, reverse=True) + if args.sort_input_data: + dev_all_chunk_features.sort(key=lambda x: x.valid_length, reverse=True) def eval_validation(ckpt_name, best_eval): """ @@ -914,8 +918,9 @@ def eval_validation(ckpt_name, best_eval): all_predictions = collections.OrderedDict() all_nbest_json = collections.OrderedDict() no_answer_score_json = collections.OrderedDict() - all_results.sort(key=lambda x: x.qas_id) - dev_all_chunk_features.sort(key=lambda x: x.qas_id) + if args.sort_input_data: + all_results.sort(key=lambda x: x.qas_id) + dev_all_chunk_features.sort(key=lambda x: x.qas_id) for index, (left_index, right_index) in enumerate(zip(dev_chunk_feature_ptr[:-1], dev_chunk_feature_ptr[1:])): chunked_features = dev_all_chunk_features[left_index:right_index] From cf93fefc4fdc827c0204182632498014f1a2638b Mon Sep 17 00:00:00 2001 From: Bartosz Kuncer Date: Tue, 12 Jul 2022 16:29:39 +0200 Subject: [PATCH 4/4] Fix bad commit --- .../commands/run_squad2_uncased_bert_base.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh b/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh index a66eef5bed..ee3d8d0208 100644 --- a/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh +++ b/scripts/question_answering/commands/run_squad2_uncased_bert_base.sh @@ -22,7 +22,7 @@ RUN_SQUAD_PATH=$(dirname "$0")/../run_squad.py # Run the script if [ ${USE_HOROVOD} -eq 0 ]; then - RUN_COMMAND="python3 ${RUN_SQUAD_PATH} --gpus -1" + RUN_COMMAND="python3 ${RUN_SQUAD_PATH} --gpus 0,1,2,3" else RUN_COMMAND="horovodrun -np 4 -H localhost:4 python3 ${RUN_SQUAD_PATH} --comm_backend horovod" fi @@ -32,6 +32,7 @@ ${RUN_COMMAND} \ --output_dir fintune_${MODEL_NAME}_squad_${VERSION} \ --version ${VERSION} \ --do_eval \ + --do_train \ --batch_size ${BATCH_SIZE} \ --num_accumulated ${NUM_ACCUMULATED} \ --layerwise_decay ${LAYERWISE_DECAY} \